Skip to content

Commit 3d5af94

Browse files
committed
ci: wire test-tsan target
Signed-off-by: Dustin Persek <dustin.persek@protonmail.com>
1 parent affa223 commit 3d5af94

2 files changed

Lines changed: 67 additions & 6 deletions

File tree

‎.github/workflows/_test.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,18 @@ jobs:
8888
env:
8989
CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}
9090

91+
test-tsan:
92+
runs-on: ubuntu-latest
93+
timeout-minutes: 120
94+
steps:
95+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
96+
97+
- name: Install deps (Ubuntu)
98+
run: sudo apt-get update && sudo apt-get install -y clang zlib1g-dev
99+
100+
- name: ThreadSanitizer tests
101+
run: make -f Makefile.cbm test-tsan CC=clang CXX=clang++
102+
91103
test-windows:
92104
needs: setup-matrix
93105
strategy:

‎Makefile.cbm‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,11 @@ CFLAGS_TEST = $(CFLAGS_COMMON) -g -O1 $(SANITIZE)
6767
CXXFLAGS_TEST = $(CXXFLAGS_COMMON) -g -O1 $(SANITIZE)
6868

6969
# TSan (can't combine with ASan)
70+
TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer
7071
CFLAGS_TSAN = $(CFLAGS_COMMON) -g -O1 \
71-
-fsanitize=thread -fno-omit-frame-pointer
72+
$(TSAN_SANITIZE)
7273
CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -g -O1 \
73-
-fsanitize=thread -fno-omit-frame-pointer
74+
$(TSAN_SANITIZE)
7475

7576
# Windows needs ws2_32 (Winsock), psapi (GetProcessMemoryInfo), and
7677
# --allow-multiple-definition (MinGW CRT symbol clashes).
@@ -88,7 +89,7 @@ endif
8889

8990
LDFLAGS = -lm -lstdc++ -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS)
9091
LDFLAGS_TEST = -lm -lstdc++ -lpthread -lz $(SANITIZE) $(WIN32_LIBS)
91-
LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz -fsanitize=thread $(WIN32_LIBS)
92+
LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS)
9293

9394
# ── Source files ─────────────────────────────────────────────────
9495

@@ -480,13 +481,17 @@ GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE)
480481
GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
481482
$(SANITIZE)
482483
GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
483-
-fsanitize=thread -fno-omit-frame-pointer
484+
$(TSAN_SANITIZE)
484485

485486
# Object files for grammars + ts_runtime + lsp_all + preprocessor
486487
GRAMMAR_OBJS_TEST = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/%.o,$(GRAMMAR_SRCS))
487488
TS_RUNTIME_OBJ_TEST = $(BUILD_DIR)/ts_runtime.o
488489
LSP_OBJ_TEST = $(BUILD_DIR)/lsp_all.o
489490
PP_OBJ_TEST = $(BUILD_DIR)/preprocessor.o
491+
GRAMMAR_OBJS_TSAN = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/tsan_%.o,$(GRAMMAR_SRCS))
492+
TS_RUNTIME_OBJ_TSAN = $(BUILD_DIR)/tsan_ts_runtime.o
493+
LSP_OBJ_TSAN = $(BUILD_DIR)/tsan_lsp_all.o
494+
PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
490495

491496
# ── Targets ──────────────────────────────────────────────────────
492497

@@ -517,60 +522,94 @@ $(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
517522
$(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
518523
$(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $<
519524

525+
$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR)
526+
$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
527+
528+
$(BUILD_DIR)/tsan_ts_runtime.o: $(CBM_DIR)/ts_runtime.c | $(BUILD_DIR)
529+
$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
530+
531+
$(BUILD_DIR)/tsan_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
532+
$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
533+
534+
$(BUILD_DIR)/tsan_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
535+
$(CXX) $(CXXFLAGS_TSAN) -w -I$(CBM_DIR)/vendored -c -o $@ $<
536+
520537
# ── Full test binary ─────────────────────────────────────────────
521538

522539
# mimalloc object files
523540
MIMALLOC_OBJ_TEST = $(BUILD_DIR)/mimalloc.o
541+
MIMALLOC_OBJ_TSAN = $(BUILD_DIR)/tsan_mimalloc.o
524542
MIMALLOC_OBJ_PROD = $(BUILD_DIR)/prod_mimalloc.o
525543

526544
$(BUILD_DIR)/mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
527545
$(CC) $(MIMALLOC_CFLAGS_TEST) -c -o $@ $<
528546

547+
$(BUILD_DIR)/tsan_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
548+
$(CC) $(MIMALLOC_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $<
549+
529550
$(BUILD_DIR)/prod_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
530551
$(CC) $(MIMALLOC_CFLAGS) -c -o $@ $<
531552

532553
# sqlite3 object files (vendored amalgamation)
533554
SQLITE3_OBJ_TEST = $(BUILD_DIR)/sqlite3.o
555+
SQLITE3_OBJ_TSAN = $(BUILD_DIR)/tsan_sqlite3.o
534556
SQLITE3_OBJ_PROD = $(BUILD_DIR)/prod_sqlite3.o
535557

536558
$(BUILD_DIR)/sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
537559
$(CC) $(SQLITE3_CFLAGS_TEST) $(SANITIZE) -c -o $@ $<
538560

561+
$(BUILD_DIR)/tsan_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
562+
$(CC) $(SQLITE3_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $<
563+
539564
$(BUILD_DIR)/prod_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
540565
$(CC) $(SQLITE3_CFLAGS) -c -o $@ $<
541566

542567
# TRE regex (only compiled on Windows — POSIX uses system <regex.h>)
543568
TRE_OBJ_TEST :=
569+
TRE_OBJ_TSAN :=
544570
TRE_OBJ_PROD :=
545571
ifeq ($(IS_MINGW),yes)
546572
TRE_OBJ_TEST = $(BUILD_DIR)/tre.o
573+
TRE_OBJ_TSAN = $(BUILD_DIR)/tsan_tre.o
547574
TRE_OBJ_PROD = $(BUILD_DIR)/prod_tre.o
548575

549576
$(BUILD_DIR)/tre.o: $(TRE_SRC) | $(BUILD_DIR)
550577
$(CC) $(TRE_CFLAGS) $(SANITIZE) -fno-sanitize=alignment -c -o $@ $<
551578

579+
$(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR)
580+
$(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $<
581+
552582
$(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR)
553583
$(CC) $(TRE_CFLAGS) -O2 -c -o $@ $<
554584
endif
555585

556586
# Vendored LZ4 (test build)
557587
LZ4_OBJ_TEST = $(BUILD_DIR)/test_lz4.o $(BUILD_DIR)/test_lz4hc.o
588+
LZ4_OBJ_TSAN = $(BUILD_DIR)/tsan_lz4.o $(BUILD_DIR)/tsan_lz4hc.o
558589
$(BUILD_DIR)/test_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
559590
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR) -c -o $@ $<
560591
$(BUILD_DIR)/test_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
561592
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
593+
$(BUILD_DIR)/tsan_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
594+
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR) -c -o $@ $<
595+
$(BUILD_DIR)/tsan_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
596+
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
562597

563598
# Vendored zstd (test build)
564599
ZSTD_OBJ_TEST = $(BUILD_DIR)/test_zstd.o
600+
ZSTD_OBJ_TSAN = $(BUILD_DIR)/tsan_zstd.o
565601
$(BUILD_DIR)/test_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
566602
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
603+
$(BUILD_DIR)/tsan_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
604+
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
567605

568606
# nomic-embed-code pretrained vector blob
569607
UNIXCODER_OBJ = $(BUILD_DIR)/unixcoder_blob.o
570608
$(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUILD_DIR)
571609
$(CC) -c -o $@ $<
572610

573611
OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ)
612+
OBJS_VENDORED_TSAN = $(MIMALLOC_OBJ_TSAN) $(SQLITE3_OBJ_TSAN) $(TRE_OBJ_TSAN) $(GRAMMAR_OBJS_TSAN) $(TS_RUNTIME_OBJ_TSAN) $(LSP_OBJ_TSAN) $(PP_OBJ_TSAN) $(LZ4_OBJ_TSAN) $(ZSTD_OBJ_TSAN) $(UNIXCODER_OBJ)
574613

575614
$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR)
576615
$(CC) $(CFLAGS_TEST) -Itests -Itests/repro -o $@ \
@@ -598,8 +637,18 @@ test-repro: $(BUILD_DIR)/test-repro-runner
598637

599638
# ── TSan full test ───────────────────────────────────────────────
600639

601-
test-tsan:
602-
@echo "TSan not yet wired for full extraction tests"
640+
TEST_TSAN_SUITES ?= mem slab_alloc parallel
641+
TSAN_OPTIONS ?= halt_on_error=1
642+
643+
$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) | $(BUILD_DIR)
644+
$(CC) $(CFLAGS_TSAN) -Itests -Itests/repro -o $@ \
645+
$(ALL_TEST_SRCS) $(PROD_SRCS) \
646+
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
647+
$(OBJS_VENDORED_TSAN) \
648+
$(LDFLAGS_TSAN)
649+
650+
test-tsan: $(BUILD_DIR)/test-runner-tsan
651+
cd $(CURDIR) && TSAN_OPTIONS="$(TSAN_OPTIONS)" $(BUILD_DIR)/test-runner-tsan $(TEST_TSAN_SUITES)
603652

604653
# ── Production binary ────────────────────────────────────────────
605654

0 commit comments

Comments
 (0)