Skip to content

Commit 1a8ff1a

Browse files
DeusDataDeusData
authored
Merge pull request DeusData#890 from dpersek/codex/issue-860-test-tsan
ci: wire the test-tsan target
2 parents 3747344 + 22506b9 commit 1a8ff1a

2 files changed

Lines changed: 67 additions & 6 deletions

File tree

‎.github/workflows/_test.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,18 @@ jobs:
8888
env:
8989
CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}
9090

91+
test-tsan:
92+
runs-on: ubuntu-latest
93+
timeout-minutes: 120
94+
steps:
95+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
96+
97+
- name: Install deps (Ubuntu)
98+
run: sudo apt-get update && sudo apt-get install -y clang zlib1g-dev
99+
100+
- name: ThreadSanitizer tests
101+
run: make -f Makefile.cbm test-tsan CC=clang CXX=clang++
102+
91103
test-windows:
92104
needs: setup-matrix
93105
strategy:

‎Makefile.cbm‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,11 @@ CFLAGS_TEST = $(CFLAGS_COMMON) -g -O1 $(SANITIZE)
6767
CXXFLAGS_TEST = $(CXXFLAGS_COMMON) -g -O1 $(SANITIZE)
6868

6969
# TSan (can't combine with ASan)
70+
TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer
7071
CFLAGS_TSAN = $(CFLAGS_COMMON) -g -O1 \
71-
-fsanitize=thread -fno-omit-frame-pointer
72+
$(TSAN_SANITIZE)
7273
CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -g -O1 \
73-
-fsanitize=thread -fno-omit-frame-pointer
74+
$(TSAN_SANITIZE)
7475

7576
# Windows needs ws2_32 (Winsock), psapi (GetProcessMemoryInfo), shell32
7677
# (CommandLineToArgvW — the UTF-8 argv path in main.c, #423/#20), and
@@ -89,7 +90,7 @@ endif
8990

9091
LDFLAGS = -lm -lstdc++ -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS)
9192
LDFLAGS_TEST = -lm -lstdc++ -lpthread -lz $(SANITIZE) $(WIN32_LIBS)
92-
LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz -fsanitize=thread $(WIN32_LIBS)
93+
LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS)
9394

9495
# ── Source files ─────────────────────────────────────────────────
9596

@@ -484,13 +485,17 @@ GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE)
484485
GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
485486
$(SANITIZE)
486487
GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
487-
-fsanitize=thread -fno-omit-frame-pointer
488+
$(TSAN_SANITIZE)
488489

489490
# Object files for grammars + ts_runtime + lsp_all + preprocessor
490491
GRAMMAR_OBJS_TEST = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/%.o,$(GRAMMAR_SRCS))
491492
TS_RUNTIME_OBJ_TEST = $(BUILD_DIR)/ts_runtime.o
492493
LSP_OBJ_TEST = $(BUILD_DIR)/lsp_all.o
493494
PP_OBJ_TEST = $(BUILD_DIR)/preprocessor.o
495+
GRAMMAR_OBJS_TSAN = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/tsan_%.o,$(GRAMMAR_SRCS))
496+
TS_RUNTIME_OBJ_TSAN = $(BUILD_DIR)/tsan_ts_runtime.o
497+
LSP_OBJ_TSAN = $(BUILD_DIR)/tsan_lsp_all.o
498+
PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
494499

495500
# ── Targets ──────────────────────────────────────────────────────
496501

@@ -521,60 +526,94 @@ $(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
521526
$(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
522527
$(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $<
523528

529+
$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR)
530+
$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
531+
532+
$(BUILD_DIR)/tsan_ts_runtime.o: $(CBM_DIR)/ts_runtime.c | $(BUILD_DIR)
533+
$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
534+
535+
$(BUILD_DIR)/tsan_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
536+
$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
537+
538+
$(BUILD_DIR)/tsan_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
539+
$(CXX) $(CXXFLAGS_TSAN) -w -I$(CBM_DIR)/vendored -c -o $@ $<
540+
524541
# ── Full test binary ─────────────────────────────────────────────
525542

526543
# mimalloc object files
527544
MIMALLOC_OBJ_TEST = $(BUILD_DIR)/mimalloc.o
545+
MIMALLOC_OBJ_TSAN = $(BUILD_DIR)/tsan_mimalloc.o
528546
MIMALLOC_OBJ_PROD = $(BUILD_DIR)/prod_mimalloc.o
529547

530548
$(BUILD_DIR)/mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
531549
$(CC) $(MIMALLOC_CFLAGS_TEST) -c -o $@ $<
532550

551+
$(BUILD_DIR)/tsan_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
552+
$(CC) $(MIMALLOC_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $<
553+
533554
$(BUILD_DIR)/prod_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
534555
$(CC) $(MIMALLOC_CFLAGS) -c -o $@ $<
535556

536557
# sqlite3 object files (vendored amalgamation)
537558
SQLITE3_OBJ_TEST = $(BUILD_DIR)/sqlite3.o
559+
SQLITE3_OBJ_TSAN = $(BUILD_DIR)/tsan_sqlite3.o
538560
SQLITE3_OBJ_PROD = $(BUILD_DIR)/prod_sqlite3.o
539561

540562
$(BUILD_DIR)/sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
541563
$(CC) $(SQLITE3_CFLAGS_TEST) $(SANITIZE) -c -o $@ $<
542564

565+
$(BUILD_DIR)/tsan_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
566+
$(CC) $(SQLITE3_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $<
567+
543568
$(BUILD_DIR)/prod_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
544569
$(CC) $(SQLITE3_CFLAGS) -c -o $@ $<
545570

546571
# TRE regex (only compiled on Windows — POSIX uses system <regex.h>)
547572
TRE_OBJ_TEST :=
573+
TRE_OBJ_TSAN :=
548574
TRE_OBJ_PROD :=
549575
ifeq ($(IS_MINGW),yes)
550576
TRE_OBJ_TEST = $(BUILD_DIR)/tre.o
577+
TRE_OBJ_TSAN = $(BUILD_DIR)/tsan_tre.o
551578
TRE_OBJ_PROD = $(BUILD_DIR)/prod_tre.o
552579

553580
$(BUILD_DIR)/tre.o: $(TRE_SRC) | $(BUILD_DIR)
554581
$(CC) $(TRE_CFLAGS) $(SANITIZE) -fno-sanitize=alignment -c -o $@ $<
555582

583+
$(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR)
584+
$(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $<
585+
556586
$(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR)
557587
$(CC) $(TRE_CFLAGS) -O2 -c -o $@ $<
558588
endif
559589

560590
# Vendored LZ4 (test build)
561591
LZ4_OBJ_TEST = $(BUILD_DIR)/test_lz4.o $(BUILD_DIR)/test_lz4hc.o
592+
LZ4_OBJ_TSAN = $(BUILD_DIR)/tsan_lz4.o $(BUILD_DIR)/tsan_lz4hc.o
562593
$(BUILD_DIR)/test_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
563594
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR) -c -o $@ $<
564595
$(BUILD_DIR)/test_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
565596
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
597+
$(BUILD_DIR)/tsan_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
598+
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR) -c -o $@ $<
599+
$(BUILD_DIR)/tsan_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
600+
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
566601

567602
# Vendored zstd (test build)
568603
ZSTD_OBJ_TEST = $(BUILD_DIR)/test_zstd.o
604+
ZSTD_OBJ_TSAN = $(BUILD_DIR)/tsan_zstd.o
569605
$(BUILD_DIR)/test_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
570606
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
607+
$(BUILD_DIR)/tsan_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
608+
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
571609

572610
# nomic-embed-code pretrained vector blob
573611
UNIXCODER_OBJ = $(BUILD_DIR)/unixcoder_blob.o
574612
$(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUILD_DIR)
575613
$(CC) -c -o $@ $<
576614

577615
OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ)
616+
OBJS_VENDORED_TSAN = $(MIMALLOC_OBJ_TSAN) $(SQLITE3_OBJ_TSAN) $(TRE_OBJ_TSAN) $(GRAMMAR_OBJS_TSAN) $(TS_RUNTIME_OBJ_TSAN) $(LSP_OBJ_TSAN) $(PP_OBJ_TSAN) $(LZ4_OBJ_TSAN) $(ZSTD_OBJ_TSAN) $(UNIXCODER_OBJ)
578617

579618
$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR)
580619
$(CC) $(CFLAGS_TEST) -Itests -Itests/repro -o $@ \
@@ -602,8 +641,18 @@ test-repro: $(BUILD_DIR)/test-repro-runner
602641

603642
# ── TSan full test ───────────────────────────────────────────────
604643

605-
test-tsan:
606-
@echo "TSan not yet wired for full extraction tests"
644+
TEST_TSAN_SUITES ?= mem slab_alloc parallel
645+
TSAN_OPTIONS ?= halt_on_error=1
646+
647+
$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) | $(BUILD_DIR)
648+
$(CC) $(CFLAGS_TSAN) -Itests -Itests/repro -o $@ \
649+
$(ALL_TEST_SRCS) $(PROD_SRCS) \
650+
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
651+
$(OBJS_VENDORED_TSAN) \
652+
$(LDFLAGS_TSAN)
653+
654+
test-tsan: $(BUILD_DIR)/test-runner-tsan
655+
cd $(CURDIR) && TSAN_OPTIONS="$(TSAN_OPTIONS)" $(BUILD_DIR)/test-runner-tsan $(TEST_TSAN_SUITES)
607656

608657
# ── Production binary ────────────────────────────────────────────
609658

0 commit comments

Comments
 (0)