@@ -40,15 +40,6 @@ enum {
4040#define PP_FIELD_HINT_CONF 0.85
4141enum { PP_CSHARP_M_PREFIX_LEN = 2 };
4242
43- /* Source-retention caps for the parallel pipeline. The extract worker
44- * copies source bytes into result->arena so the fused cross-file LSP
45- * step in resolve_worker can run without re-reading from disk. Bound
46- * peak RSS with a per-file cap (skip retention for pathological huge
47- * generated files) and a total project-wide cap (skip when budget
48- * exhausted — cross-file LSP becomes a no-op for those late files,
49- * defs/calls already extracted are unaffected). */
50- #define PP_RETAIN_PER_FILE_MAX_BYTES (100LL * 1024 * 1024)
51- #define PP_RETAIN_TOTAL_BUDGET_BYTES (2LL * 1024 * 1024 * 1024)
5243#include "pipeline/pipeline.h"
5344#include "pipeline/pipeline_internal.h"
5445#include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */
@@ -78,6 +69,63 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
7869#include <string.h>
7970#include <time.h>
8071
72+ static size_t cbm_parallel_extract_default_total_cap (void ) {
73+ const size_t hard_max = 2ULL * 1024 * 1024 * 1024 ;
74+ size_t cap = cbm_mem_budget ();
75+ if (cap > 0 ) {
76+ cap /= 4 ;
77+ } else {
78+ cap = hard_max ;
79+ }
80+ if (cap > hard_max ) {
81+ cap = hard_max ;
82+ }
83+ return cap ;
84+ }
85+
86+ static size_t cbm_parallel_extract_default_per_file_cap (size_t total_cap ) {
87+ const size_t hard_max = 100ULL * 1024 * 1024 ;
88+ size_t cap = hard_max ;
89+ if (cap > total_cap ) {
90+ cap = total_cap ;
91+ }
92+ return cap ;
93+ }
94+
95+ static cbm_parallel_extract_opts_t cbm_parallel_extract_resolve_opts (
96+ const cbm_parallel_extract_opts_t * opts ) {
97+ const size_t hard_total_max = 2ULL * 1024 * 1024 * 1024 ;
98+ const size_t hard_per_file_max = 100ULL * 1024 * 1024 ;
99+ cbm_parallel_extract_opts_t resolved = {
100+ .retain_sources = true,
101+ .retain_total_budget_bytes = cbm_parallel_extract_default_total_cap (),
102+ .retain_per_file_max_bytes = 0 ,
103+ };
104+ resolved .retain_per_file_max_bytes =
105+ cbm_parallel_extract_default_per_file_cap (resolved .retain_total_budget_bytes );
106+ if (opts ) {
107+ if (opts -> retain_sources_set ) {
108+ resolved .retain_sources = opts -> retain_sources ;
109+ }
110+ if (opts -> retain_total_budget_bytes > 0 ) {
111+ resolved .retain_total_budget_bytes = opts -> retain_total_budget_bytes ;
112+ }
113+ if (opts -> retain_per_file_max_bytes > 0 ) {
114+ resolved .retain_per_file_max_bytes = opts -> retain_per_file_max_bytes ;
115+ }
116+ }
117+ if (resolved .retain_total_budget_bytes > hard_total_max ) {
118+ resolved .retain_total_budget_bytes = hard_total_max ;
119+ }
120+ if (resolved .retain_per_file_max_bytes > hard_per_file_max ) {
121+ resolved .retain_per_file_max_bytes = hard_per_file_max ;
122+ }
123+ if (resolved .retain_per_file_max_bytes > resolved .retain_total_budget_bytes ) {
124+ resolved .retain_per_file_max_bytes = resolved .retain_total_budget_bytes ;
125+ }
126+ return resolved ;
127+ }
128+
81129static uint64_t extract_now_ns (void ) {
82130 struct timespec ts ;
83131 cbm_clock_gettime (CLOCK_MONOTONIC , & ts );
@@ -494,6 +542,10 @@ typedef struct {
494542 _Atomic int next_file_idx ;
495543
496544 cbm_pkg_entries_t * pkg_entries ; /* per-worker manifest arrays (separate allocation) */
545+ bool retain_sources ;
546+ size_t retain_total_budget_bytes ;
547+ size_t retain_per_file_max_bytes ;
548+
497549 _Atomic int64_t retained_bytes ; /* total source bytes copied into result arenas */
498550} extract_ctx_t ;
499551
@@ -631,16 +683,14 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
631683 source_len , & ec -> pkg_entries [worker_id ]);
632684 }
633685
634- /* Retain source bytes in result->arena so the fused cross-file
635- * LSP step in resolve_worker can run without re-reading from
636- * disk. Capped per-file (PP_RETAIN_PER_FILE_MAX_BYTES) and
637- * globally (PP_RETAIN_TOTAL_BUDGET_BYTES) to bound peak RSS.
638- * Skipping retention just means cross-file LSP no-ops for this
639- * file — defs/calls already extracted are unaffected. */
640- if (source_len > 0 && (int64_t )source_len <= PP_RETAIN_PER_FILE_MAX_BYTES ) {
686+ /* Retain source bytes only when the resolved retention options allow it.
687+ * Skipping retention means the fused cross-file LSP step will no-op for
688+ * this file — defs/calls already extracted are unaffected. */
689+ if (ec -> retain_sources && source_len > 0 &&
690+ (size_t )source_len <= ec -> retain_per_file_max_bytes ) {
641691 int64_t prior = atomic_fetch_add_explicit (& ec -> retained_bytes , (int64_t )source_len ,
642692 memory_order_relaxed );
643- if (prior + (int64_t )source_len <= PP_RETAIN_TOTAL_BUDGET_BYTES ) {
693+ if (( size_t )( prior + (int64_t )source_len ) <= ec -> retain_total_budget_bytes ) {
644694 char * copy = (char * )cbm_arena_alloc (& result -> arena , (size_t )source_len + 1 );
645695 if (copy ) {
646696 memcpy (copy , source , (size_t )source_len );
@@ -719,15 +769,24 @@ static void log_extract_mem_stats(int worker_count) {
719769 }
720770}
721771
722- int cbm_parallel_extract (cbm_pipeline_ctx_t * ctx , const cbm_file_info_t * files , int file_count ,
723- CBMFileResult * * result_cache , _Atomic int64_t * shared_ids ,
724- int worker_count ) {
772+ int cbm_parallel_extract_ex (cbm_pipeline_ctx_t * ctx , const cbm_file_info_t * files , int file_count ,
773+ CBMFileResult * * result_cache , _Atomic int64_t * shared_ids ,
774+ int worker_count , const cbm_parallel_extract_opts_t * opts ) {
775+ cbm_parallel_extract_opts_t resolved_opts = cbm_parallel_extract_resolve_opts (opts );
776+
725777 if (file_count == 0 ) {
726778 return 0 ;
727779 }
728780
729781 cbm_log_info ("parallel.extract.start" , "files" , itoa_log (file_count ), "workers" ,
730782 itoa_log (worker_count ));
783+ {
784+ size_t mb = (size_t )CBM_SZ_1K * CBM_SZ_1K ;
785+ cbm_log_info ("parallel.extract.retention" , "retain_sources" ,
786+ resolved_opts .retain_sources ? "true" : "false" , "total_mb" ,
787+ itoa_log ((int )(resolved_opts .retain_total_budget_bytes / mb )), "per_file_mb" ,
788+ itoa_log ((int )(resolved_opts .retain_per_file_max_bytes / mb )));
789+ }
731790
732791 /* Log per-worker memory budget */
733792 if (cbm_mem_budget () > 0 ) {
@@ -748,7 +807,10 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
748807
749808 /* Sub-phase: Sort files by descending size for tail-latency reduction */
750809 CBM_PROF_START (t_sort );
751- file_sort_entry_t * sorted = malloc (file_count * sizeof (file_sort_entry_t ));
810+ file_sort_entry_t * sorted = malloc ((size_t )file_count * sizeof (file_sort_entry_t ));
811+ if (!sorted ) {
812+ return CBM_NOT_FOUND ;
813+ }
752814 for (int i = 0 ; i < file_count ; i ++ ) {
753815 sorted [i ].idx = i ;
754816 sorted [i ].size = files [i ].size ;
@@ -766,8 +828,12 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
766828 memset (workers , 0 , (size_t )worker_count * sizeof (extract_worker_state_t ));
767829
768830 /* Per-worker manifest entry arrays (separate from cache-line-aligned worker state) */
769- cbm_pkg_entries_t * pkg_entries = calloc (worker_count , sizeof (cbm_pkg_entries_t ));
770-
831+ cbm_pkg_entries_t * pkg_entries = calloc ((size_t )worker_count , sizeof (cbm_pkg_entries_t ));
832+ if (!pkg_entries ) {
833+ cbm_aligned_free (workers );
834+ free (sorted );
835+ return CBM_NOT_FOUND ;
836+ }
771837 extract_ctx_t ec = {
772838 .files = files ,
773839 .sorted = sorted ,
@@ -780,15 +846,18 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
780846 .shared_ids = shared_ids ,
781847 .cancelled = ctx -> cancelled ,
782848 .pkg_entries = pkg_entries ,
849+ .retain_sources = resolved_opts .retain_sources ,
850+ .retain_total_budget_bytes = resolved_opts .retain_total_budget_bytes ,
851+ .retain_per_file_max_bytes = resolved_opts .retain_per_file_max_bytes ,
783852 };
784853 atomic_init (& ec .next_worker_id , 0 );
785854 atomic_init (& ec .next_file_idx , 0 );
786855 atomic_init (& ec .retained_bytes , 0 );
787856
788857 /* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
789858 CBM_PROF_START (t_dispatch );
790- cbm_parallel_for_opts_t opts = {.max_workers = worker_count , .force_pthreads = false};
791- cbm_parallel_for (worker_count , extract_worker , & ec , opts );
859+ cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count , .force_pthreads = false};
860+ cbm_parallel_for (worker_count , extract_worker , & ec , parallel_opts );
792861 CBM_PROF_END_N ("parallel_extract" , "3_dispatch_workers_parallel" , t_dispatch , file_count );
793862
794863 /* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */
@@ -821,7 +890,12 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
821890 return 0 ;
822891}
823892
824- /* ── Phase 3B: Serial Registry Build ─────────────────────────────── */
893+ int cbm_parallel_extract (cbm_pipeline_ctx_t * ctx , const cbm_file_info_t * files , int file_count ,
894+ CBMFileResult * * result_cache , _Atomic int64_t * shared_ids ,
895+ int worker_count ) {
896+ return cbm_parallel_extract_ex (ctx , files , file_count , result_cache , shared_ids , worker_count ,
897+ NULL );
898+ }
825899
826900/* Register one definition and create DEFINES + DEFINES_METHOD edges. Returns edge count. */
827901static int register_and_link_def (cbm_pipeline_ctx_t * ctx , const CBMDefinition * def , const char * rel ,
@@ -2270,18 +2344,25 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
22702344 * Runs BEFORE resolve_file_calls so its additions to
22712345 * result->resolved_calls are picked up by
22722346 * cbm_pipeline_find_lsp_resolution when calls become CALLS
2273- * edges. Requires result-> source to have been retained in
2274- * result->arena during extract (PP_RETAIN_*); files over the
2275- * cap or past the budget have result->source==NULL and are
2276- * counted as skipped_no_source — defs/calls already in the
2277- * extract are unaffected .
2347+ * edges. Prefer source bytes retained in result->arena; if
2348+ * the low-RAM retention cap skipped this file, fall back to a
2349+ * bounded per-file read and free it immediately after the LSP
2350+ * call. This keeps project-wide retention bounded without
2351+ * dropping cross-file LSP correctness for unretained files .
22782352 *
22792353 * Slab reclaim afterward: the LSP re-parses via tree-sitter,
22802354 * which allocates through this worker's TLS slab. Reclaiming
22812355 * here keeps the slab high-water bounded as the resolve phase
22822356 * walks across thousands of files in a single worker thread. */
22832357 if (cross_lsp_eligible ) {
2284- if (result -> source && result -> source_len > 0 ) {
2358+ char * lsp_source_owned = NULL ;
2359+ const char * lsp_source = result -> source ;
2360+ int lsp_source_len = result -> source_len ;
2361+ if ((!lsp_source || lsp_source_len <= 0 ) && rc -> files [file_idx ].path ) {
2362+ lsp_source_owned = read_file (rc -> files [file_idx ].path , & lsp_source_len );
2363+ lsp_source = lsp_source_owned ;
2364+ }
2365+ if (lsp_source && lsp_source_len > 0 ) {
22852366 const char * def_module = rc -> def_modules ? rc -> def_modules [file_idx ] : module_qn ;
22862367
22872368 uint64_t lsp_t0 = extract_now_ns ();
@@ -2315,25 +2396,24 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
23152396 }
23162397 case CBM_LANG_PYTHON :
23172398 cbm_run_py_lsp_cross_with_registry (
2318- & result -> arena , result -> source , result -> source_len , def_module ,
2319- prebuilt , imp_keys , imp_vals , imp_count , result -> cached_tree ,
2399+ & result -> arena , lsp_source , lsp_source_len , def_module , prebuilt ,
2400+ imp_keys , imp_vals , imp_count , result -> cached_tree ,
23202401 & result -> resolved_calls );
23212402 used_prebuilt = true;
23222403 break ;
23232404 case CBM_LANG_C :
23242405 case CBM_LANG_CPP :
23252406 case CBM_LANG_CUDA :
23262407 cbm_run_c_lsp_cross_with_registry (
2327- & result -> arena , result -> source , result -> source_len , def_module ,
2408+ & result -> arena , lsp_source , lsp_source_len , def_module ,
23282409 (lang != CBM_LANG_C ), prebuilt , imp_keys , imp_vals , imp_count ,
23292410 result -> cached_tree , & result -> resolved_calls );
23302411 used_prebuilt = true;
23312412 break ;
23322413 case CBM_LANG_CSHARP :
2333- cbm_run_cs_lsp_cross_with_registry (& result -> arena , result -> source ,
2334- result -> source_len , def_module , prebuilt ,
2335- imp_vals , imp_count , result -> cached_tree ,
2336- & result -> resolved_calls );
2414+ cbm_run_cs_lsp_cross_with_registry (
2415+ & result -> arena , lsp_source , lsp_source_len , def_module , prebuilt ,
2416+ imp_vals , imp_count , result -> cached_tree , & result -> resolved_calls );
23372417 used_prebuilt = true;
23382418 break ;
23392419 case CBM_LANG_JAVASCRIPT :
@@ -2361,8 +2441,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
23612441 }
23622442 }
23632443 cbm_run_ts_lsp_cross_with_registry (
2364- & result -> arena , result -> source , result -> source_len , def_module , js , jsx ,
2365- dts , prebuilt , ts_defs , ts_def_count , imp_keys , imp_vals , imp_count ,
2444+ & result -> arena , lsp_source , lsp_source_len , def_module , js , jsx , dts ,
2445+ prebuilt , ts_defs , ts_def_count , imp_keys , imp_vals , imp_count ,
23662446 result -> cached_tree , & result -> resolved_calls );
23672447 free (ts_filtered );
23682448 used_prebuilt = true;
@@ -2394,16 +2474,16 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
23942474 lang == CBM_LANG_TSX ) {
23952475 bool js , jsx , dts ;
23962476 cbm_pxc_ts_modes (lang , rel , & js , & jsx , & dts );
2397- cbm_pxc_run_one_ts (result , result -> source , result -> source_len , def_module ,
2477+ cbm_pxc_run_one_ts (result , lsp_source , lsp_source_len , def_module ,
23982478 file_defs , file_def_count , imp_keys , imp_vals , imp_count ,
23992479 js , jsx , dts );
24002480 } else {
2401- cbm_pxc_run_one (lang , result , result -> source , result -> source_len ,
2402- def_module , file_defs , file_def_count , imp_keys , imp_vals ,
2403- imp_count );
2481+ cbm_pxc_run_one (lang , result , lsp_source , lsp_source_len , def_module ,
2482+ file_defs , file_def_count , imp_keys , imp_vals , imp_count );
24042483 }
24052484 }
24062485 free (filtered );
2486+ free_source (lsp_source_owned );
24072487 /* Contract: cbm_slab_reclaim() requires the thread parser to be
24082488 * destroyed first; otherwise its lexer holds slab pointers
24092489 * (lexer.included_ranges) that get freed underneath it, causing
0 commit comments