Skip to content

Commit 9b85dd3

Browse files
authored
Merge pull request DeusData#2180 from DeusData/fix/extract-cpu-time-budget
fix(extract): budget tree-sitter parses by thread CPU time, not wall clock
2 parents 713764b + 5270162 commit 9b85dd3

4 files changed

Lines changed: 120 additions & 5 deletions

File tree

‎internal/cbm/cbm.c‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -213,9 +213,42 @@ static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point,
213213

214214
// --- Parse timeout callback ---
215215

216+
/* Budget for the tree-sitter progress callback. The PRIMARY gate is per-thread
217+
* CPU time: a worker descheduled under CI contention burns WALL time but not
218+
* CPU, so a starved-but-parseable file must NOT be abandoned merely because the
219+
* 5 s budget elapsed in wall-clock against near-zero CPU. That false "parse
220+
* timeout" silently dropped a file's defs and, with them, every cross-file edge
221+
* those defs anchored (e.g. a Celery.send_task definition backing an ASYNC_CALLS
222+
* edge). A generous WALL ceiling stays as a backstop so a genuinely
223+
* stuck/spinning parse still terminates in bounded time. */
224+
#define CBM_PARSE_WALL_CEILING_FACTOR 12ULL /* ~60 s ceiling for the 5 s CPU budget */
225+
226+
typedef struct {
227+
uint64_t cpu_deadline_ns; // trip once this thread's CPU time passes it
228+
uint64_t wall_ceiling_ns; // hard wall backstop for a spinning/stuck parse
229+
} CBMParseBudget;
230+
231+
#ifdef CBM_ENABLE_TEST_SEAMS
232+
/* Deterministic RED-repro seam (armed by CBM_TEST_WALL_STALL_ON in
233+
* cbm_extract_file_ex): when set, the timeout callback reads the WALL clock this
234+
* many ns ahead of reality while CPU time is untouched — emulating a worker
235+
* descheduled long enough for the old wall-only budget to elapse against
236+
* near-zero CPU. With the CPU-time budget the parse still completes; a wall-only
237+
* / tight-ceiling budget drops the file. Thread-local so it cannot leak across
238+
* worker threads. Compiled only into seam-enabled test artifacts. */
239+
static CBM_TLS uint64_t tl_parse_wall_seam_offset_ns = 0;
240+
#endif
241+
242+
/* tree-sitter's TSProgressCallback mandates a non-const TSParseState*; a const
243+
* parameter here would not match the opts.progress_callback assignment below. */
244+
// cppcheck-suppress constParameterCallback
216245
static bool cbm_timeout_cb(TSParseState *state) {
217-
uint64_t deadline = *(uint64_t *)state->payload;
218-
return now_ns() > deadline;
246+
const CBMParseBudget *budget = (const CBMParseBudget *)state->payload;
247+
uint64_t wall = now_ns();
248+
#ifdef CBM_ENABLE_TEST_SEAMS
249+
wall += tl_parse_wall_seam_offset_ns;
250+
#endif
251+
return cbm_thread_cpu_time_ns() > budget->cpu_deadline_ns || wall > budget->wall_ceiling_ns;
219252
}
220253

221254
// --- Thread-local parser pool ---
@@ -1620,11 +1653,26 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
16201653
};
16211654

16221655
TSParseOptions opts = {0};
1623-
uint64_t deadline_ns = 0; // cppcheck-suppress unreadVariable
1656+
CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable
16241657
if (timeout_micros > 0) {
1625-
deadline_ns = t0 + ((uint64_t)timeout_micros * USEC_TO_NSEC);
1626-
opts.payload = &deadline_ns;
1658+
uint64_t budget_ns = (uint64_t)timeout_micros * USEC_TO_NSEC;
1659+
// Descheduling burns wall time but not CPU: gate on this thread's CPU
1660+
// time so a starved-but-parseable file is not abandoned, with a generous
1661+
// wall ceiling as a backstop against a genuinely spinning/stuck parse.
1662+
budget.cpu_deadline_ns = cbm_thread_cpu_time_ns() + budget_ns;
1663+
budget.wall_ceiling_ns = t0 + budget_ns * CBM_PARSE_WALL_CEILING_FACTOR;
1664+
opts.payload = &budget;
16271665
opts.progress_callback = cbm_timeout_cb;
1666+
#ifdef CBM_ENABLE_TEST_SEAMS
1667+
tl_parse_wall_seam_offset_ns = 0;
1668+
const char *stall_on = getenv("CBM_TEST_WALL_STALL_ON");
1669+
if (stall_on && stall_on[0] && rel_path && strstr(rel_path, stall_on)) {
1670+
// Push the wall reading past the 1x budget (the old wall-only budget
1671+
// trips) but well under the generous ceiling (the CPU-time budget
1672+
// survives): budget + 1 s, deterministic, no real timing involved.
1673+
tl_parse_wall_seam_offset_ns = budget_ns + NSEC_PER_SEC;
1674+
}
1675+
#endif
16281676
}
16291677

16301678
TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts);

‎src/foundation/compat.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,30 @@ int cbm_clock_gettime(int clk_id, struct timespec *tp) {
316316
}
317317
#endif
318318

319+
/* ── Per-thread CPU time ──────────────────────────────────────── */
320+
321+
uint64_t cbm_thread_cpu_time_ns(void) {
322+
#ifdef _WIN32
323+
FILETIME creation, exit_time, kernel, user;
324+
if (!GetThreadTimes(GetCurrentThread(), &creation, &exit_time, &kernel, &user)) {
325+
return 0;
326+
}
327+
/* kernel/user each carry a 64-bit count of 100-ns ticks split across a
328+
* FILETIME's two DWORDs; recombine, sum, and scale to nanoseconds. The
329+
* shift-combine avoids casting to ULARGE_INTEGER (alignment-unsafe per
330+
* MSDN, and its union members trip cppcheck's unreadVariable). */
331+
uint64_t kernel_ticks = ((uint64_t)kernel.dwHighDateTime << 32) | kernel.dwLowDateTime;
332+
uint64_t user_ticks = ((uint64_t)user.dwHighDateTime << 32) | user.dwLowDateTime;
333+
return (kernel_ticks + user_ticks) * 100ULL;
334+
#else
335+
struct timespec ts;
336+
if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) {
337+
return 0;
338+
}
339+
return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec;
340+
#endif
341+
}
342+
319343
/* ── getline (Windows lacks it) ───────────────────────────────── */
320344

321345
#ifdef _WIN32

‎src/foundation/compat.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <errno.h>
1212
#include <stdbool.h>
1313
#include <stddef.h>
14+
#include <stdint.h>
1415
#include <stdio.h>
1516
/* stdlib.h declares getenv (cbm_tmpdir) and, on Windows, _putenv_s (cbm_setenv/
1617
* cbm_unsetenv). The x86-64 mingw toolchain pulled it in transitively, but the
@@ -99,6 +100,16 @@ static inline int cbm_nanosleep(const struct timespec *req, struct timespec *rem
99100
/* Sleeps for the full requested duration even when POSIX signals interrupt it. */
100101
int cbm_nanosleep_full(const struct timespec *req);
101102

103+
/* Per-CALLING-THREAD CPU time in nanoseconds. Unlike cbm_clock_gettime — which
104+
* measures WALL time on every platform (QueryPerformanceCounter on Windows) —
105+
* this advances only while the calling thread is actually scheduled on a CPU, so
106+
* a descheduled/starved thread accrues (almost) none. Use it to budget work
107+
* against real CPU consumed rather than wall-clock, which a contended host
108+
* inflates without the thread doing any work. POSIX (incl. macOS 10.12+):
109+
* CLOCK_THREAD_CPUTIME_ID. Windows: GetThreadTimes kernel+user. Returns 0 if the
110+
* platform clock is unavailable. Implemented in compat.c. */
111+
uint64_t cbm_thread_cpu_time_ns(void);
112+
102113
/* ── gmtime_r (Windows lacks it) ─────────────────────────────── */
103114
#ifdef _WIN32
104115
static inline struct tm *cbm_gmtime_r(const time_t *timep, struct tm *result) {

‎tests/test_edge_types_probe.c‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -842,6 +842,37 @@ TEST(async_calls_celery_python) {
842842
PASS();
843843
}
844844

845+
/* RED-repro: a wall-clock / scheduling artifact must NOT drop a parseable
846+
* file's defs. The CBM_TEST_WALL_STALL_ON seam forces the parse-timeout callback
847+
* to read the WALL clock as (budget + 1 s) ahead while CPU time is untouched —
848+
* emulating a worker descheduled under CI contention. celery/app.py DEFINES
849+
* Celery.send_task; under the old wall-only 5 s budget that file is abandoned
850+
* with zero defs, the "celery" QN the caller resolves against vanishes, and the
851+
* ASYNC_CALLS edge count drops to 0 (the intermittent async_calls_celery_python
852+
* failure). With the CPU-time budget the parse still completes (CPU under budget,
853+
* wall under the generous ceiling) and the edge survives. Deterministic: the
854+
* seam is a fixed offset, never real timing. Binds only under CBM_ENABLE_TEST_SEAMS
855+
* (always set for the test-runner); a no-seam build exercises the plain edge. */
856+
TEST(async_calls_celery_wall_stall_seam) {
857+
static const EtFile f[] = {
858+
{"celery/app.py",
859+
"class Celery:\n"
860+
" def task(self):\n def decorator(fn): return fn\n return decorator\n\n"
861+
" def send_task(self, name, args=None): return (name, args)\n\n"
862+
"app = Celery()\n"},
863+
{"tasks/order_tasks.py",
864+
"from celery.app import app\n\n\n"
865+
"def dispatch_order_created(order_id):\n"
866+
" return app.send_task('order_created', args=[order_id])\n\n\n"
867+
"def dispatch_order_shipped(order_id):\n"
868+
" return app.send_task('order_shipped', args=[order_id])\n"}};
869+
cbm_setenv("CBM_TEST_WALL_STALL_ON", "celery/app.py", 1);
870+
int ok = et_edge_present(f, 2, "ASYNC_CALLS", 1);
871+
cbm_unsetenv("CBM_TEST_WALL_STALL_ON");
872+
ASSERT_TRUE(ok);
873+
PASS();
874+
}
875+
845876
/* Sidekiq (Ruby) — "Sidekiq" substring in resolved QN */
846877
TEST(async_calls_sidekiq_ruby) {
847878
static const EtFile f[] = {
@@ -1625,6 +1656,7 @@ SUITE(edge_types_probe) {
16251656

16261657
/* ASYNC_CALLS — message queue dispatch (5 brokers × languages) */
16271658
RUN_TEST(async_calls_celery_python);
1659+
RUN_TEST(async_calls_celery_wall_stall_seam);
16281660
RUN_TEST(async_calls_sidekiq_ruby);
16291661
RUN_TEST(async_calls_kafkajs_ts);
16301662
RUN_TEST(async_calls_sqs_go);

0 commit comments

Comments
 (0)