|
18 | 18 | #include "discover/discover.h" |
19 | 19 | #include "foundation/platform.h" |
20 | 20 | #include "foundation/log.h" |
| 21 | +#include "foundation/mem.h" |
21 | 22 | #include "cbm.h" |
22 | 23 | #include "result_spill.h" |
23 | 24 |
|
@@ -616,6 +617,148 @@ TEST(parallel_spill_mode_builds_the_same_graph) { |
616 | 617 | PASS(); |
617 | 618 | } |
618 | 619 |
|
| 620 | +/* ── #2184: the post-extraction phases are charged before they run ─── */ |
| 621 | + |
| 622 | +/* Order-independent fingerprint of a whole graph: every node (label, QN, file, |
| 623 | + * lines, properties) and every edge (type, source QN, target QN, properties) |
| 624 | + * hashed and folded with a sum and an xor. Two graphs that differ in any field |
| 625 | + * of any node or edge -- not just in a per-type count -- disagree here. */ |
| 626 | +typedef struct { |
| 627 | + const cbm_gbuf_t *gb; |
| 628 | + uint64_t sum; |
| 629 | + uint64_t xr; |
| 630 | + long count; |
| 631 | +} graph_fp_t; |
| 632 | + |
| 633 | +static uint64_t fp_mix(uint64_t h, const char *s) { |
| 634 | + for (const unsigned char *p = (const unsigned char *)(s ? s : "\x01"); *p; p++) { |
| 635 | + h = (h ^ *p) * 1099511628211ULL; /* FNV-1a */ |
| 636 | + } |
| 637 | + return (h ^ 0xffU) * 1099511628211ULL; /* field separator */ |
| 638 | +} |
| 639 | + |
| 640 | +static uint64_t fp_mix_int(uint64_t h, long long v) { |
| 641 | + char buf[32]; |
| 642 | + snprintf(buf, sizeof(buf), "%lld", v); |
| 643 | + return fp_mix(h, buf); |
| 644 | +} |
| 645 | + |
| 646 | +static void fp_fold(graph_fp_t *fp, uint64_t h) { |
| 647 | + fp->sum += h; |
| 648 | + fp->xr ^= h * 0x9E3779B97F4A7C15ULL; |
| 649 | + fp->count++; |
| 650 | +} |
| 651 | + |
| 652 | +static void fp_visit_node(const cbm_gbuf_node_t *n, void *ud) { |
| 653 | + uint64_t h = fp_mix(1469598103934665603ULL, "N"); |
| 654 | + h = fp_mix(h, n->label); |
| 655 | + h = fp_mix(h, n->qualified_name); |
| 656 | + h = fp_mix(h, n->file_path); |
| 657 | + h = fp_mix_int(h, n->start_line); |
| 658 | + h = fp_mix_int(h, n->end_line); |
| 659 | + h = fp_mix(h, n->properties_json); |
| 660 | + fp_fold(ud, h); |
| 661 | +} |
| 662 | + |
| 663 | +static void fp_visit_edge(const cbm_gbuf_edge_t *e, void *ud) { |
| 664 | + graph_fp_t *fp = ud; |
| 665 | + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(fp->gb, e->source_id); |
| 666 | + const cbm_gbuf_node_t *dst = cbm_gbuf_find_by_id(fp->gb, e->target_id); |
| 667 | + uint64_t h = fp_mix(1469598103934665603ULL, "E"); |
| 668 | + h = fp_mix(h, e->type); |
| 669 | + h = fp_mix(h, src ? src->qualified_name : NULL); |
| 670 | + h = fp_mix(h, dst ? dst->qualified_name : NULL); |
| 671 | + h = fp_mix(h, e->properties_json); |
| 672 | + fp_fold(fp, h); |
| 673 | +} |
| 674 | + |
| 675 | +static graph_fp_t graph_fingerprint(const cbm_gbuf_t *gb) { |
| 676 | + graph_fp_t fp = {.gb = gb}; |
| 677 | + cbm_gbuf_foreach_node(gb, fp_visit_node, &fp); |
| 678 | + cbm_gbuf_foreach_edge(gb, fp_visit_edge, &fp); |
| 679 | + return fp; |
| 680 | +} |
| 681 | + |
| 682 | +/* Mutator hook: runs between extraction and registry build, i.e. exactly where |
| 683 | + * the post-extraction phases start. Records how many results are still held |
| 684 | + * in memory there. */ |
| 685 | +static void count_cached_results(CBMFileResult **cache, int file_count, void *ud) { |
| 686 | + int n = 0; |
| 687 | + for (int i = 0; i < file_count; i++) { |
| 688 | + n += cache[i] != NULL; |
| 689 | + } |
| 690 | + *(int *)ud = n; |
| 691 | +} |
| 692 | + |
| 693 | +/* #2184: spill was entered only DURING extraction, when the charge crossed |
| 694 | + * budget - budget/16. A run that ended extraction just under that line kept |
| 695 | + * every result in memory, and the phases that cannot spill (registry build, |
| 696 | + * cross-LSP prepare, resolve) then grew the process past the budget -- openclaw |
| 697 | + * at 8 workers, 4079 MB budget: 3678 MB at extraction end, 5829 MB in resolve. |
| 698 | + * The extraction end now projects that growth from the result counts and |
| 699 | + * spills first when budget - budget/16 would be crossed. |
| 700 | + * |
| 701 | + * The charge is pinned through the test seam (it is the process footprint |
| 702 | + * otherwise): run A ends extraction ONE BYTE under the latch -- the extraction |
| 703 | + * gate never fires, only the projection can spill; run B has the whole budget |
| 704 | + * free -- the projection fits and nothing may spill. Both graphs must be |
| 705 | + * identical, field for field, to each other and to the plain in-memory run. */ |
| 706 | +TEST(parallel_post_extract_projection_spills_before_resolve) { |
| 707 | + if (ensure_parity_setup() != 0) |
| 708 | + FAIL("setup failed"); |
| 709 | + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; |
| 710 | + cbm_file_info_t *files = NULL; |
| 711 | + int file_count = 0; |
| 712 | + ASSERT_EQ(cbm_discover(g_par_tmpdir, &opts, &files, &file_count), 0); |
| 713 | + ASSERT_GT(file_count, 0); |
| 714 | + |
| 715 | + const size_t saved_budget = cbm_mem_budget(); |
| 716 | + const size_t budget = (size_t)1024 * 1024 * 1024; |
| 717 | + const size_t latch = budget - budget / 16; /* spill latches on charged > latch */ |
| 718 | + cbm_mem_set_budget_for_tests(budget); |
| 719 | + g_harness_spill = true; |
| 720 | + |
| 721 | + cbm_mem_set_charged_for_tests(latch); |
| 722 | + int cached_near = -1; |
| 723 | + cbm_gbuf_t *gb_near = |
| 724 | + run_parallel_with_extract_opts_and_mutator("par-test", g_par_tmpdir, files, file_count, 2, |
| 725 | + NULL, count_cached_results, &cached_near, false); |
| 726 | + int64_t parked_near = g_harness_parked; |
| 727 | + |
| 728 | + cbm_mem_set_charged_for_tests((size_t)1); |
| 729 | + int cached_roomy = -1; |
| 730 | + cbm_gbuf_t *roomy = run_parallel_with_extract_opts_and_mutator( |
| 731 | + "par-test", g_par_tmpdir, files, file_count, 2, NULL, count_cached_results, &cached_roomy, |
| 732 | + false); |
| 733 | + int64_t parked_roomy = g_harness_parked; |
| 734 | + |
| 735 | + cbm_mem_set_charged_for_tests(0); |
| 736 | + g_harness_spill = false; |
| 737 | + cbm_mem_set_budget_for_tests(saved_budget); |
| 738 | + cbm_discover_free(files, file_count); |
| 739 | + ASSERT(gb_near != NULL); |
| 740 | + ASSERT(roomy != NULL); |
| 741 | + |
| 742 | + /* A: every result went to disk before registry build. */ |
| 743 | + ASSERT_EQ((int)parked_near, file_count); |
| 744 | + ASSERT_EQ(cached_near, 0); |
| 745 | + /* B: the projection fits, so no store was opened and results stay cached. */ |
| 746 | + ASSERT_EQ((int)parked_roomy, -1); |
| 747 | + ASSERT_GT(cached_roomy, 0); |
| 748 | + |
| 749 | + graph_fp_t fp_near = graph_fingerprint(gb_near); |
| 750 | + graph_fp_t fp_roomy = graph_fingerprint(roomy); |
| 751 | + graph_fp_t fp_mem = graph_fingerprint(g_par_gbuf); |
| 752 | + ASSERT_GT(fp_mem.count, 0); |
| 753 | + ASSERT_EQ(fp_near.count, fp_mem.count); |
| 754 | + ASSERT_EQ(fp_roomy.count, fp_mem.count); |
| 755 | + ASSERT_TRUE(fp_near.sum == fp_mem.sum && fp_near.xr == fp_mem.xr); |
| 756 | + ASSERT_TRUE(fp_roomy.sum == fp_mem.sum && fp_roomy.xr == fp_mem.xr); |
| 757 | + cbm_gbuf_free(gb_near); |
| 758 | + cbm_gbuf_free(roomy); |
| 759 | + PASS(); |
| 760 | +} |
| 761 | + |
619 | 762 | /* ── Empty file list ──────────────────────────────────────────────── */ |
620 | 763 |
|
621 | 764 | TEST(parallel_empty_files) { |
@@ -4472,6 +4615,7 @@ SUITE(parallel) { |
4472 | 4615 | RUN_TEST(parallel_semantic_fixture_expected_counts); |
4473 | 4616 | RUN_TEST(parallel_total_edges); |
4474 | 4617 | RUN_TEST(parallel_spill_mode_builds_the_same_graph); |
| 4618 | + RUN_TEST(parallel_post_extract_projection_spills_before_resolve); |
4475 | 4619 | RUN_TEST(parallel_empty_files); |
4476 | 4620 | RUN_TEST(parallel_args_json_no_overflow); |
4477 | 4621 |
|
|
0 commit comments