Skip to content

Commit fb8d14f

Browse files
committed
doc-links(cs): what the Linux leg showed: path-ordered binding, a test thread's caches, split suites
The first run of the doc-link suites on the Linux arm64 leg (gcc, ASan with LeakSanitizer) found three things macOS does not show: - The node pass bound files in the order the file system listed them, and the scratch table's cost depends on that order (the table is emptied per file and made anew only after a far larger file). doc_mentions_cs_scratch_table_work counted 102,912 steps on Linux against 26,048 on macOS for the same files. Files are now bound in path order: a file's bindings do not depend on the others, so only the cost changes, and it is the same on every system. The test itself had a second dependence: its 20,000-field file exceeded the extraction budget on that leg (5 s, defs=0), so the test ran without it; 4,000 fields were still cut there beside five other suites. It now uses 1,000 fields and 20 small files (3,008 steps against a bound of 4,160; 40,512 with the S13 fix reverted) and requires the large file to reach the index whole. - A test thread that extracted C# on its own small stack ended without freeing its parser and node-kind caches, as worker threads do: 40 KB reported leaked at exit. It frees them now. - The suite ran 1,014 s against the harness's 900 s per-suite wall clock (gcc on aarch64 makes ASan's stack-use-after-return check about 100x slower per call). Its 119 tests now run as doc_mentions (72), doc_mentions_cost (9, the work-counter tests) and doc_mentions_msbuild (38): about 230 s, 210 s and 480 s there. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 346b856 commit fb8d14f

3 files changed

Lines changed: 118 additions & 59 deletions

File tree

‎src/pipeline/doc_links_cs.c‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6373,14 +6373,41 @@ static const char *build_files(cs_index_t *ix, const cbm_doclink_build_in_t *in)
63736373
return bad;
63746374
}
63756375

6376-
/* The graph node of every declaration. false when memory ran out. */
6376+
typedef struct {
6377+
const char *path;
6378+
int idx;
6379+
} cs_file_order_t;
6380+
6381+
static int file_order_cmp(const void *a, const void *b) {
6382+
const cs_file_order_t *x = (const cs_file_order_t *)a;
6383+
const cs_file_order_t *y = (const cs_file_order_t *)b;
6384+
int c = strcmp(x->path ? x->path : "", y->path ? y->path : "");
6385+
return c ? c : (x->idx > y->idx) - (x->idx < y->idx);
6386+
}
6387+
6388+
/* The graph node of every declaration. false when memory ran out. Files are
6389+
* bound in path order: a file's bindings do not depend on the others, but the
6390+
* scratch table's cost does on the order (it is emptied per file), and the
6391+
* order the file system listed the files in differs between systems. */
63776392
static bool build_nodes(cs_index_t *ix, const cbm_gbuf_t *g) {
63786393
cs_node_pass_t np = {.names = cbm_ht_create(CBM_SZ_256), .sized = CBM_SZ_256};
63796394
cbm_arena_init(&np.keys);
6380-
bool ok = np.names != NULL;
6395+
cs_file_order_t *order = (cs_file_order_t *)cbm_alloc(
6396+
CBM_MEM_CLASS_OTHER, (size_t)(ix->nfiles ? ix->nfiles : 1) * sizeof(*order));
6397+
bool ok = np.names != NULL && order != NULL;
6398+
if (!order) {
6399+
ix->oom = true;
6400+
}
6401+
for (int i = 0; ok && i < ix->nfiles; i++) {
6402+
order[i] = (cs_file_order_t){ix->files[i].rel_path, i};
6403+
}
6404+
if (ok && ix->nfiles > 1) {
6405+
qsort(order, (size_t)ix->nfiles, sizeof(*order), file_order_cmp);
6406+
}
63816407
for (int i = 0; ok && i < ix->nfiles; i++) {
6382-
ok = bind_nodes(ix, &ix->files[i], g, &np);
6408+
ok = bind_nodes(ix, &ix->files[order[i].idx], g, &np);
63836409
}
6410+
cbm_free(CBM_MEM_CLASS_OTHER, order);
63846411
cbm_ht_free(np.names);
63856412
cbm_arena_destroy(&np.keys);
63866413
cbm_free(CBM_MEM_CLASS_OTHER, np.last);

‎tests/test_doc_mentions.c‎

Lines changed: 84 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "test_doc_mentions_helpers.h"
1414

1515
#include "cbm.h"
16+
#include "helpers.h" /* cbm_kind_in_set_free_cache */
1617
#include "doclink.h"
1718
#include "lang_specs.h"
1819
#include "foundation/compat_thread.h"
@@ -5980,10 +5981,14 @@ typedef struct {
59805981
CBMFileResult *result;
59815982
} dm_extract_job_t;
59825983

5983-
/* cbm_thread_create body: extract job->src as C#. */
5984+
/* cbm_thread_create body: extract job->src as C#, then free this thread's
5985+
* parser and caches as a worker thread does when it ends (LeakSanitizer
5986+
* reports them otherwise). */
59845987
static void *dm_extract_thread(void *arg) {
59855988
dm_extract_job_t *job = (dm_extract_job_t *)arg;
59865989
job->result = dm_extract(job->src, CBM_LANG_CSHARP, job->rel_path);
5990+
cbm_destroy_thread_parser();
5991+
cbm_kind_in_set_free_cache();
59875992
return NULL;
59885993
}
59895994

@@ -9892,11 +9897,18 @@ TEST(doc_mentions_msbuild_malformed_props_opens) {
98929897
PASS();
98939898
}
98949899

9895-
enum { DM_BIG_MEMBERS = 20000, DM_SMALL_FILES = 200 };
9900+
/* Sized so that the large file parses far within the extraction budget on
9901+
* the slowest sanitizer leg, under load too (20,000 fields were cut there,
9902+
* and 4,000 beside five other suites, leaving the test without its large
9903+
* file). Every small file costs at least the scratch table's minimum size:
9904+
* these 20 and the large file come to 3,008 steps, within the bound, while
9905+
* the regression, DM_SMALL_FILES x 2 x DM_BIG_MEMBERS, is ten times it. */
9906+
enum { DM_BIG_MEMBERS = 1000, DM_SMALL_FILES = 20 };
98969907

98979908
/* What the resolver looks at to build its index over one file of
98989909
* DM_BIG_MEMBERS fields followed (in path order) by DM_SMALL_FILES files of
9899-
* one type and one field each. 0 when the repository cannot be indexed. */
9910+
* one type and one field each. 0 when the repository cannot be indexed or the
9911+
* large file did not reach the index whole. */
99009912
static uint64_t dm_scratch_work(void) {
99019913
char tmp[256];
99029914
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_scratch_XXXXXX");
@@ -9930,16 +9942,23 @@ static uint64_t dm_scratch_work(void) {
99309942
cbm_doclink_cs_test_work_reset();
99319943
uint64_t work = dm_index(tmp, db, NULL) == 0 ? cbm_doclink_cs_test_scratch_work() : 0;
99329944
int rows = dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved");
9945+
int fields = dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'Field' AND "
9946+
"file_path = 'a/Big.cs'");
9947+
if (fields != DM_BIG_MEMBERS) {
9948+
printf(" scratch table: a/Big.cs reached the index with %d of %d fields\n", fields,
9949+
(int)DM_BIG_MEMBERS);
9950+
}
99339951
dm_unlink_db(db);
99349952
th_rmtree(tmp);
9935-
return rows == 1 ? work : 0;
9953+
return rows == 1 && fields == DM_BIG_MEMBERS ? work : 0;
99369954
}
99379955

99389956
/* The per-file scratch table of the index build is not emptied at the size
99399957
* one large file grew it to: after a file of DM_BIG_MEMBERS names, every one
9940-
* of DM_SMALL_FILES small files costs about its own size. The work stays
9941-
* within a small multiple of all the names the files declare (emptying the
9942-
* large table for every small file cost DM_SMALL_FILES times its size). */
9958+
* of DM_SMALL_FILES small files costs about the table's minimum size. The
9959+
* work stays within a small multiple of all the names the files declare
9960+
* (emptying the large table for every small file cost DM_SMALL_FILES times
9961+
* its size). */
99439962
TEST(doc_mentions_cs_scratch_table_work) {
99449963
uint64_t names = (uint64_t)DM_BIG_MEMBERS + (uint64_t)DM_SMALL_FILES * 2;
99459964
uint64_t work = dm_scratch_work();
@@ -10473,27 +10492,6 @@ SUITE(doc_mentions) {
1047310492
RUN_TEST(doc_mentions_cs_namespace_publication);
1047410493
RUN_TEST(doc_mentions_cs_scope_parse_errors);
1047510494
RUN_TEST(doc_mentions_cs_norm_type);
10476-
RUN_TEST(doc_mentions_msbuild_usings);
10477-
RUN_TEST(doc_mentions_msbuild_targets_isolation);
10478-
RUN_TEST(doc_mentions_msbuild_targets_failure);
10479-
RUN_TEST(doc_mentions_msbuild_items_isolation);
10480-
RUN_TEST(doc_mentions_msbuild_items_failure);
10481-
RUN_TEST(doc_mentions_msbuild_items_lifetime);
10482-
RUN_TEST(doc_mentions_msbuild_items_work);
10483-
RUN_TEST(doc_mentions_msbuild_targets_work);
10484-
RUN_TEST(doc_mentions_msbuild_nearest_work);
10485-
RUN_TEST(doc_mentions_msbuild_shared_closure_work);
10486-
RUN_TEST(doc_mentions_msbuild_shared_file_work);
10487-
RUN_TEST(doc_mentions_msbuild_shared_closure_isolation);
10488-
RUN_TEST(doc_mentions_msbuild_components_failure);
10489-
RUN_TEST(doc_mentions_msbuild_components_revision);
10490-
RUN_TEST(doc_mentions_msbuild_components_mixed_absence);
10491-
RUN_TEST(doc_mentions_msbuild_prefix_work);
10492-
RUN_TEST(doc_mentions_msbuild_prefix_isolation);
10493-
RUN_TEST(doc_mentions_msbuild_prefix_failure);
10494-
RUN_TEST(doc_mentions_msbuild_eval_storage);
10495-
RUN_TEST(doc_mentions_msbuild_value_lifetimes);
10496-
RUN_TEST(doc_mentions_msbuild_value_allocation);
1049710495
RUN_TEST(doc_mentions_resolver_rules);
1049810496
RUN_TEST(doc_mentions_resolver_arity_and_members);
1049910497
RUN_TEST(doc_mentions_resolver_parse_errors);
@@ -10508,34 +10506,14 @@ SUITE(doc_mentions) {
1050810506
RUN_TEST(doc_mentions_incremental_equals_full);
1050910507
RUN_TEST(doc_mentions_parallel_equals_sequential);
1051010508
RUN_TEST(doc_mentions_cs_scan_dollar_run);
10511-
RUN_TEST(doc_mentions_cs_scan_branch_memory);
10512-
RUN_TEST(doc_mentions_cs_scan_branch_merge_work);
10513-
RUN_TEST(doc_mentions_cs_scan_declarator_modifier_work);
1051410509
RUN_TEST(doc_mentions_cs_shared_doc_sources);
1051510510
RUN_TEST(doc_mentions_cs_shared_doc_once);
10516-
RUN_TEST(doc_mentions_cs_shared_doc_work);
1051710511
RUN_TEST(doc_mentions_cs_shared_doc_replay);
1051810512
RUN_TEST(doc_mentions_cs_shared_doc_allocation_failure);
1051910513
RUN_TEST(doc_mentions_cs_scan_header_reads);
1052010514
RUN_TEST(doc_mentions_cs_scan_nested_holes);
1052110515
RUN_TEST(doc_mentions_cs_scan_holes_stack);
10522-
RUN_TEST(doc_mentions_msbuild_blob);
10523-
RUN_TEST(doc_mentions_msbuild_import_group_blob_growth);
10524-
RUN_TEST(doc_mentions_msbuild_import_group_roundtrip);
10525-
RUN_TEST(doc_mentions_msbuild_import_group_condition_work);
10526-
RUN_TEST(doc_mentions_msbuild_item_group_condition_work);
10527-
RUN_TEST(doc_mentions_msbuild_import_group_condition_once);
10528-
RUN_TEST(doc_mentions_msbuild_legacy_import_blob);
10529-
RUN_TEST(doc_mentions_msbuild_bad_import_group_blob);
10530-
RUN_TEST(doc_mentions_msbuild_imports);
10531-
RUN_TEST(doc_mentions_msbuild_conditions);
10532-
RUN_TEST(doc_mentions_msbuild_unknown_spreads);
10533-
RUN_TEST(doc_mentions_msbuild_gate);
1053410516
RUN_TEST(doc_mentions_incremental_project_files);
10535-
#ifndef _WIN32
10536-
RUN_TEST(doc_mentions_msbuild_linked_project_file);
10537-
RUN_TEST(doc_mentions_msbuild_linked_import_directory);
10538-
#endif
1053910517
RUN_TEST(doc_mentions_cs_lookup_order);
1054010518
RUN_TEST(doc_mentions_cs_type_parameters);
1054110519
RUN_TEST(doc_mentions_cs_usings);
@@ -10555,10 +10533,6 @@ SUITE(doc_mentions) {
1055510533
RUN_TEST(doc_mentions_cs_doc_ids);
1055610534
RUN_TEST(doc_mentions_cs_reasons);
1055710535
RUN_TEST(doc_mentions_cs_text_forms);
10558-
RUN_TEST(doc_mentions_cs_lookup_cost);
10559-
RUN_TEST(doc_mentions_cs_import_lookup_cost);
10560-
RUN_TEST(doc_mentions_cs_lookup_repeated_name_work);
10561-
RUN_TEST(doc_mentions_cs_unit_usings_across_files_work);
1056210536
RUN_TEST(doc_mentions_cs_unit_usings_unchanged);
1056310537
RUN_TEST(doc_mentions_cs_lookup_ambiguous_stops);
1056410538
RUN_TEST(doc_mentions_cs_import_stage_aliases);
@@ -10567,20 +10541,74 @@ SUITE(doc_mentions) {
1056710541
RUN_TEST(doc_mentions_cs_no_silent_limits);
1056810542
RUN_TEST(doc_mentions_cs_many_assemblies);
1056910543
RUN_TEST(doc_mentions_incremental_using_of_based_type);
10570-
RUN_TEST(doc_mentions_msbuild_many_project_files);
1057110544
RUN_TEST(doc_mentions_cs_scan_nesting_limits);
1057210545
RUN_TEST(doc_mentions_cs_damaged_stored_scope);
1057310546
RUN_TEST(doc_mentions_cs_portable_scope_bound);
1057410547
RUN_TEST(doc_mentions_cs_unbalanced_brackets);
1057510548
RUN_TEST(doc_mentions_cs_stored_scope_nesting);
1057610549
RUN_TEST(doc_mentions_index_status_unknown_reason);
10577-
RUN_TEST(doc_mentions_msbuild_unread_project_opens);
10578-
RUN_TEST(doc_mentions_msbuild_malformed_props_opens);
10579-
RUN_TEST(doc_mentions_cs_scratch_table_work);
1058010550
RUN_TEST(doc_mentions_incremental_non_ascii_name);
1058110551
RUN_TEST(doc_mentions_alloc_failure_status);
1058210552
RUN_TEST(doc_mentions_directives_only_failure);
1058310553
RUN_TEST(doc_mentions_cs_scanner_output_reads_back);
1058410554
RUN_TEST(doc_mentions_cs_rejected_scope_contained);
1058510555
RUN_TEST(doc_mentions_cs_rejected_scope_across_runs);
1058610556
}
10557+
10558+
/* The cost tests (work counters held to a multiple of the input) and the MSBuild
10559+
* evaluator run as suites of their own, so that each stays within the per-suite
10560+
* wall clock of the harness on the slowest sanitizer legs. */
10561+
SUITE(doc_mentions_cost) {
10562+
RUN_TEST(doc_mentions_cs_scan_branch_memory);
10563+
RUN_TEST(doc_mentions_cs_scan_branch_merge_work);
10564+
RUN_TEST(doc_mentions_cs_scan_declarator_modifier_work);
10565+
RUN_TEST(doc_mentions_cs_shared_doc_work);
10566+
RUN_TEST(doc_mentions_cs_lookup_cost);
10567+
RUN_TEST(doc_mentions_cs_import_lookup_cost);
10568+
RUN_TEST(doc_mentions_cs_lookup_repeated_name_work);
10569+
RUN_TEST(doc_mentions_cs_unit_usings_across_files_work);
10570+
RUN_TEST(doc_mentions_cs_scratch_table_work);
10571+
}
10572+
10573+
SUITE(doc_mentions_msbuild) {
10574+
RUN_TEST(doc_mentions_msbuild_usings);
10575+
RUN_TEST(doc_mentions_msbuild_targets_isolation);
10576+
RUN_TEST(doc_mentions_msbuild_targets_failure);
10577+
RUN_TEST(doc_mentions_msbuild_items_isolation);
10578+
RUN_TEST(doc_mentions_msbuild_items_failure);
10579+
RUN_TEST(doc_mentions_msbuild_items_lifetime);
10580+
RUN_TEST(doc_mentions_msbuild_items_work);
10581+
RUN_TEST(doc_mentions_msbuild_targets_work);
10582+
RUN_TEST(doc_mentions_msbuild_nearest_work);
10583+
RUN_TEST(doc_mentions_msbuild_shared_closure_work);
10584+
RUN_TEST(doc_mentions_msbuild_shared_file_work);
10585+
RUN_TEST(doc_mentions_msbuild_shared_closure_isolation);
10586+
RUN_TEST(doc_mentions_msbuild_components_failure);
10587+
RUN_TEST(doc_mentions_msbuild_components_revision);
10588+
RUN_TEST(doc_mentions_msbuild_components_mixed_absence);
10589+
RUN_TEST(doc_mentions_msbuild_prefix_work);
10590+
RUN_TEST(doc_mentions_msbuild_prefix_isolation);
10591+
RUN_TEST(doc_mentions_msbuild_prefix_failure);
10592+
RUN_TEST(doc_mentions_msbuild_eval_storage);
10593+
RUN_TEST(doc_mentions_msbuild_value_lifetimes);
10594+
RUN_TEST(doc_mentions_msbuild_value_allocation);
10595+
RUN_TEST(doc_mentions_msbuild_blob);
10596+
RUN_TEST(doc_mentions_msbuild_import_group_blob_growth);
10597+
RUN_TEST(doc_mentions_msbuild_import_group_roundtrip);
10598+
RUN_TEST(doc_mentions_msbuild_import_group_condition_work);
10599+
RUN_TEST(doc_mentions_msbuild_item_group_condition_work);
10600+
RUN_TEST(doc_mentions_msbuild_import_group_condition_once);
10601+
RUN_TEST(doc_mentions_msbuild_legacy_import_blob);
10602+
RUN_TEST(doc_mentions_msbuild_bad_import_group_blob);
10603+
RUN_TEST(doc_mentions_msbuild_imports);
10604+
RUN_TEST(doc_mentions_msbuild_conditions);
10605+
RUN_TEST(doc_mentions_msbuild_unknown_spreads);
10606+
RUN_TEST(doc_mentions_msbuild_gate);
10607+
#ifndef _WIN32
10608+
RUN_TEST(doc_mentions_msbuild_linked_project_file);
10609+
RUN_TEST(doc_mentions_msbuild_linked_import_directory);
10610+
#endif
10611+
RUN_TEST(doc_mentions_msbuild_many_project_files);
10612+
RUN_TEST(doc_mentions_msbuild_unread_project_opens);
10613+
RUN_TEST(doc_mentions_msbuild_malformed_props_opens);
10614+
}

‎tests/test_main.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1819,6 +1819,8 @@ extern void suite_traces(void);
18191819
extern void suite_configlink(void);
18201820
extern void suite_doclinks(void);
18211821
extern void suite_doc_mentions(void);
1822+
extern void suite_doc_mentions_cost(void);
1823+
extern void suite_doc_mentions_msbuild(void);
18221824
extern void suite_infrascan(void);
18231825
extern void suite_cli(void);
18241826
extern void suite_agent_clients(void);
@@ -2284,6 +2286,8 @@ int main(int argc, char **argv) {
22842286

22852287
/* Doc-comment references -> MENTIONS */
22862288
RUN_SELECTED_SUITE(doc_mentions);
2289+
RUN_SELECTED_SUITE(doc_mentions_cost);
2290+
RUN_SELECTED_SUITE(doc_mentions_msbuild);
22872291

22882292
/* Infrastructure scanning */
22892293
RUN_SELECTED_SUITE(infrascan);

0 commit comments

Comments
 (0)