Skip to content

Commit 19783ed

Browse files
committed
doc-links(cs): a unit's using directives are asked once per query, not once per file (R1)
The using step of a lookup is remembered per file (S5), so the unit's directives -- its global usings and <Using> items, asked at the global namespace -- were walked again by every file that asks a name: files x min(directives, scopes declaring the name), a hang in resolution for a project with many global usings and a common name. What a unit directive gives a query reads only the context's unit, assembly and product flag (level_using and what it calls), never the file. The run now keeps, per (unit, assembly, product/test, query), the unit directives that give the query anything at all, in order, up to where they alone decide it (cs_unit_memo). Each file asks those after its own directives and stops where the walk of all of them stops. The edges and rows are unchanged: a directive that leaves an empty step empty makes no change in any step, and a step that starts with the file's candidates is decided no later than the unit's directives alone are (a candidate counts unless it is the step's first). The memo is made once the index is complete (the build's own lookups see directives that are not all resolved yet) and is shared by the resolve workers: each list is found exactly once, by the first worker that asks; the others that ask for that key take its lock until the list is there, so the work of a run does not depend on the scheduling. Out of memory, it stops remembering and the walk is the full one. A test seam builds the index without the memo. Tests: cs_unit_usings_across_files_work (k global usings, k scopes declaring the name, f files naming it once: k 150 -> 300 and f 100 -> 200 give 768 -> 1520 steps, 1.98 x; without the memo 17003 -> 64403, 3.79 x); cs_unit_usings_unchanged (the same edges and rows with and without the memo, over the cases of file and unit candidates: none, other, same, same then other, test-only, another unit). Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 5fd1a07 commit 19783ed

3 files changed

Lines changed: 365 additions & 6 deletions

File tree

‎src/pipeline/doc_links.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,10 @@ void cbm_doclinks_test_fail_edge_insert_after(int nth);
242242
/* Test seam (doc_links_cs.c): ask the reader whether it takes a scope blob.
243243
* (The scope of one file is spoiled where it is written: doclink.h.) */
244244
bool cbm_doclink_cs_test_scope_parses(const char *scope);
245+
/* Test seam (doc_links_cs.c): false builds the C# indexes from now on without
246+
* the run's memo of the units' using steps (every unit step walks its
247+
* directives), so a run can be held against one with it; true restores it. */
248+
void cbm_doclink_cs_test_unit_memo(bool on);
245249
#endif
246250

247251
/* True when `rel_path` is a scope input of some language (scope_input). */

‎src/pipeline/doc_links_cs.c‎

Lines changed: 204 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@
155155
#include "helpers.h" /* cbm_fqn_module_source_lang */
156156
#include "pipeline/doc_links_msbuild.h"
157157
#include "foundation/arena.h"
158+
#include "foundation/compat_thread.h"
158159
#include "foundation/constants.h"
159160
#include "foundation/hash_table.h"
160161
#include "foundation/log.h"
@@ -547,6 +548,9 @@ typedef struct {
547548
bool bind_inherited; /* CS_BIND_INHERITED */
548549
cs_undo_t undo; /* of the file being parsed */
549550
int rejected; /* files whose scope, written in this run, was refused */
551+
/* the unit part of the using steps, shared by the resolve workers; made
552+
* once the index is complete (NULL before, and when memory ran out) */
553+
struct cs_unit_memo *unit_memo;
550554
} cs_index_t;
551555

552556
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
@@ -4597,8 +4601,53 @@ static int candidate_id_cmp(const void *a, const void *b) {
45974601
return int_cmp(a, b);
45984602
}
45994603

4604+
/* The directives of a using step that give a query anything, in order (R1):
4605+
* noted while the step is walked, for the unit memo. */
4606+
typedef struct {
4607+
int *ids;
4608+
int n;
4609+
int cap;
4610+
bool failed; /* memory ran out: the list is not whole */
4611+
} cs_active_rec_t;
4612+
4613+
static void active_add(cs_active_rec_t *rec, int id) {
4614+
if (rec->n == rec->cap) {
4615+
int cap = rec->cap ? rec->cap * PAIR_LEN : CBM_SZ_16;
4616+
int *grown = (int *)cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)cap * sizeof(int));
4617+
if (!grown) {
4618+
rec->failed = true;
4619+
return;
4620+
}
4621+
if (rec->n > 0) {
4622+
memcpy(grown, rec->ids, (size_t)rec->n * sizeof(int));
4623+
}
4624+
cbm_free(CBM_MEM_CLASS_OTHER, rec->ids);
4625+
rec->ids = grown;
4626+
rec->cap = cap;
4627+
}
4628+
rec->ids[rec->n++] = id;
4629+
}
4630+
4631+
/* Directive `id` of a step, for the query. With `rec`, it is noted when it
4632+
* gives the query anything at all: one that leaves an empty step empty adds
4633+
* no candidate and sets no flag whatever the step holds, so leaving it out
4634+
* changes no lookup. */
4635+
static void using_visit(const cs_ctx_t *c, const cs_using_t *us, int id, const cs_query_t *q,
4636+
bool a_type_name, cs_found_t *fd, cs_active_rec_t *rec) {
4637+
if (rec) {
4638+
cs_found_t probe = {0};
4639+
level_using(c, &us[id], q, a_type_name, &probe);
4640+
if (probe.n == 0 && !probe.invisible && !probe.joined && probe.why == CS_WHY_SCOPE) {
4641+
return;
4642+
}
4643+
active_add(rec, id);
4644+
}
4645+
level_using(c, &us[id], q, a_type_name, fd);
4646+
}
4647+
46004648
static void level_usings(const cs_ctx_t *c, const cs_using_t *us, int n,
4601-
const cs_using_index_t *index, const cs_query_t *q, cs_found_t *fd) {
4649+
const cs_using_index_t *index, const cs_query_t *q, cs_found_t *fd,
4650+
cs_active_rec_t *rec) {
46024651
if (!n) {
46034652
return;
46044653
}
@@ -4613,7 +4662,7 @@ static void level_usings(const cs_ctx_t *c, const cs_using_t *us, int n,
46134662
* more postings than it has directives. The baseline scan stays cheap. */
46144663
if (!indexed || hi - lo >= (size_t)n) {
46154664
for (int i = 0; i < n && !found_decided(fd); i++) {
4616-
level_using(c, &us[i], q, a_type_name, fd);
4665+
using_visit(c, us, i, q, a_type_name, fd, rec);
46174666
}
46184667
return;
46194668
}
@@ -4636,7 +4685,7 @@ static void level_usings(const cs_ctx_t *c, const cs_using_t *us, int n,
46364685
/* Own and twin postings may select the SAME directive twice.
46374686
* Distinct directive IDs must still be resolved separately. */
46384687
if (!i || ids.ids[i] != ids.ids[i - SKIP_ONE]) {
4639-
level_using(c, &us[ids.ids[i]], q, a_type_name, fd);
4688+
using_visit(c, us, ids.ids[i], q, a_type_name, fd, rec);
46404689
}
46414690
}
46424691
}
@@ -4645,7 +4694,7 @@ static void level_usings(const cs_ctx_t *c, const cs_using_t *us, int n,
46454694
}
46464695
if (!complete) {
46474696
for (int i = 0; i < n && !found_decided(fd); i++) {
4648-
level_using(c, &us[i], q, a_type_name, fd);
4697+
using_visit(c, us, i, q, a_type_name, fd, rec);
46494698
}
46504699
}
46514700
}
@@ -4715,6 +4764,139 @@ static void memo_put(cs_memo_t *m, const char *key, const cs_found_t *step) {
47154764
m->off = cbm_ht_get(m->steps, k) != v; /* an insert that did not take */
47164765
}
47174766

4767+
/* The unit part of a using step (R1): which of a unit's directives give a
4768+
* query anything, in order, up to where they alone decide it. What a
4769+
* directive gives reads only the context's unit, assembly and product flag
4770+
* (level_using), so the list is the same for every file of the unit that
4771+
* asks the query: it is found once per run, shared by the resolve workers
4772+
* (guarded), and each file then asks only those directives after its own --
4773+
* stopping where the walk of all of them would, which is never later than
4774+
* where they alone are decided. Each list is found once: by the first
4775+
* worker that asks for it, while the others that ask for the same key hold
4776+
* on until it is there (a step's cost never depends on the scheduling). A
4777+
* memo that ran out of memory stops remembering; the lookups stay exact. */
4778+
typedef struct cs_unit_step {
4779+
cbm_mutex_t mu; /* held by the worker that finds the list, until it is there */
4780+
int *ids; /* the directives, in order (memory-core block, or NULL) */
4781+
int n;
4782+
bool failed; /* memory ran out finding it: askers walk all directives */
4783+
struct cs_unit_step *next; /* every step, for the memo's release */
4784+
} cs_unit_step_t;
4785+
4786+
typedef struct cs_unit_memo {
4787+
cbm_mutex_t mu; /* guards steps, arena, all and off */
4788+
CBMHashTable *steps; /* key -> cs_unit_step_t in `arena` */
4789+
CBMArena arena;
4790+
cs_unit_step_t *all;
4791+
bool off;
4792+
} cs_unit_memo_t;
4793+
4794+
static cs_unit_memo_t *unit_memo_new(void) {
4795+
cs_unit_memo_t *m = (cs_unit_memo_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*m));
4796+
if (!m) {
4797+
return NULL;
4798+
}
4799+
cbm_mutex_init(&m->mu);
4800+
cbm_arena_init_lazy(&m->arena, CBM_ARENA_APPEND_BLOCK);
4801+
return m;
4802+
}
4803+
4804+
static void unit_memo_free(cs_unit_memo_t *m) {
4805+
if (!m) {
4806+
return;
4807+
}
4808+
for (cs_unit_step_t *e = m->all; e; e = e->next) {
4809+
cbm_mutex_destroy(&e->mu);
4810+
cbm_free(CBM_MEM_CLASS_OTHER, e->ids);
4811+
}
4812+
cbm_ht_free(m->steps);
4813+
cbm_arena_destroy(&m->arena);
4814+
cbm_mutex_destroy(&m->mu);
4815+
cbm_free(CBM_MEM_CLASS_OTHER, m);
4816+
}
4817+
4818+
/* The key of one unit step: everything it reads. false when it does not fit. */
4819+
static bool unit_key(char *buf, size_t cap, const cs_ctx_t *c, const cs_query_t *q) {
4820+
int n = snprintf(buf, cap, "%d|%d|%d|%d|%d%d%d%c|%s", c->unit, c->group, (int)c->prod, q->arity,
4821+
(int)q->types_only, (int)q->statics, (int)q->ctors, q->kind ? q->kind : '-',
4822+
q->name);
4823+
return n > 0 && (size_t)n < cap;
4824+
}
4825+
4826+
/* The step under `key` (the caller holds the memo's lock); a new one is
4827+
* added with its lock held, and *mine set: the caller finds its list. NULL
4828+
* when memory ran out. */
4829+
static cs_unit_step_t *unit_step_at(cs_unit_memo_t *m, const char *key, bool *mine) {
4830+
*mine = false;
4831+
if (!m->steps) {
4832+
m->steps = cbm_ht_create(CBM_SZ_64);
4833+
if (!m->steps) {
4834+
return NULL;
4835+
}
4836+
}
4837+
cs_unit_step_t *had = (cs_unit_step_t *)cbm_ht_get(m->steps, key);
4838+
if (had) {
4839+
return had;
4840+
}
4841+
char *k = cbm_arena_strdup(&m->arena, key);
4842+
cs_unit_step_t *e = (cs_unit_step_t *)cbm_arena_alloc(&m->arena, sizeof(*e));
4843+
if (!k || !e) {
4844+
return NULL;
4845+
}
4846+
memset(e, 0, sizeof(*e));
4847+
cbm_mutex_init(&e->mu);
4848+
cbm_mutex_lock(&e->mu);
4849+
e->next = m->all;
4850+
m->all = e;
4851+
cbm_ht_set(m->steps, k, e);
4852+
if (cbm_ht_get(m->steps, k) != e) {
4853+
e->failed = true; /* an insert that did not take: kept for release only */
4854+
cbm_mutex_unlock(&e->mu);
4855+
return NULL;
4856+
}
4857+
*mine = true;
4858+
return e;
4859+
}
4860+
4861+
/* What the unit's directives give the query after the region's (`fd`). */
4862+
static void unit_usings(const cs_ctx_t *c, const cs_unit_t *unit, const cs_query_t *q,
4863+
cs_found_t *fd) {
4864+
cs_unit_memo_t *m = c->ix->unit_memo;
4865+
char key[CS_MEMO_KEY];
4866+
cs_unit_step_t *e = NULL;
4867+
bool mine = false;
4868+
if (m && unit_key(key, sizeof(key), c, q)) {
4869+
cs_work(SKIP_ONE);
4870+
cbm_mutex_lock(&m->mu);
4871+
e = m->off ? NULL : unit_step_at(m, key, &mine);
4872+
m->off = m->off || !e;
4873+
cbm_mutex_unlock(&m->mu);
4874+
}
4875+
if (e && mine) {
4876+
cs_active_rec_t rec = {0};
4877+
cs_found_t alone = {0};
4878+
level_usings(c, unit->usings, unit->nusings, &unit->using_index, q, &alone, &rec);
4879+
e->failed = rec.failed;
4880+
e->ids = rec.failed ? NULL : rec.ids;
4881+
e->n = rec.failed ? 0 : rec.n;
4882+
if (rec.failed) {
4883+
cbm_free(CBM_MEM_CLASS_OTHER, rec.ids);
4884+
}
4885+
cbm_mutex_unlock(&e->mu);
4886+
} else if (e) {
4887+
cbm_mutex_lock(&e->mu); /* the list is there once its finder lets go */
4888+
cbm_mutex_unlock(&e->mu);
4889+
}
4890+
if (!e || e->failed) {
4891+
level_usings(c, unit->usings, unit->nusings, &unit->using_index, q, fd, NULL);
4892+
return;
4893+
}
4894+
bool a_type_name = cbm_ht_get(c->ix->type_names, q->name) != NULL;
4895+
for (int i = 0; i < e->n && !found_decided(fd); i++) {
4896+
level_using(c, &unit->usings[e->ids[i]], q, a_type_name, fd);
4897+
}
4898+
}
4899+
47184900
/* What the directives of this scope level give the query: the region's own
47194901
* (`us`, when asked) and, at the global namespace, the unit's. Asked once per
47204902
* key when the context has a memo. */
@@ -4727,9 +4909,9 @@ static void using_step(const cs_ctx_t *c, int region, const cs_using_t *us, int
47274909
if (keyed && memo_get(c->memo, key, step)) {
47284910
return;
47294911
}
4730-
level_usings(c, us, nus, usi, q, step);
4912+
level_usings(c, us, nus, usi, q, step, NULL);
47314913
if (unit && !found_decided(step)) {
4732-
level_usings(c, unit->usings, unit->nusings, &unit->using_index, q, step);
4914+
unit_usings(c, unit, q, step);
47334915
}
47344916
if (keyed) {
47354917
memo_put(c->memo, key, step);
@@ -5826,6 +6008,7 @@ static void cs_destroy(void *index) {
58266008
(unsigned long long)(ix->stats ? atomic_load(&ix->stats->rejected) : 0));
58276009
cbm_log_warn("doc_links.cs.rejected_scopes", "files", files, "references", refs);
58286010
}
6011+
unit_memo_free(ix->unit_memo);
58296012
cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.flags);
58306013
cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.marks);
58316014
cbm_free(CBM_MEM_CLASS_OTHER, ix->stats);
@@ -6096,6 +6279,13 @@ static int cs_scope_accepted(const char *scope) {
60966279
bool cbm_doclink_cs_test_scope_parses(const char *scope) {
60976280
return cs_scope_accepted(scope) == SKIP_ONE;
60986281
}
6282+
6283+
/* Test seam: set, the indexes are built without the unit memo. */
6284+
static atomic_bool cs_test_no_unit_memo;
6285+
6286+
void cbm_doclink_cs_test_unit_memo(bool on) {
6287+
atomic_store(&cs_test_no_unit_memo, !on);
6288+
}
60996289
#endif
61006290

61016291
/* Every file of the index with what its scope declares. A scope read back
@@ -6244,6 +6434,14 @@ static void *cs_build(const cbm_doclink_build_in_t *in) {
62446434
if (!resolve_bases(ix) || !spread_open(ix) || ix->oom) {
62456435
return build_failed(ix, "bases", "alloc", NULL);
62466436
}
6437+
/* only now: what the build's own lookups saw of the units' directives
6438+
* was not yet the whole index (NULL when memory ran out: no memo) */
6439+
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
6440+
if (!atomic_load(&cs_test_no_unit_memo))
6441+
#endif
6442+
{
6443+
ix->unit_memo = unit_memo_new();
6444+
}
62476445
log_index(ix, &msb);
62486446
return ix;
62496447
}

0 commit comments

Comments
 (0)