Skip to content

Commit a412642

Browse files
committed
perf(store,ts): dedup coverage shadow-graph dirs; TS export re-export boundary
Two measured changes on the TypeScript corpus (81,397 files; v0.9.0 17.9 s, before 44.1 s, after 37.5 s): - cov_rebuild_shadow_graph upserted every directory segment for every failure row — 13,243 parse-partial baseline files under one tests/ subtree meant ~80k redundant node/edge round-trips, 9.1 s of a 9.2 s coverage_replace. An in-rebuild path->id map creates each directory once; identical graph (edges deduped by unique key before, absent now). Coverage block 9,162 -> 2,920 ms. Sub-block timings (publish.timing.coverage: del/rows/prune/meta/commit + row_count + detail_bytes) are kept — the caller-level number could not name the culprit. - JS/TS export_statement is an import CONTEXT only in its re-export forms (source field, or a bare specifier list without a declaration). The old is_export_of_declaration blacklist missed TS-only forms (ambient_declaration, function_signature, module_declaration), running declare-heavy subtrees (.d.ts, export namespace) behind inside_import: suppressed usages + per-identifier ancestor walks. Positive detection replaces the blacklist; +3,954 restored usage edges on the corpus, nodes identical. (Measured perf-neutral here — kept for correctness.) Suites green incl. store_nodes/edges/search, mcp, extraction, ts_lsp, complexity, and the 53-language calls-breadth contract. Refs DeusData#1669. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent a4c0ffb commit a412642

2 files changed

Lines changed: 116 additions & 2 deletions

File tree

‎internal/cbm/extract_unified.c‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1646,6 +1646,27 @@ static bool is_actual_import_boundary(CBMExtractCtx *ctx, TSNode node, const CBM
16461646
char *name = ts_node_is_null(head) ? NULL : cbm_node_text(ctx->arena, head, ctx->source);
16471647

16481648
switch (ctx->language) {
1649+
case CBM_LANG_JAVASCRIPT:
1650+
case CBM_LANG_TYPESCRIPT:
1651+
case CBM_LANG_TSX:
1652+
if (strcmp(kind, "export_statement") == 0) {
1653+
/* An export is an import CONTEXT only in its re-export forms:
1654+
* `export ... from 'mod'` (source field) or a bare specifier list
1655+
* `export { a, b }` with no declaration. An export OF a declaration
1656+
* must not put the declaration's body behind inside_import — the
1657+
* old kind-blacklist (is_export_of_declaration) missed the TS-only
1658+
* forms (ambient_declaration, function_signature,
1659+
* module_declaration), so declare-heavy code (.d.ts, baselines,
1660+
* `export namespace`) ran whole subtrees as import context:
1661+
* suppressed usages plus per-identifier ancestor walks. Positive
1662+
* detection replaces the blacklist. */
1663+
if (!ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("source")))) {
1664+
return true;
1665+
}
1666+
return !ts_node_is_null(cbm_find_child_by_kind(node, "export_clause")) &&
1667+
ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("declaration")));
1668+
}
1669+
return true; /* import_statement / import / require / extends: unchanged */
16491670
case CBM_LANG_CSHARP:
16501671
/* cs_import_types also lists namespace_declaration (so the import pass
16511672
* can map namespace names) and using_statement (C#'s RAII block, a

‎src/store/store.c‎

Lines changed: 95 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2668,6 +2668,20 @@ static void cov_failure_fingerprint(const cbm_coverage_row_t *rows, int count,
26682668
out[CBM_SHA256_HEX_LEN] = 0;
26692669
}
26702670

2671+
static void cov_dir_ids_free_entry(const char *key, void *val, void *ud) {
2672+
(void)ud;
2673+
free((char *)key);
2674+
free(val);
2675+
}
2676+
2677+
static void cov_dir_ids_free(CBMHashTable *ht) {
2678+
if (!ht) {
2679+
return;
2680+
}
2681+
cbm_ht_foreach(ht, cov_dir_ids_free_entry, NULL);
2682+
cbm_ht_free(ht);
2683+
}
2684+
26712685
static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
26722686
char covproj[CBM_SZ_512];
26732687
cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project);
@@ -2765,6 +2779,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
27652779
return CBM_STORE_ERR;
27662780
}
27672781

2782+
/* Directory nodes repeat massively across failure rows (13k baseline files
2783+
* under one tests/ subtree meant ~80k redundant upsert/edge round-trips —
2784+
* 9.1 s of a 9.2 s coverage_replace on the TypeScript corpus). Dedup them
2785+
* in-memory for the rebuild: first sight of a directory creates its node
2786+
* and containment edge; every later file under it is a hash hit. */
2787+
CBMHashTable *dir_ids = cbm_ht_create(CBM_SZ_256);
27682788
for (int i = 0; i < count; i++) {
27692789
const char *rel = rows[i].rel_path;
27702790
if (!rel || !rel[0]) {
@@ -2787,6 +2807,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
27872807
/* Truncate at this slash → pathbuf is the directory prefix; the
27882808
* upsert binds copies, so restore the slash right after. */
27892809
*p = '\0';
2810+
int64_t *cached = dir_ids ? (int64_t *)cbm_ht_get(dir_ids, pathbuf) : NULL;
2811+
if (cached) {
2812+
parent = *cached;
2813+
*p = '/';
2814+
continue;
2815+
}
27902816
const char *seg = strrchr(pathbuf, '/');
27912817
cbm_node_t folder = {.project = covproj,
27922818
.label = "Folder",
@@ -2795,8 +2821,9 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
27952821
.file_path = pathbuf,
27962822
.properties_json = "{}"};
27972823
int64_t fid = cbm_store_upsert_node(s, &folder);
2798-
*p = '/';
27992824
if (fid <= 0) {
2825+
*p = '/';
2826+
cov_dir_ids_free(dir_ids);
28002827
cbm_store_free_coverage(rows, count);
28012828
return CBM_STORE_ERR;
28022829
}
@@ -2806,9 +2833,24 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
28062833
.type = "CONTAINS_FOLDER",
28072834
.properties_json = "{}"};
28082835
if (cbm_store_insert_edge(s, &e) <= 0) {
2836+
*p = '/';
2837+
cov_dir_ids_free(dir_ids);
28092838
cbm_store_free_coverage(rows, count);
28102839
return CBM_STORE_ERR;
28112840
}
2841+
if (dir_ids) {
2842+
int64_t *idv = (int64_t *)malloc(sizeof(*idv));
2843+
if (idv) {
2844+
*idv = fid;
2845+
char *kdup = strdup(pathbuf);
2846+
if (kdup) {
2847+
cbm_ht_set(dir_ids, kdup, idv);
2848+
} else {
2849+
free(idv);
2850+
}
2851+
}
2852+
}
2853+
*p = '/';
28122854
parent = fid;
28132855
}
28142856

@@ -2826,6 +2868,7 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
28262868
.properties_json = props};
28272869
int64_t file_id = cbm_store_upsert_node(s, &file);
28282870
if (file_id <= 0) {
2871+
cov_dir_ids_free(dir_ids);
28292872
cbm_store_free_coverage(rows, count);
28302873
return CBM_STORE_ERR;
28312874
}
@@ -2835,10 +2878,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
28352878
.type = "CONTAINS_FILE",
28362879
.properties_json = "{}"};
28372880
if (cbm_store_insert_edge(s, &e) <= 0) {
2881+
cov_dir_ids_free(dir_ids);
28382882
cbm_store_free_coverage(rows, count);
28392883
return CBM_STORE_ERR;
28402884
}
28412885
}
2886+
cov_dir_ids_free(dir_ids);
28422887
cbm_store_free_coverage(rows, count);
28432888
{
28442889
sqlite3_stmt *set = NULL;
@@ -2862,6 +2907,15 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
28622907
if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) {
28632908
return CBM_STORE_ERR;
28642909
}
2910+
/* Sub-block timings (publish.timing style): coverage_replace measured 9 s
2911+
* on the TypeScript corpus and the caller-level block could not say WHY —
2912+
* delete, 13k row inserts with large error_ranges payloads, the NOT-IN
2913+
* prune, meta, and COMMIT are very different suspects. */
2914+
struct timespec cov_t0;
2915+
struct timespec cov_t1;
2916+
long cov_ms[5] = {0, 0, 0, 0, 0};
2917+
long long cov_detail_bytes = 0;
2918+
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t0);
28652919
sqlite3_stmt *del = NULL;
28662920
if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND,
28672921
&del, NULL) != SQLITE_OK) {
@@ -2877,6 +2931,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
28772931
(void)exec_sql(s, "ROLLBACK;");
28782932
return CBM_STORE_ERR;
28792933
}
2934+
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
2935+
cov_ms[0] =
2936+
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
2937+
cov_t0 = cov_t1;
28802938
sqlite3_stmt *ins = NULL;
28812939
if (sqlite3_prepare_v2(
28822940
s->db,
@@ -2894,6 +2952,7 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
28942952
bind_text(ins, SKIP_ONE, project);
28952953
bind_text(ins, ST_COL_2, rows[i].rel_path);
28962954
bind_text(ins, ST_COL_3, rows[i].kind);
2955+
cov_detail_bytes += rows[i].detail ? (long long)strlen(rows[i].detail) : 0;
28972956
bind_text(ins, CBM_SZ_4, rows[i].detail ? rows[i].detail : "");
28982957
if (sqlite3_step(ins) != SQLITE_DONE) {
28992958
store_set_error_sqlite(s, "coverage insert");
@@ -2904,6 +2963,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
29042963
sqlite3_reset(ins);
29052964
}
29062965
sqlite3_finalize(ins);
2966+
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
2967+
cov_ms[1] =
2968+
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
2969+
cov_t0 = cov_t1;
29072970
/* Prune FAILURE rows for files no longer known to the index (deleted from
29082971
* the repo): file_hashes is the authoritative live-file set after
29092972
* persist. By-design not_indexed_* rows are exempt — deliberately
@@ -2927,6 +2990,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
29272990
(void)exec_sql(s, "ROLLBACK;");
29282991
return CBM_STORE_ERR;
29292992
}
2993+
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
2994+
cov_ms[2] =
2995+
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
2996+
cov_t0 = cov_t1;
29302997

29312998
if (meta) {
29322999
char recorded_at[CBM_SZ_64];
@@ -3002,7 +3069,33 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
30023069
(void)exec_sql(s, "ROLLBACK;");
30033070
return CBM_STORE_ERR;
30043071
}
3005-
return exec_sql(s, "COMMIT;");
3072+
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
3073+
cov_ms[3] =
3074+
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
3075+
cov_t0 = cov_t1;
3076+
int commit_rc = exec_sql(s, "COMMIT;");
3077+
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
3078+
cov_ms[4] =
3079+
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
3080+
{
3081+
char b0[ST_BUF_64];
3082+
char b1[ST_BUF_64];
3083+
char b2[ST_BUF_64];
3084+
char b3[ST_BUF_64];
3085+
char b4[ST_BUF_64];
3086+
char bn[ST_BUF_64];
3087+
char bb[ST_BUF_64];
3088+
snprintf(b0, sizeof(b0), "%ld", cov_ms[0]);
3089+
snprintf(b1, sizeof(b1), "%ld", cov_ms[1]);
3090+
snprintf(b2, sizeof(b2), "%ld", cov_ms[2]);
3091+
snprintf(b3, sizeof(b3), "%ld", cov_ms[3]);
3092+
snprintf(b4, sizeof(b4), "%ld", cov_ms[4]);
3093+
snprintf(bn, sizeof(bn), "%d", count);
3094+
snprintf(bb, sizeof(bb), "%lld", cov_detail_bytes);
3095+
cbm_log_info("publish.timing.coverage", "del", b0, "rows", b1, "prune", b2, "meta", b3,
3096+
"commit", b4, "row_count", bn, "detail_bytes", bb);
3097+
}
3098+
return commit_rc;
30063099
}
30073100

30083101
int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows,

0 commit comments

Comments
 (0)