Skip to content

Commit 6ae6cd4

Browse files
cdeustclaude
andcommitted
fix(pipeline): address doclinks review feedback
Replace raw malloc/free with cbm_alloc/cbm_free (CBM_MEM_CLASS_EXTRACT) and a growable CBM_DYN_ARRAY for matches. Remove the 4KiB line-copy cap; lines are split in place instead of copied. Remove the DOCLINK_MAX_REFS(256) cap; the ref vector now grows, is sorted by target id, then collapsed. Track reference kind (bare, relative, rooted) so /x never attempts a directory join and ./x never falls back to root. Report per-file skips via cbm_pipeline_add_file_error and cbm_log_warn, and check cbm_pipeline_check_cancel in the per-file loop. Add negative-control tests for all of the above, plus CRLF handling, fenced-code-block pinning, bare URLs, backtick-basename exactness, and ".." escaping. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Signed-off-by: Clément Deust <clement.deust@gmail.com>
1 parent f3bdf11 commit 6ae6cd4

5 files changed

Lines changed: 432 additions & 80 deletions

File tree

‎src/pipeline/pass_doclinks.c‎

Lines changed: 154 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
#include "foundation/log.h"
3131
#include "foundation/compat_fs.h"
3232
#include "foundation/limits.h"
33+
#include "foundation/mem_core.h"
34+
#include "foundation/dyn_array.h"
3335

3436
#include <stdbool.h>
3537
#include <stdint.h>
@@ -50,8 +52,7 @@
5052
#define DOCLINK_EDGE_TYPE "REFERENCES_FILE"
5153

5254
enum {
53-
DOCLINK_MAX_REFS = CBM_SZ_256, /* distinct targets per referencing file */
54-
DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */
55+
DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */
5556
};
5657

5758
/* ── Path classification ─────────────────────────────────────────── */
@@ -71,23 +72,51 @@ static bool doclink_is_markdown_path(const char *path) {
7172
return ext && (strcmp(ext, ".md") == 0 || strcmp(ext, ".mdx") == 0);
7273
}
7374

74-
/* ── File reading (mirrors pass_semantic.c read_file, minus TS pad) ── */
75-
76-
static char *doclink_read_file(const char *path) {
75+
/* ── File reading (mirrors pass_definitions.c read_file, minus TS pad) ──
76+
*
77+
* Reports *out_status so the caller can attribute a skip to the right
78+
* reason (open failure vs. oversized vs. OOM) instead of a silent drop —
79+
* hitting a limit here degrades to a REPORTED skip, per limits.h. */
80+
static char *doclink_read_file(const char *path, long *out_size, cbm_read_status_t *out_status) {
81+
if (out_size) {
82+
*out_size = 0;
83+
}
84+
if (out_status) {
85+
*out_status = CBM_READ_OK;
86+
}
7787
FILE *f = cbm_fopen(path, "rb");
7888
if (!f) {
89+
if (out_status) {
90+
*out_status = CBM_READ_OPEN_FAIL;
91+
}
7992
return NULL;
8093
}
8194
(void)fseek(f, 0, SEEK_END);
8295
long size = ftell(f);
8396
(void)fseek(f, 0, SEEK_SET);
84-
if (size <= 0 || size > cbm_max_file_bytes()) {
97+
if (out_size) {
98+
*out_size = size;
99+
}
100+
if (size <= 0) {
101+
(void)fclose(f);
102+
if (out_status) {
103+
*out_status = CBM_READ_EMPTY;
104+
}
105+
return NULL;
106+
}
107+
if (size > cbm_max_file_bytes()) {
85108
(void)fclose(f);
109+
if (out_status) {
110+
*out_status = CBM_READ_OVERSIZED;
111+
}
86112
return NULL;
87113
}
88-
char *buf = malloc((size_t)size + SKIP_ONE);
114+
char *buf = cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)size + SKIP_ONE);
89115
if (!buf) {
90116
(void)fclose(f);
117+
if (out_status) {
118+
*out_status = CBM_READ_OOM;
119+
}
91120
return NULL;
92121
}
93122
size_t nread = fread(buf, SKIP_ONE, (size_t)size, f);
@@ -140,41 +169,53 @@ static bool doclink_normalize(const char *in, char *out, size_t out_sz) {
140169
return out_len > 0;
141170
}
142171

143-
/* Per-file accumulator: dedupes repeated references to the same target. */
172+
/* One raw match before dedup. Collected in a growable vector (no per-file
173+
* cap) and collapsed by doclink_flush(). */
144174
typedef struct {
145175
int64_t target_id;
146-
int count;
147176
double confidence;
148177
const char *strategy; /* static string literal */
149-
} doclink_ref_t;
178+
} doclink_match_t;
150179

151180
typedef struct {
152181
cbm_gbuf_t *gb;
153182
CBMHashTable *files_by_path; /* rel_path → cbm_gbuf_node_t* (borrowed) */
154183
const cbm_gbuf_node_t *src; /* referencing File node */
155184
char src_dir[CBM_SZ_512]; /* its directory ("" at repo root) */
156-
doclink_ref_t refs[DOCLINK_MAX_REFS];
157-
int ref_count;
158-
bool truncated;
185+
CBM_DYN_ARRAY(doclink_match_t) matches;
159186
} doclink_ctx_t;
160187

161-
/* Resolve a reference against the referencing file's directory, then the
162-
* repo root, mirroring how humans write doc links. Returns the already-
163-
* indexed File node or NULL — unresolvable references are dropped. */
188+
/* How a reference was written, which decides where it may resolve. Mixing
189+
* these up let a rooted "/x" still match a same-named file in the
190+
* referencing directory, and let an explicit "./x" fall back to an
191+
* unrelated same-named file at the repo root. */
192+
typedef enum {
193+
DOCLINK_REF_BARE = 0, /* "x": referencing dir, then repo root */
194+
DOCLINK_REF_RELATIVE, /* "./x": referencing dir only */
195+
DOCLINK_REF_ROOTED, /* "/x": repo root only */
196+
} doclink_ref_kind_t;
197+
198+
/* Resolve a reference against the referencing file's directory and/or the
199+
* repo root, per its kind. Returns the already-indexed File node or NULL —
200+
* unresolvable references are dropped. */
164201
static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref) {
165202
const char *r = ref;
166-
while (r[0] == '.' && r[SKIP_ONE] == '/') {
167-
r += PAIR_LEN;
168-
}
169-
if (r[0] == '/') {
203+
doclink_ref_kind_t kind = DOCLINK_REF_BARE;
204+
if (r[0] == '.' && r[SKIP_ONE] == '/') {
205+
kind = DOCLINK_REF_RELATIVE;
206+
while (r[0] == '.' && r[SKIP_ONE] == '/') {
207+
r += PAIR_LEN;
208+
}
209+
} else if (r[0] == '/') {
210+
kind = DOCLINK_REF_ROOTED;
170211
r++; /* "/docs/x.md" is repo-root-relative by doc convention */
171212
}
172213
if (r[0] == '\0') {
173214
return NULL;
174215
}
175216

176217
char norm[CBM_SZ_512];
177-
if (dc->src_dir[0] != '\0') {
218+
if (kind != DOCLINK_REF_ROOTED && dc->src_dir[0] != '\0') {
178219
char joined[CBM_SZ_512];
179220
int n = snprintf(joined, sizeof(joined), "%s/%s", dc->src_dir, r);
180221
if (n > 0 && (size_t)n < sizeof(joined) && doclink_normalize(joined, norm, sizeof(norm))) {
@@ -184,61 +225,67 @@ static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref
184225
}
185226
}
186227
}
187-
if (doclink_normalize(r, norm, sizeof(norm))) {
228+
if (kind != DOCLINK_REF_RELATIVE && doclink_normalize(r, norm, sizeof(norm))) {
188229
return cbm_ht_get(dc->files_by_path, norm);
189230
}
190231
return NULL;
191232
}
192233

193-
/* Record one match. Same-pair repeats bump the count; a higher-confidence
194-
* strategy upgrades the edge's confidence + strategy label. */
234+
/* Record one match. No cap, no dedup here — dc->matches is a plain append
235+
* log; doclink_flush() sorts and collapses same-target matches so a
236+
* generated file with hundreds of distinct targets loses nothing. */
195237
static void doclink_record(doclink_ctx_t *dc, const cbm_gbuf_node_t *target, double confidence,
196238
const char *strategy) {
197239
if (!target || target->id == dc->src->id) {
198240
return; /* never self-reference */
199241
}
200-
for (int i = 0; i < dc->ref_count; i++) {
201-
if (dc->refs[i].target_id == target->id) {
202-
dc->refs[i].count++;
203-
if (confidence > dc->refs[i].confidence) {
204-
dc->refs[i].confidence = confidence;
205-
dc->refs[i].strategy = strategy;
206-
}
207-
return;
208-
}
209-
}
210-
if (dc->ref_count >= DOCLINK_MAX_REFS) {
211-
dc->truncated = true;
212-
return;
213-
}
214-
dc->refs[dc->ref_count].target_id = target->id;
215-
dc->refs[dc->ref_count].count = SKIP_ONE;
216-
dc->refs[dc->ref_count].confidence = confidence;
217-
dc->refs[dc->ref_count].strategy = strategy;
218-
dc->ref_count++;
242+
doclink_match_t m = {.target_id = target->id, .confidence = confidence, .strategy = strategy};
243+
cbm_da_push(&dc->matches, m);
244+
}
245+
246+
static int doclink_match_cmp(const void *a, const void *b) {
247+
int64_t ta = ((const doclink_match_t *)a)->target_id;
248+
int64_t tb = ((const doclink_match_t *)b)->target_id;
249+
return (ta > tb) - (ta < tb);
219250
}
220251

221-
/* Emit accumulated references as REFERENCES_FILE edges. Returns edge count. */
252+
/* Emit accumulated references as REFERENCES_FILE edges. Sorts by target id
253+
* (O(M log M)) and collapses same-target runs: count = occurrences, the
254+
* highest-confidence match in the run wins strategy + confidence. Returns
255+
* edge count. */
222256
static int doclink_flush(doclink_ctx_t *dc) {
257+
if (dc->matches.count > 1) {
258+
qsort(dc->matches.items, (size_t)dc->matches.count, sizeof(dc->matches.items[0]),
259+
doclink_match_cmp);
260+
}
261+
223262
int emitted = 0;
224-
for (int i = 0; i < dc->ref_count; i++) {
263+
int i = 0;
264+
while (i < dc->matches.count) {
265+
int64_t target_id = dc->matches.items[i].target_id;
266+
double confidence = dc->matches.items[i].confidence;
267+
const char *strategy = dc->matches.items[i].strategy;
268+
int count = SKIP_ONE;
269+
int j = i + SKIP_ONE;
270+
while (j < dc->matches.count && dc->matches.items[j].target_id == target_id) {
271+
count++;
272+
if (dc->matches.items[j].confidence > confidence) {
273+
confidence = dc->matches.items[j].confidence;
274+
strategy = dc->matches.items[j].strategy;
275+
}
276+
j++;
277+
}
278+
225279
char props[CBM_SZ_256];
226280
(void)snprintf(props, sizeof(props),
227-
"{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}",
228-
dc->refs[i].strategy, dc->refs[i].confidence, dc->refs[i].count);
229-
if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, dc->refs[i].target_id, DOCLINK_EDGE_TYPE,
230-
props) > 0) {
281+
"{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}", strategy,
282+
confidence, count);
283+
if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, target_id, DOCLINK_EDGE_TYPE, props) > 0) {
231284
emitted++;
232285
}
286+
i = j;
233287
}
234-
if (dc->truncated) {
235-
char cap_buf[CBM_SZ_16];
236-
(void)snprintf(cap_buf, sizeof(cap_buf), "%d", DOCLINK_MAX_REFS);
237-
cbm_log_info("doclinks.truncated", "file", dc->src->file_path ? dc->src->file_path : "",
238-
"cap", cap_buf);
239-
}
240-
dc->ref_count = 0;
241-
dc->truncated = false;
288+
cbm_da_clear(&dc->matches);
242289
return emitted;
243290
}
244291

@@ -380,11 +427,15 @@ static void doclink_scan_md_line(doclink_ctx_t *dc, char *line) {
380427

381428
/* ── Per-file driver ─────────────────────────────────────────────── */
382429

383-
/* Scan one referencing file's content line by line and emit its edges. */
384-
static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, const char *source) {
430+
/* Scan one referencing file's content line by line and emit its edges.
431+
* `source` is the caller's private mutable buffer (freshly read for this
432+
* file, never reused): lines are terminated in place ('\n' -> '\0') and
433+
* scanned by pointer, so there is no fixed-size copy and no line-length
434+
* cap to straddle a token across (a prior CBM_SZ_4K copy buffer could cut
435+
* "src/foo.cpp" to "src/foo.c" at the boundary and bind to the wrong
436+
* file). A trailing '\r' (CRLF) is trimmed the same way. */
437+
static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, char *source) {
385438
dc->src = node;
386-
dc->ref_count = 0;
387-
dc->truncated = false;
388439
dc->src_dir[0] = '\0';
389440
const char *slash = strrchr(node->file_path, '/');
390441
if (slash) {
@@ -396,18 +447,20 @@ static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, con
396447
dc->src_dir[dlen] = '\0';
397448
}
398449

399-
const char *p = source;
400-
char line[CBM_SZ_4K];
450+
char *p = source;
401451
while (*p) {
402-
const char *eol = strchr(p, '\n');
403-
size_t line_len = eol ? (size_t)(eol - p) : strlen(p);
404-
if (line_len >= sizeof(line)) {
405-
line_len = sizeof(line) - SKIP_ONE;
452+
char *line = p;
453+
char *eol = strchr(p, '\n');
454+
if (eol) {
455+
*eol = '\0';
456+
p = eol + SKIP_ONE;
457+
} else {
458+
p += strlen(p);
459+
}
460+
size_t line_len = strlen(line);
461+
if (line_len > 0 && line[line_len - SKIP_ONE] == '\r') {
462+
line[line_len - SKIP_ONE] = '\0';
406463
}
407-
memcpy(line, p, line_len);
408-
line[line_len] = '\0';
409-
p = eol ? eol + SKIP_ONE : p + line_len;
410-
411464
doclink_scan_md_line(dc, line);
412465
}
413466
return doclink_flush(dc);
@@ -426,10 +479,16 @@ static bool doclink_has_doc_files(const cbm_gbuf_node_t *const *files, int file_
426479
}
427480

428481
/* Scan every markdown File node's on-disk content, emitting edges.
429-
* md_edges receives the emitted edge count. */
430-
static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path,
482+
* md_edges receives the emitted edge count. Checks cancellation once per
483+
* file — on a docs-heavy repo this is the whole pass's cancel latency —
484+
* and every skip is reported via cbm_pipeline_add_file_error, never
485+
* silent. */
486+
static void doclink_scan_repo(cbm_pipeline_ctx_t *ctx, doclink_ctx_t *dc, const char *repo_path,
431487
const cbm_gbuf_node_t *const *files, int file_count, int *md_edges) {
432488
for (int i = 0; i < file_count; i++) {
489+
if (cbm_pipeline_check_cancel(ctx)) {
490+
return;
491+
}
433492
if (!files[i]->file_path || !doclink_is_markdown_path(files[i]->file_path)) {
434493
continue;
435494
}
@@ -439,12 +498,29 @@ static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path,
439498
if (n <= 0 || (size_t)n >= sizeof(abs_path)) {
440499
continue;
441500
}
442-
char *source = doclink_read_file(abs_path);
501+
long file_size = 0;
502+
cbm_read_status_t rst = CBM_READ_OK;
503+
char *source = doclink_read_file(abs_path, &file_size, &rst);
443504
if (!source) {
505+
if (rst == CBM_READ_OVERSIZED) {
506+
long cap = cbm_max_file_bytes();
507+
char reason[96];
508+
(void)snprintf(reason, sizeof(reason), "oversized (%lld MB > %lld MB)",
509+
(long long)(file_size / (CBM_SZ_1K * CBM_SZ_1K)),
510+
(long long)(cap / (CBM_SZ_1K * CBM_SZ_1K)));
511+
cbm_pipeline_add_file_error(ctx->pipeline, files[i]->file_path, reason,
512+
"oversized");
513+
cbm_log_warn("doclinks.file_oversized", "path", files[i]->file_path);
514+
} else if (rst == CBM_READ_OPEN_FAIL || rst == CBM_READ_OOM) {
515+
cbm_pipeline_add_file_error(ctx->pipeline, files[i]->file_path, "read failed",
516+
"read");
517+
cbm_log_warn("doclinks.file_unreadable", "path", files[i]->file_path);
518+
}
519+
/* CBM_READ_EMPTY: benign 0-byte file, nothing to index, not reported. */
444520
continue;
445521
}
446522
int emitted = doclink_scan_file(dc, files[i], source);
447-
free(source);
523+
cbm_free(CBM_MEM_CLASS_EXTRACT, source);
448524
*md_edges += emitted;
449525
}
450526
}
@@ -458,7 +534,7 @@ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) {
458534
return 0;
459535
}
460536

461-
/* Early exit: no markdown/shell files means nothing to scan. */
537+
/* Early exit: no markdown files means nothing to scan. */
462538
if (!doclink_has_doc_files(files, file_count)) {
463539
cbm_log_info("doclinks.skip", "reason", "no_doc_files");
464540
return 0;
@@ -483,7 +559,8 @@ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) {
483559
}
484560

485561
int md_edges = 0;
486-
doclink_scan_repo(&dc, ctx->repo_path, files, file_count, &md_edges);
562+
doclink_scan_repo(ctx, &dc, ctx->repo_path, files, file_count, &md_edges);
563+
cbm_da_free(&dc.matches);
487564
cbm_ht_free(dc.files_by_path);
488565

489566
char buf1[CBM_SZ_16];

‎src/pipeline/pipeline_internal.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,7 @@ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project);
665665
/* Pre-dump pass: config ↔ code linking. */
666666
int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx);
667667

668-
/* Pre-dump pass: markdown/shell → file REFERENCES_FILE linking. */
668+
/* Pre-dump pass: markdown → file REFERENCES_FILE linking. */
669669
int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx);
670670

671671
/* Pre-dump pass: SIMILAR_TO edges via MinHash fingerprinting. */

0 commit comments

Comments
 (0)