Skip to content

Commit b5185d1

Browse files
authored
Merge pull request DeusData#2095 from DeusData/fix/1973-nested-gitignore-depth
fix(discover): stack every nested .gitignore on a path, not just the shallowest (DeusData#1973)
2 parents 6741c29 + d3eb7e9 commit b5185d1

2 files changed

Lines changed: 232 additions & 51 deletions

File tree

‎src/discover/discover.c‎

Lines changed: 103 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -549,6 +549,61 @@ static const char *local_rel_path(const char *rel_path, const char *local_prefix
549549
return rel_path;
550550
}
551551

552+
/* One .gitignore on the path from the repository root down to a walk frame.
553+
* Links form a root->leaf chain; every frame below the directory that owns a
554+
* matcher borrows a pointer to the deepest link governing it, so a directory
555+
* without a .gitignore of its own simply shares its parent's link. Links live
556+
* on the heap (the frame stack is realloc'd and popped) and walk_dir owns them
557+
* through the `owned_next` list. `prefix` is the walk-relative directory the
558+
* matcher was loaded from ("" for the root). */
559+
typedef struct gitignore_link {
560+
const cbm_gitignore_t *gi;
561+
const struct gitignore_link *parent;
562+
struct gitignore_link *owned_next;
563+
char prefix[];
564+
} gitignore_link_t;
565+
566+
static gitignore_link_t *gitignore_link_new(const cbm_gitignore_t *gi, const char *prefix,
567+
const gitignore_link_t *parent,
568+
gitignore_link_t **owned_links) {
569+
size_t prefix_size = strlen(prefix) + SKIP_ONE;
570+
gitignore_link_t *link = malloc(sizeof(*link) + prefix_size);
571+
if (!link) {
572+
return NULL;
573+
}
574+
link->gi = gi;
575+
link->parent = parent;
576+
memcpy(link->prefix, prefix, prefix_size);
577+
link->owned_next = *owned_links;
578+
*owned_links = link;
579+
return link;
580+
}
581+
582+
static void gitignore_links_free(gitignore_link_t *link) {
583+
while (link) {
584+
gitignore_link_t *next = link->owned_next;
585+
free(link);
586+
link = next;
587+
}
588+
}
589+
590+
/* Verdict of every .gitignore between the repository root and the directory
591+
* being walked, with git's precedence: the deepest file that has an opinion
592+
* wins (a lower-level file takes precedence over every higher-level one), and
593+
* within one file the last matching pattern wins. Returns >0 ignored, <0
594+
* re-included by a negation, 0 when no file mentions the path. Cost is one
595+
* match per .gitignore on the path — O(depth), never a rescan. */
596+
static int gitignore_chain_result(const gitignore_link_t *link, const char *rel_path, bool is_dir) {
597+
for (; link; link = link->parent) {
598+
int verdict =
599+
cbm_gitignore_match_result(link->gi, local_rel_path(rel_path, link->prefix), is_dir);
600+
if (verdict != 0) {
601+
return verdict;
602+
}
603+
}
604+
return 0;
605+
}
606+
552607
/* Non-negatable safety core: built-in skip dirs that a .cbmignore negation
553608
* can NEVER un-skip. A repo-committed .cbmignore must not be able to defeat
554609
* OOM/safety skips: .git holds VCS internals (and the info/exclude sources,
@@ -601,10 +656,10 @@ static bool dir_is_cache_tree(const char *abs_path) {
601656
}
602657

603658
static bool should_skip_directory(const char *entry_name, const char *rel_path,
604-
const cbm_discover_opts_t *opts, const cbm_gitignore_t *gitignore,
659+
const cbm_discover_opts_t *opts,
660+
const gitignore_link_t *ignore_chain,
605661
const cbm_gitignore_t *global_gi,
606-
const cbm_gitignore_t *cbmignore, const cbm_gitignore_t *local_gi,
607-
const char *local_gi_prefix) {
662+
const cbm_gitignore_t *cbmignore) {
608663
if (cbm_should_skip_dir(entry_name, opts ? opts->mode : CBM_MODE_FULL)) {
609664
/* #500: a .cbmignore negation (e.g. "!obj/") whose rule is the last
610665
* match for this dir un-skips a built-in skip-list dir — except the
@@ -616,16 +671,10 @@ static bool should_skip_directory(const char *entry_name, const char *rel_path,
616671
return true;
617672
}
618673
}
619-
if (gitignore && cbm_gitignore_matches(gitignore, rel_path, true)) {
674+
if (gitignore_chain_result(ignore_chain, rel_path, true) > 0) {
620675
return true;
621676
}
622677
bool global_ignored = global_gi && cbm_gitignore_matches(global_gi, rel_path, true);
623-
if (local_gi) {
624-
const char *lrel = local_rel_path(rel_path, local_gi_prefix);
625-
if (cbm_gitignore_matches(local_gi, lrel, true)) {
626-
return true;
627-
}
628-
}
629678
if (cbmignore) {
630679
int cbm_result = cbm_gitignore_match_result(cbmignore, rel_path, true);
631680
if (cbm_result > 0) {
@@ -645,11 +694,9 @@ static bool should_skip_directory(const char *entry_name, const char *rel_path,
645694
* match — are IDENTICAL to the original boolean predicate. */
646695
static const char *file_skip_reason(const char *entry_name, const char *rel_path,
647696
const cbm_discover_opts_t *opts,
648-
const cbm_gitignore_t *gitignore,
697+
const gitignore_link_t *ignore_chain,
649698
const cbm_gitignore_t *global_gi,
650-
const cbm_gitignore_t *cbmignore,
651-
const cbm_gitignore_t *local_gi, const char *local_gi_prefix,
652-
off_t file_size) {
699+
const cbm_gitignore_t *cbmignore, off_t file_size) {
653700
cbm_index_mode_t mode = opts ? opts->mode : CBM_MODE_FULL;
654701
if (cbm_has_ignored_suffix(entry_name, mode)) {
655702
return "ignored-suffix";
@@ -660,16 +707,10 @@ static const char *file_skip_reason(const char *entry_name, const char *rel_path
660707
if (cbm_matches_fast_pattern(entry_name, mode)) {
661708
return "fast-pattern";
662709
}
663-
if (gitignore && cbm_gitignore_matches(gitignore, rel_path, false)) {
710+
if (gitignore_chain_result(ignore_chain, rel_path, false) > 0) {
664711
return "gitignore";
665712
}
666713
bool global_ignored = global_gi && cbm_gitignore_matches(global_gi, rel_path, false);
667-
if (local_gi) {
668-
const char *lrel = local_rel_path(rel_path, local_gi_prefix);
669-
if (cbm_gitignore_matches(local_gi, lrel, false)) {
670-
return "gitignore";
671-
}
672-
}
673714
if (cbmignore) {
674715
int cbm_result = cbm_gitignore_match_result(cbmignore, rel_path, false);
675716
if (cbm_result > 0) {
@@ -792,12 +833,12 @@ static int safe_stat(const char *abs_path, struct stat *st, bool *is_symlink) {
792833

793834
/* Process a single regular file entry during directory walk. */
794835
static void walk_dir_process_file(const char *abs_path, const char *rel_path, const char *name,
795-
const cbm_discover_opts_t *opts, const cbm_gitignore_t *gitignore,
836+
const cbm_discover_opts_t *opts,
837+
const gitignore_link_t *ignore_chain,
796838
const cbm_gitignore_t *global_gi,
797-
const cbm_gitignore_t *cbmignore, const cbm_gitignore_t *local_gi,
798-
const char *local_gi_prefix, off_t size, file_list_t *out) {
799-
const char *skip_reason = file_skip_reason(name, rel_path, opts, gitignore, global_gi,
800-
cbmignore, local_gi, local_gi_prefix, size);
839+
const cbm_gitignore_t *cbmignore, off_t size, file_list_t *out) {
840+
const char *skip_reason =
841+
file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, size);
801842
if (skip_reason) {
802843
/* Deliberately not indexed (#963) — record so callers can surface it.
803844
* Unsupported-language files below are NOT recorded: "no grammar for
@@ -816,8 +857,7 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co
816857
typedef struct {
817858
char dir[CBM_SZ_4K];
818859
char prefix[CBM_SZ_4K];
819-
cbm_gitignore_t *local_gi; /* nested .gitignore for this subtree */
820-
char local_gi_prefix[CBM_SZ_4K]; /* rel_prefix when local_gi was loaded */
860+
const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */
821861
} walk_frame_t;
822862
/* Initial capacity only — the stack grows on demand. A single directory can
823863
* hold more pending sibling frames than any fixed cap (dotnet/runtime has 855
@@ -831,9 +871,15 @@ typedef struct {
831871
int cap;
832872
} walk_stack_t;
833873
/* Build abs/rel paths and process one directory entry. */
834-
/* Try to load a nested .gitignore from this directory. Returns owned pointer or NULL. */
874+
/* Try to load a nested .gitignore from this directory. Returns owned pointer or
875+
* NULL. Every directory below the root is probed, whether or not an ancestor
876+
* already contributed a matcher: git stacks ALL .gitignore files on a path, and
877+
* skipping the deeper ones once a shallower one existed was the #1973 blow-up
878+
* (an EMPTY storage/.gitignore hid storage/dump/.gitignore's "*", so thousands
879+
* of git-ignored dumps were discovered and indexed until the OOM killer hit).
880+
* The root's own .gitignore is loaded by cbm_discover (merged with info/exclude). */
835881
static cbm_gitignore_t *try_load_nested_gitignore(const walk_frame_t *frame) {
836-
if (frame->local_gi || frame->prefix[0] == '\0') {
882+
if (frame->prefix[0] == '\0') {
837883
return NULL;
838884
}
839885
char gi_path[CBM_SZ_4K];
@@ -868,19 +914,12 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char
868914
out->failed = true;
869915
return;
870916
}
871-
slot->local_gi = parent->local_gi;
872-
int local_prefix_length =
873-
snprintf(slot->local_gi_prefix, CBM_SZ_4K, "%s", parent->local_gi_prefix);
874-
if (local_prefix_length < 0 || local_prefix_length >= CBM_SZ_4K) {
875-
out->failed = true;
876-
return;
877-
}
917+
slot->ignore_chain = parent->ignore_chain;
878918
ws->top++;
879919
}
880920

881921
static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *frame,
882922
const cbm_discover_opts_t *opts,
883-
const cbm_gitignore_t *gitignore,
884923
const cbm_gitignore_t *global_gi,
885924
const cbm_gitignore_t *cbmignore, walk_stack_t *ws,
886925
file_list_t *out) {
@@ -915,16 +954,16 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram
915954

916955
if (S_ISDIR(st.st_mode)) {
917956
if (!dir_is_cache_tree(abs_path) &&
918-
!should_skip_directory(entry->name, rel_path, opts, gitignore, global_gi, cbmignore,
919-
frame->local_gi, frame->local_gi_prefix)) {
957+
!should_skip_directory(entry->name, rel_path, opts, frame->ignore_chain, global_gi,
958+
cbmignore)) {
920959
walk_push_subdir(ws, abs_path, rel_path, frame, out);
921960
} else {
922961
/* Record the excluded subtree root so callers can report it (#411). */
923962
file_list_add_excluded(out, rel_path);
924963
}
925964
} else if (S_ISREG(st.st_mode)) {
926-
walk_dir_process_file(abs_path, rel_path, entry->name, opts, gitignore, global_gi,
927-
cbmignore, frame->local_gi, frame->local_gi_prefix, st.st_size, out);
965+
walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi,
966+
cbmignore, st.st_size, out);
928967
}
929968
}
930969

@@ -958,11 +997,12 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis
958997
out->failed = true;
959998
return;
960999
}
961-
/* Collect all owned gitignores — freed at the end because child frames
962-
* on the stack hold borrowed pointers to them. */
1000+
/* Collect all owned gitignores and chain links — freed at the end because
1001+
* child frames on the stack hold borrowed pointers to them. */
9631002
cbm_gitignore_t **owned_gis = NULL;
9641003
size_t owned_count = 0;
9651004
size_t owned_capacity = 0;
1005+
gitignore_link_t *owned_links = NULL;
9661006

9671007
int initial_directory_length = snprintf(ws.frames[0].dir, CBM_SZ_4K, "%s", dir_path);
9681008
int initial_prefix_length = snprintf(ws.frames[0].prefix, CBM_SZ_4K, "%s", rel_prefix);
@@ -972,23 +1012,34 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis
9721012
free(ws.frames);
9731013
return;
9741014
}
1015+
if (gitignore) {
1016+
ws.frames[0].ignore_chain = gitignore_link_new(gitignore, rel_prefix, NULL, &owned_links);
1017+
if (!ws.frames[0].ignore_chain) {
1018+
out->failed = true;
1019+
free(ws.frames);
1020+
return;
1021+
}
1022+
}
9751023
ws.top++;
9761024

9771025
while (ws.top > 0 && !file_list_should_stop(out)) {
9781026
walk_frame_t frame = ws.frames[--ws.top];
9791027

9801028
cbm_gitignore_t *loaded = try_load_nested_gitignore(&frame);
9811029
if (loaded) {
982-
int local_prefix_length =
983-
snprintf(frame.local_gi_prefix, sizeof(frame.local_gi_prefix), "%s", frame.prefix);
984-
if (local_prefix_length < 0 ||
985-
(size_t)local_prefix_length >= sizeof(frame.local_gi_prefix) ||
986-
!walk_owned_gitignore_append(&owned_gis, &owned_count, &owned_capacity, loaded)) {
1030+
if (!walk_owned_gitignore_append(&owned_gis, &owned_count, &owned_capacity, loaded)) {
9871031
cbm_gitignore_free(loaded);
9881032
out->failed = true;
9891033
break;
9901034
}
991-
frame.local_gi = loaded;
1035+
/* owned_gis owns `loaded` from here on, even if the link fails. */
1036+
const gitignore_link_t *link =
1037+
gitignore_link_new(loaded, frame.prefix, frame.ignore_chain, &owned_links);
1038+
if (!link) {
1039+
out->failed = true;
1040+
break;
1041+
}
1042+
frame.ignore_chain = link;
9921043
}
9931044

9941045
cbm_dir_t *d = cbm_opendir(frame.dir);
@@ -1001,14 +1052,15 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis
10011052

10021053
cbm_dirent_t *entry;
10031054
while (!file_list_should_stop(out) && (entry = cbm_readdir(d)) != NULL) {
1004-
walk_dir_process_entry(entry, &frame, opts, gitignore, global_gi, cbmignore, &ws, out);
1055+
walk_dir_process_entry(entry, &frame, opts, global_gi, cbmignore, &ws, out);
10051056
}
10061057
cbm_closedir(d);
10071058
}
10081059
for (size_t i = 0; i < owned_count; i++) {
10091060
cbm_gitignore_free(owned_gis[i]);
10101061
}
10111062
free(owned_gis);
1063+
gitignore_links_free(owned_links);
10121064
free(ws.frames);
10131065
}
10141066

0 commit comments

Comments
 (0)