diff --git a/src/discover/discover.c b/src/discover/discover.c index 3bf4f034c6..134dd18c88 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -14,13 +14,21 @@ #include "cbm.h" // CBMLanguage, CBM_LANG_COUNT, CBM_LANG_JSON #include "foundation/constants.h" +#include "foundation/compat.h" #include "foundation/compat_fs.h" +#include "foundation/hash_table.h" #include "foundation/limits.h" #include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "foundation/subprocess.h" #include "foundation/workspace.h" #include "foundation/platform.h" +#include "mcp/test_impact_engine.h" // cbm_test_impact_find_git #ifdef _WIN32 #include "foundation/win_utf8.h" +#include // _close +#else +#include // close #endif #include #include // int64_t @@ -436,6 +444,25 @@ bool cbm_matches_fast_pattern(const char *filename, cbm_index_mode_t mode) { /* ── Dynamic file list ────────────────────────── */ +/* Paths git tracks under the discovery root, and every directory above one. + * git applies ignore rules to untracked paths only, so .gitignore, + * info/exclude and the global excludes file never hide a tracked path. Both + * tables stay NULL when the root is not in a git repository or git cannot + * run, and the ignore rules then apply exactly as before. */ +typedef struct { + char *paths; /* `git ls-files -z` output; holds the keys of `files` */ + CBMHashTable *files; /* tracked path */ + CBMHashTable *dirs; /* directory above a tracked path (owned key) */ +} discover_tracked_t; + +static bool tracked_has_file(const discover_tracked_t *tracked, const char *rel_path) { + return tracked && tracked->files && cbm_ht_has(tracked->files, rel_path); +} + +static bool tracked_has_dir(const discover_tracked_t *tracked, const char *rel_path) { + return tracked && tracked->dirs && cbm_ht_has(tracked->dirs, rel_path); +} + typedef struct { cbm_file_info_t *files; int count; @@ -465,6 +492,7 @@ typedef struct { int ignored_count; int ignored_cap; int ignored_total; + const discover_tracked_t *tracked; } file_list_t; static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource, @@ -765,11 +793,16 @@ static bool dir_is_cache_tree(const char *abs_path) { return abs_path[n] == '\0' || abs_path[n] == '/' || abs_path[n] == '\\'; } -static bool should_skip_directory(const char *entry_name, const char *rel_path, - const cbm_discover_opts_t *opts, - const gitignore_link_t *ignore_chain, - const cbm_gitignore_t *global_gi, - const cbm_gitignore_t *cbmignore) { +/* What the ignore rules say about a directory. DIR_GIT_IGNORED comes from + * git's own rules (.gitignore files, info/exclude, the global excludes file), + * which yield to tracked content; built-in skips and .cbmignore never do. */ +typedef enum { DIR_KEEP, DIR_SKIP, DIR_GIT_IGNORED } dir_verdict_t; + +static dir_verdict_t directory_verdict(const char *entry_name, const char *rel_path, + const cbm_discover_opts_t *opts, + const gitignore_link_t *ignore_chain, + const cbm_gitignore_t *global_gi, + const cbm_gitignore_t *cbmignore) { if (cbm_should_skip_dir(entry_name, opts ? opts->mode : CBM_MODE_FULL) || cbm_discover_path_has_skip_suffix(rel_path)) { /* #500: a .cbmignore negation (e.g. "!obj/") whose rule is the last @@ -779,35 +812,36 @@ static bool should_skip_directory(const char *entry_name, const char *rel_path, bool unskipped = cbmignore && !cbm_discover_is_safety_core_directory(entry_name) && cbm_gitignore_match_result(cbmignore, rel_path, true) < 0; if (!unskipped) { - return true; + return DIR_SKIP; } } - if (gitignore_chain_result(ignore_chain, rel_path, true) > 0) { - return true; - } + bool git_ignored = gitignore_chain_result(ignore_chain, rel_path, true) > 0; bool global_ignored = global_gi && cbm_gitignore_matches(global_gi, rel_path, true); if (cbmignore) { int cbm_result = cbm_gitignore_match_result(cbmignore, rel_path, true); if (cbm_result > 0) { - return true; + return DIR_SKIP; } - if (cbm_result < 0 && global_ignored) { - return false; + if (cbm_result < 0) { + global_ignored = false; /* a .cbmignore negation un-ignores a global match */ } } - return global_ignored; + return git_ignored || global_ignored ? DIR_GIT_IGNORED : DIR_KEEP; } /* Check if a regular file should be skipped (filters + gitignore + size). */ /* Classify why a file is skipped. Returns a static reason string (the #963 * "purposely not indexed" class) or NULL to keep the file. Check order and * semantics — including the .cbmignore negation un-ignoring a global-gitignore - * match — are IDENTICAL to the original boolean predicate. */ + * match — are IDENTICAL to the original boolean predicate. git's own rules + * never hide a tracked file; inside a git-ignored directory (entered only for + * its tracked content) every untracked file is ignored, as in git. */ static const char *file_skip_reason(const char *entry_name, const char *rel_path, const cbm_discover_opts_t *opts, const gitignore_link_t *ignore_chain, const cbm_gitignore_t *global_gi, - const cbm_gitignore_t *cbmignore, off_t file_size) { + const cbm_gitignore_t *cbmignore, off_t file_size, bool tracked, + bool in_ignored_dir) { cbm_index_mode_t mode = opts ? opts->mode : CBM_MODE_FULL; if (cbm_has_ignored_suffix(entry_name, mode)) { return "ignored-suffix"; @@ -818,10 +852,11 @@ static const char *file_skip_reason(const char *entry_name, const char *rel_path if (cbm_matches_fast_pattern(entry_name, mode)) { return "fast-pattern"; } - if (gitignore_chain_result(ignore_chain, rel_path, false) > 0) { + if (!tracked && (in_ignored_dir || gitignore_chain_result(ignore_chain, rel_path, false) > 0)) { return "gitignore"; } - bool global_ignored = global_gi && cbm_gitignore_matches(global_gi, rel_path, false); + bool global_ignored = + !tracked && global_gi && cbm_gitignore_matches(global_gi, rel_path, false); if (cbmignore) { int cbm_result = cbm_gitignore_match_result(cbmignore, rel_path, false); if (cbm_result > 0) { @@ -1049,9 +1084,10 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co const gitignore_link_t *ignore_chain, const cbm_gitignore_t *global_gi, const cbm_gitignore_t *cbmignore, const sphinx_suffixes_t *sphinx, - const struct stat *st, file_list_t *out) { + bool in_ignored_dir, const struct stat *st, file_list_t *out) { const char *skip_reason = - file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size); + file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size, + tracked_has_file(out->tracked, rel_path), in_ignored_dir); if (skip_reason) { /* Deliberately not indexed (#963) — record so callers can surface it. * Unsupported-language files below are NOT recorded: "no grammar for @@ -1075,6 +1111,7 @@ typedef struct { char prefix[CBM_SZ_4K]; const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */ sphinx_suffixes_t sphinx; /* the documentation set's extra reST suffixes */ + bool git_ignored; /* ignored by git's rules, entered only for its tracked content */ } walk_frame_t; /* Initial capacity only — the stack grows on demand. A single directory can * hold more pending sibling frames than any fixed cap (dotnet/runtime has 855 @@ -1112,7 +1149,7 @@ static cbm_gitignore_t *try_load_nested_gitignore(const walk_frame_t *frame) { * context. Grows the stack geometrically; the caller's `parent` must not * point into the stack array (walk_dir pops into a local copy). */ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char *rel_path, - const walk_frame_t *parent, file_list_t *out) { + const walk_frame_t *parent, bool git_ignored, file_list_t *out) { if (ws->top >= ws->cap) { int new_cap = ws->cap * 2; walk_frame_t *grown = realloc(ws->frames, (size_t)new_cap * sizeof(*grown)); @@ -1133,6 +1170,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char } slot->ignore_chain = parent->ignore_chain; slot->sphinx = parent->sphinx; + slot->git_ignored = git_ignored; ws->top++; } @@ -1171,17 +1209,23 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram } if (S_ISDIR(st.st_mode)) { - if (!dir_is_cache_tree(abs_path) && - !should_skip_directory(entry->name, rel_path, opts, frame->ignore_chain, global_gi, - cbmignore)) { - walk_push_subdir(ws, abs_path, rel_path, frame, out); + dir_verdict_t verdict = dir_is_cache_tree(abs_path) + ? DIR_SKIP + : directory_verdict(entry->name, rel_path, opts, + frame->ignore_chain, global_gi, cbmignore); + if (verdict == DIR_KEEP && frame->git_ignored) { + verdict = DIR_GIT_IGNORED; /* below a git-ignored directory, as in git */ + } + if (verdict == DIR_KEEP || + (verdict == DIR_GIT_IGNORED && tracked_has_dir(out->tracked, rel_path))) { + walk_push_subdir(ws, abs_path, rel_path, frame, verdict == DIR_GIT_IGNORED, out); } else { /* Record the excluded subtree root so callers can report it (#411). */ file_list_add_excluded(out, rel_path); } } else if (S_ISREG(st.st_mode)) { walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi, - cbmignore, &frame->sphinx, &st, out); + cbmignore, &frame->sphinx, frame->git_ignored, &st, out); } } @@ -1315,6 +1359,131 @@ static bool discover_path_is_absolute(const char *path) { #endif } +static void tracked_free_key(const char *key, void *value, void *userdata) { + (void)value; + (void)userdata; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void tracked_free(discover_tracked_t *tracked) { + if (tracked->dirs) { + cbm_ht_foreach(tracked->dirs, tracked_free_key, NULL); + } + cbm_ht_free(tracked->dirs); + cbm_ht_free(tracked->files); + cbm_free(CBM_MEM_CLASS_OTHER, tracked->paths); + memset(tracked, 0, sizeof(*tracked)); +} + +/* The bytes of the capture file at path, NUL-terminated; NULL on any failure. */ +static char *tracked_slurp(const char *path, size_t *length) { + int64_t size = cbm_file_size(path); + if (size < 0 || (uint64_t)size >= SIZE_MAX) { + return NULL; + } + char *buffer = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SKIP_ONE); + FILE *fp = buffer ? cbm_fopen(path, "rb") : NULL; + bool ok = fp && fread(buffer, 1, (size_t)size, fp) == (size_t)size; + if (fp) { + (void)fclose(fp); + } + if (!ok) { + cbm_free(CBM_MEM_CLASS_OTHER, buffer); + return NULL; + } + buffer[size] = '\0'; + *length = (size_t)size; + return buffer; +} + +/* The whole `git ls-files -z` output for repo_path (paths relative to it), + * NUL-terminated; NULL when git cannot run or fails. Git runs without a + * shell: the repository path is one argv entry, and the child environment + * drops repository-local variables so an inherited GIT_DIR cannot redirect -C. */ +static char *tracked_read(const char *repo_path, size_t *length) { + *length = 0; + char git[CBM_SZ_4K]; + char capture[CBM_SZ_4K]; + int n = snprintf(capture, sizeof(capture), "%s/cbm-discover-git-XXXXXX", cbm_tmpdir()); + if (!cbm_test_impact_find_git(git, sizeof(git)) || n <= 0 || (size_t)n >= sizeof(capture)) { + return NULL; + } + int fd = cbm_mkstemp(capture); + if (fd < 0) { + return NULL; + } +#ifdef _WIN32 + (void)_close(fd); +#else + (void)close(fd); +#endif + const char *argv[] = {git, "-C", repo_path, "-c", "core.fsmonitor=false", + "ls-files", "-z", NULL}; + cbm_proc_opts_t opts = {.bin = git, + .argv = argv, + .cancel_grace_ms = CBM_SUBPROCESS_DEFAULT_CANCEL_GRACE_MS, + .strip_git_repo_env = true, + .stdout_file = capture}; + cbm_proc_result_t result = {0}; + bool ran = cbm_subprocess_run(&opts, &result) == 0 && result.outcome == CBM_PROC_CLEAN && + result.exit_code == 0; + char *buffer = ran ? tracked_slurp(capture, length) : NULL; + (void)cbm_unlink(capture); + return buffer; +} + +/* Record every directory above `path` (its '/'-separated prefixes). */ +static bool tracked_add_dirs(CBMHashTable *dirs, const char *path) { + char prefix[CBM_SZ_4K]; + for (const char *slash = strchr(path, '/'); slash; slash = strchr(slash + SKIP_ONE, '/')) { + size_t n = (size_t)(slash - path); + if (n == 0 || n >= sizeof(prefix)) { + continue; + } + memcpy(prefix, path, n); + prefix[n] = '\0'; + if (cbm_ht_has(dirs, prefix)) { + continue; + } + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, prefix); + if (!key) { + return false; + } + cbm_ht_set(dirs, key, key); + } + return true; +} + +/* Load what git tracks under repo_path; leaves `tracked` empty on any failure, + * so the ignore rules then apply exactly as they did without it. */ +static void tracked_load(discover_tracked_t *tracked, const char *repo_path) { + memset(tracked, 0, sizeof(*tracked)); + size_t length = 0; + tracked->paths = tracked_read(repo_path, &length); + if (!tracked->paths || length == 0) { + tracked_free(tracked); + return; + } + tracked->files = cbm_ht_create(CBM_SZ_1K); + tracked->dirs = cbm_ht_create(CBM_SZ_256); + if (!tracked->files || !tracked->dirs) { + tracked_free(tracked); + return; + } + for (size_t at = 0; at < length;) { + const char *path = tracked->paths + at; + size_t n = strlen(path); + if (n > 0) { + cbm_ht_set(tracked->files, path, (void *)path); + if (!tracked_add_dirs(tracked->dirs, path)) { + tracked_free(tracked); + return; + } + } + at += n + SKIP_ONE; + } +} + /* Resolve the shared "common" git directory for repo_path. * Handles three layouts: * 1. /.git is a directory - ordinary repo; common_dir == /.git @@ -1679,10 +1848,16 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc .collect_excluded = !count_only && excluded_out != NULL, .collect_ignored = !count_only && ignored_out != NULL, }; + discover_tracked_t tracked = {0}; + if (is_git_repo) { + tracked_load(&tracked, repo_path); + fl.tracked = &tracked; + } walk_cache_dir_snapshot(); walk_dir(repo_path, "", opts, gitignore, ancestors.chain, global_gi, cbmignore, &fl); /* Cleanup */ + tracked_free(&tracked); ancestor_ignores_free(&ancestors); cbm_gitignore_free(gitignore); cbm_gitignore_free(global_gi); diff --git a/tests/test_discover.c b/tests/test_discover.c index 2a3047332e..3b1c5a213f 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -891,6 +891,76 @@ TEST(discover_gitignore_dir_excluded_issue234) { PASS(); } +static int disc_git(const char *dir, const char *args) { + char cmd[1024]; + snprintf(cmd, sizeof(cmd), + "git -C \"%s\" -c user.name=t -c user.email=t@t.io -c init.defaultBranch=master %s", + dir, args); + return system(cmd); +} + +static bool disc_has(const cbm_file_info_t *files, int count, const char *rel_path) { + for (int i = 0; i < count; i++) { + if (strcmp(files[i].rel_path, rel_path) == 0) { + return true; + } + } + return false; +} + +/* A .gitignore never hides a file git tracks: git applies ignore rules to + * untracked paths only. cluster-api's nested `.gitignore` line `manager`, + * meant for a built binary, hid its tracked config/manager/ kustomization. + * Untracked ignored files stay out, also inside a directory that is entered + * only for its tracked content. */ +TEST(discover_gitignore_never_hides_tracked_files) { + char *base = th_mktempdir("cbm_disc_tracked"); + ASSERT(base != NULL); + if (disc_git(base, "init -q") != 0) { + th_cleanup(base); + FAIL("git init failed"); + } + th_write_file(TH_PATH(base, ".gitignore"), "manager\n*.gen.go\n"); + th_write_file(TH_PATH(base, "main.go"), "package main\n\nfunc main() {}\n"); + th_write_file(TH_PATH(base, "config/manager/manager.go"), "package manager\n\nfunc Run() {}\n"); + th_write_file(TH_PATH(base, "config/manager/scratch.go"), "package manager\n\nfunc S() {}\n"); + th_write_file(TH_PATH(base, "config/manager/sub/deep.go"), "package sub\n\nfunc Deep() {}\n"); + th_write_file(TH_PATH(base, "config/manager/tmp/build.go"), "package tmp\n\nfunc B() {}\n"); + th_write_file(TH_PATH(base, "api/types.gen.go"), "package api\n\nfunc Gen() {}\n"); + th_write_file(TH_PATH(base, "api/other.gen.go"), "package api\n\nfunc Other() {}\n"); + int add_rc = disc_git(base, "add .gitignore main.go"); + int force_rc = disc_git(base, "add -f config/manager/manager.go config/manager/sub/deep.go " + "api/types.gen.go"); + + cbm_discover_opts_t opts = {0}; + cbm_file_info_t *files = NULL; + int count = 0; + int rc = cbm_discover(base, &opts, &files, &count); + bool main_found = rc == 0 && disc_has(files, count, "main.go"); + bool manager_found = rc == 0 && disc_has(files, count, "config/manager/manager.go"); + bool deep_found = rc == 0 && disc_has(files, count, "config/manager/sub/deep.go"); + bool gen_found = rc == 0 && disc_has(files, count, "api/types.gen.go"); + bool scratch_found = rc == 0 && disc_has(files, count, "config/manager/scratch.go"); + bool build_found = rc == 0 && disc_has(files, count, "config/manager/tmp/build.go"); + bool other_found = rc == 0 && disc_has(files, count, "api/other.gen.go"); + cbm_discover_free(files, count); + th_cleanup(base); + + ASSERT_EQ(add_rc, 0); + ASSERT_EQ(force_rc, 0); + ASSERT_EQ(rc, 0); + ASSERT_TRUE(main_found); + /* Tracked: kept although .gitignore matches them or a directory above. */ + ASSERT_TRUE(manager_found); + ASSERT_TRUE(deep_found); + ASSERT_TRUE(gen_found); + /* Untracked and ignored: still out, also below the entered directory. */ + ASSERT_FALSE(scratch_found); + ASSERT_FALSE(build_found); + ASSERT_FALSE(other_found); + PASS(); +} + TEST(discover_max_file_size) { char *base = th_mktempdir("cbm_disc_size"); ASSERT(base != NULL); @@ -2382,6 +2452,7 @@ SUITE(discover) { RUN_TEST(discover_missing_global_excludes_is_noop); RUN_TEST(discover_cbmignore_negates_global_ignore); RUN_TEST(discover_gitignore_dir_excluded_issue234); + RUN_TEST(discover_gitignore_never_hides_tracked_files); RUN_TEST(discover_max_file_size); RUN_TEST(discover_null_path); RUN_TEST(discover_nonexistent_path);