Skip to content

Commit 98218a2

Browse files
fix(windows): use _wfopen for non-ASCII project paths (DeusData#700)
Standard fopen on Windows uses the ANSI/system codepage and silently fails to open files whose absolute path contains non-ASCII characters (accented, CJK, etc.). This causes the pipeline to skip parsing those files entirely — File nodes are created by directory traversal (which already uses wide Win32 APIs) but tree-sitter never receives the source, so no Function/Class/Method nodes are extracted. Add cbm_fopen(path, mode) to compat_fs: - Windows: converts the UTF-8 path to UTF-16LE via cbm_utf8_to_wide (existing helper in win_utf8.h) and calls _wfopen. - POSIX: delegates to fopen (no-op change, paths are already UTF-8). Replace every fopen call that operates on user project file paths: - src/pipeline/pass_parallel.c (primary indexing path) - src/pipeline/pass_definitions.c - src/pipeline/pass_calls.c - src/pipeline/pass_semantic.c - src/pipeline/pass_usages.c - src/pipeline/pass_lsp_cross.c - src/pipeline/pass_pkgmap.c - src/pipeline/pass_k8s.c - src/pipeline/pass_envscan.c - src/pipeline/path_alias.c (tsconfig detection) - src/pipeline/artifact.c - src/discover/language.c (content-sniff for language detection) - src/discover/gitignore.c - src/discover/discover.c - src/discover/userconfig.c - src/mcp/mcp.c (read_file_lines, adr.md reader) Fixes DeusData#700 Signed-off-by: ShauryaaSharma <shauryasofficial27@gmail.com>
1 parent 7824e50 commit 98218a2

18 files changed

Lines changed: 55 additions & 19 deletions

‎src/discover/discover.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ static bool expand_git_path(const char *path, char *out, size_t out_sz) {
245245
}
246246

247247
static bool read_core_excludes_file(const char *config_path, char *out, size_t out_sz) {
248-
FILE *f = fopen(config_path, "r");
248+
FILE *f = cbm_fopen(config_path, "r");
249249
if (!f) {
250250
return false;
251251
}

‎src/discover/gitignore.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
* - patterns with / are rooted (anchored to base)
1313
*/
1414
#include "foundation/constants.h"
15+
#include "foundation/compat_fs.h"
1516

1617
enum { GI_INIT_CAP = 16, GI_CHAR_IDX1 = 1, GI_CHAR_IDX2 = 2, GI_SKIP3 = 3 };
1718
#include "discover/discover.h"
@@ -287,7 +288,7 @@ cbm_gitignore_t *cbm_gitignore_load(const char *path) {
287288
return NULL;
288289
}
289290

290-
FILE *f = fopen(path, "r");
291+
FILE *f = cbm_fopen(path, "r");
291292
if (!f) {
292293
return NULL;
293294
}

‎src/discover/language.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "cbm.h" // CBMLanguage, CBM_LANG_*
1212

1313
#include "foundation/constants.h"
14+
#include "foundation/compat_fs.h"
1415

1516
enum { LANG_SCAN_PASSES = 2 };
1617
#define SLEN(s) (sizeof(s) - 1)
@@ -998,7 +999,7 @@ CBMLanguage cbm_disambiguate_m(const char *path) {
998999
return CBM_LANG_MATLAB;
9991000
}
10001001

1001-
FILE *f = fopen(path, "r");
1002+
FILE *f = cbm_fopen(path, "r");
10021003
if (!f) {
10031004
return CBM_LANG_MATLAB;
10041005
}

‎src/discover/userconfig.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "cbm.h" /* CBMLanguage, CBM_LANG_* */
1414
#include "foundation/constants.h"
1515
#include "foundation/platform.h" /* cbm_safe_getenv */
16+
#include "foundation/compat_fs.h"
1617

1718
enum { MAX_CONFIG_SIZE = 65536 };
1819
#include "foundation/log.h"
@@ -231,7 +232,7 @@ static int parse_extra_extensions(yyjson_val *root, cbm_userext_t **entries, int
231232
* Returns 0 on success (or absent file), -1 on alloc failure.
232233
*/
233234
static int load_config_file(const char *path, cbm_userext_t **entries, int *count) {
234-
FILE *f = fopen(path, "rb");
235+
FILE *f = cbm_fopen(path, "rb");
235236
if (!f) {
236237
return 0; /* file absent — silently ignore */
237238
}

‎src/foundation/compat_fs.c‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,22 @@ int cbm_pclose(FILE *f) {
130130
return _pclose(f);
131131
}
132132

133+
FILE *cbm_fopen(const char *path, const char *mode) {
134+
wchar_t *wpath = cbm_utf8_to_wide(path);
135+
if (!wpath) {
136+
return NULL;
137+
}
138+
wchar_t *wmode = cbm_utf8_to_wide(mode);
139+
if (!wmode) {
140+
free(wpath);
141+
return NULL;
142+
}
143+
FILE *f = _wfopen(wpath, wmode);
144+
free(wpath);
145+
free(wmode);
146+
return f;
147+
}
148+
133149
bool cbm_mkdir_p(const char *path, int mode) {
134150
(void)mode;
135151
wchar_t *wpath = cbm_utf8_to_wide(path);
@@ -262,6 +278,10 @@ int cbm_pclose(FILE *f) {
262278
return pclose(f);
263279
}
264280

281+
FILE *cbm_fopen(const char *path, const char *mode) {
282+
return fopen(path, mode);
283+
}
284+
265285
bool cbm_mkdir_p(const char *path, int mode) {
266286
/* Try direct mkdir first */
267287
if (mkdir(path, (mode_t)mode) == 0) {

‎src/foundation/compat_fs.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,16 @@ int cbm_unlink(const char *path);
5050
/* Delete an empty directory. Returns 0 on success. */
5151
int cbm_rmdir(const char *path);
5252

53+
/* Open a file by UTF-8 path.
54+
* On Windows, converts to wide-char and calls _wfopen so paths with
55+
* non-ASCII characters (accents, CJK, etc.) are handled correctly.
56+
* On POSIX, delegates to fopen. mode must be an ASCII string. */
57+
FILE *cbm_fopen(const char *path, const char *mode);
58+
5359
/* Execute a command without shell interpretation.
5460
* argv is a NULL-terminated array: {"cmd", "arg1", "arg2", NULL}.
5561
* Returns the process exit code, or -1 on fork/exec failure.
56-
* POSIX: fork() + execvp(). Windows: _spawnvp(). */
62+
* POSIX: fork() + execvp(). Windows: CreateProcess with proper quoting. */
5763
int cbm_exec_no_shell(const char *const *argv);
5864

5965
#endif /* CBM_COMPAT_FS_H */

‎src/mcp/mcp.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2782,7 +2782,7 @@ static void free_node_contents(cbm_node_t *n) {
27822782
/* ── Helper: read lines [start, end] from a file ─────────────── */
27832783

27842784
static char *read_file_lines(const char *path, int start, int end) {
2785-
FILE *fp = fopen(path, "r");
2785+
FILE *fp = cbm_fopen(path, "r");
27862786
if (!fp) {
27872787
return NULL;
27882788
}
@@ -4639,7 +4639,7 @@ static char *adr_read_legacy_file(const char *root_path) {
46394639
}
46404640
char adr_path[CBM_SZ_4K];
46414641
snprintf(adr_path, sizeof(adr_path), "%s/.codebase-memory/adr.md", root_path);
4642-
FILE *fp = fopen(adr_path, "r");
4642+
FILE *fp = cbm_fopen(adr_path, "r");
46434643
if (!fp) {
46444644
return NULL;
46454645
}

‎src/pipeline/artifact.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ static bool artifact_path(char *buf, size_t bufsz, const char *repo_path, const
122122

123123
/* Read entire file into malloc'd buffer. Sets *out_len. Returns NULL on error. */
124124
static char *read_file_alloc(const char *path, size_t *out_len) {
125-
FILE *fp = fopen(path, "rb");
125+
FILE *fp = cbm_fopen(path, "rb");
126126
if (!fp) {
127127
return NULL;
128128
}

‎src/pipeline/pass_calls.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ enum { PC_RING = 4, PC_RING_MASK = 3, PC_SIG_SCAN = 15, PC_REGEX_GRP = 2 };
2222
#include "graph_buffer/graph_buffer.h"
2323
#include "foundation/log.h"
2424
#include "foundation/compat.h"
25+
#include "foundation/compat_fs.h"
2526
#include "foundation/str_util.h"
2627
#include "cbm.h"
2728
#include "service_patterns.h"
@@ -43,7 +44,7 @@ static bool pc_module_is_dir(CBMLanguage lang) {
4344

4445
/* Read entire file into heap-allocated buffer. Caller must free(). */
4546
static char *read_file(const char *path, int *out_len) {
46-
FILE *f = fopen(path, "rb");
47+
FILE *f = cbm_fopen(path, "rb");
4748
if (!f) {
4849
return NULL;
4950
}

‎src/pipeline/pass_definitions.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };
2222
#include "graph_buffer/graph_buffer.h"
2323
#include "foundation/log.h"
2424
#include "foundation/compat.h"
25+
#include "foundation/compat_fs.h"
2526
#include "cbm.h"
2627
#include "simhash/minhash.h"
2728
#include "semantic/ast_profile.h"
@@ -33,7 +34,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };
3334
/* Read entire file into heap-allocated buffer. Returns NULL on error.
3435
* Caller must free(). Sets *out_len to byte count. */
3536
static char *read_file(const char *path, int *out_len) {
36-
FILE *f = fopen(path, "rb");
37+
FILE *f = cbm_fopen(path, "rb");
3738
if (!f) {
3839
return NULL;
3940
}

0 commit comments

Comments
 (0)