Skip to content

Commit a5351d3

Browse files
authored
Merge pull request DeusData#891 from Flipper1994/fix/win-utf8-argv
fix(win): deliver non-ASCII CLI paths end-to-end (wide argv + CreateProcessW)
2 parents fea48a5 + 14142b3 commit a5351d3

8 files changed

Lines changed: 143 additions & 365 deletions

File tree

‎.github/workflows/_test.yml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,9 @@ jobs:
127127
# over stdio / CLI / HTTP and fail if a Windows bug already fixed on main comes
128128
# back -- non-ASCII repo paths dropping definitions (#636/#357, fixed by #700),
129129
# the PreToolUse hook augmenter no-op on drive-letter cwd (#618, fixed by #619),
130-
# and the UI directory picker not enumerating drives (#548, roots field). The
131-
# still-open narrow-argv repro (#423/#20) is opt-in and excluded via -GuardsOnly.
130+
# the UI directory picker not enumerating drives (#548, roots field), and non-ASCII
131+
# CLI arguments being mangled by the narrow-argv main() (#423/#20, fixed by the
132+
# wide-argv entrypoint that reads GetCommandLineW).
132133
test-windows-guards:
133134
runs-on: windows-latest
134135
timeout-minutes: 60
@@ -155,9 +156,10 @@ jobs:
155156
# guard's HTTP UI is available. Functional gate only (no sanitizers).
156157
run: scripts/build.sh --with-ui CC=clang CXX=clang++
157158

158-
- name: Windows regression guards (#636/#357, #618, #548)
159+
- name: Windows regression guards (#636/#357, #618, #548, #423/#20)
159160
shell: pwsh
160-
# -GuardsOnly runs the three green guards and gates on them; the runner
161-
# runs indexing in-process (CBM_INDEX_SUPERVISOR=0) so a guard reflects the
162-
# path/hook/drive fix under test, not the orthogonal index-worker spawn.
161+
# -GuardsOnly runs the four green guards and gates on them. The runner sets
162+
# CBM_INDEX_SUPERVISOR=0 for determinism, but the non-ASCII CLI guard drops that
163+
# override so it exercises the real supervisor->worker spawn (where #423/#20's
164+
# second half lives); the other guards test path/hook/drive fixes in-process.
163165
run: ./scripts/test-windows.ps1 -GuardsOnly -Binary build/c/codebase-memory-mcp.exe

‎Makefile.cbm‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,14 @@ CFLAGS_TSAN = $(CFLAGS_COMMON) -g -O1 \
7272
CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -g -O1 \
7373
-fsanitize=thread -fno-omit-frame-pointer
7474

75-
# Windows needs ws2_32 (Winsock), psapi (GetProcessMemoryInfo), and
75+
# Windows needs ws2_32 (Winsock), psapi (GetProcessMemoryInfo), shell32
76+
# (CommandLineToArgvW — the UTF-8 argv path in main.c, #423/#20), and
7677
# --allow-multiple-definition (MinGW CRT symbol clashes).
7778
# Auto-detected via compiler; no manual override needed.
7879
IS_MINGW := $(shell echo | $(CC) -dM -E - 2>/dev/null | grep -q 'define _WIN32 ' && echo yes || echo no)
7980
WIN32_LIBS :=
8081
ifeq ($(IS_MINGW),yes)
81-
WIN32_LIBS := -lws2_32 -lpsapi -Wl,--allow-multiple-definition -Wl,--stack,8388608 -static
82+
WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -Wl,--allow-multiple-definition -Wl,--stack,8388608 -static
8283
endif
8384

8485
# STATIC=1 produces a fully static binary (for Alpine/musl portable builds)

‎scripts/test-windows.ps1‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,20 @@
1616
* test_non_ascii_path.py guards #636/#357 (fixed by #700)
1717
* test_hook_augment.py guards #618 (fixed by #619)
1818
* test_ui_drive_listing.py guards #548 (roots field)
19+
* test_cli_non_ascii_arg.py guards #423/#20 (wide-argv main())
1920
2021
KNOWN REDS - genuine, still-open Windows bugs reproduced at the product
2122
surface. They are EXPECTED to be RED (exit 1) and are opt-in
2223
(never gate CI). If one turns GREEN the underlying bug was
2324
fixed and it should be promoted to a guard.
24-
* test_cli_non_ascii_arg.py reproduces #423/#20 (narrow
25-
argv main() - no wide command line)
25+
* (none currently - test_cli_non_ascii_arg.py was promoted to a
26+
guard when the wide-argv fix for #423/#20 landed)
2627
27-
Determinism: indexing runs in-process (CBM_INDEX_SUPERVISOR=0). These tests
28-
exercise path / hook / drive handling, not the index-supervisor subprocess
29-
path; the pass-level readers (#700's cbm_fopen routing) run in-process either
30-
way, so the guard coverage is identical while results stay independent of the
31-
local toolchain's worker-spawn behavior.
28+
Determinism: the runner sets CBM_INDEX_SUPERVISOR=0 so the path / hook / drive
29+
guards index in-process (the pass-level readers under test, e.g. #700's cbm_fopen
30+
routing, run in-process either way). The non-ASCII CLI guard is the exception - it
31+
drops that override to cross the real supervisor -> worker spawn, where the second
32+
half of #423/#20 lives (CreateProcessW delivering the wide command line).
3233
3334
On native Windows the MinGW/LLVM toolchain ships no libasan/libubsan, so the
3435
build disables sanitizers (SANITIZE=). Where the toolchain provides
@@ -104,14 +105,14 @@ $env:CBM_INDEX_SUPERVISOR = "0" # in-process indexing (see .DESCRIPTION)
104105
$guards = @(
105106
"tests\windows\test_non_ascii_path.py",
106107
"tests\windows\test_hook_augment.py",
107-
"tests\windows\test_ui_drive_listing.py"
108-
)
109-
110-
# Opt-in known-red repros - EXPECTED red (exit 1); never gate CI.
111-
$knownReds = @(
108+
"tests\windows\test_ui_drive_listing.py",
112109
"tests\windows\test_cli_non_ascii_arg.py"
113110
)
114111

112+
# Opt-in known-red repros - EXPECTED red (exit 1); never gate CI. Currently empty:
113+
# test_cli_non_ascii_arg.py was promoted to a guard when #423/#20's wide-argv fix landed.
114+
$knownReds = @()
115+
115116
$guardFailures = @()
116117
$guardSkips = @()
117118
$fixedKeepers = @()
@@ -159,7 +160,7 @@ if ($fixedKeepers.Count -gt 0) {
159160
}
160161
if ($guardFailures.Count -gt 0) {
161162
Write-Host ("REGRESSION: {0} green guard(s) went red: {1}" -f $guardFailures.Count, ($guardFailures -join ", ")) -ForegroundColor Red
162-
Write-Host "A previously-fixed Windows bug is broken again. See tests/windows/RED_TEST_ANALYSIS.md." -ForegroundColor Red
163+
Write-Host "A previously-fixed Windows bug is broken again (see the guard's docstring and its referenced issue)." -ForegroundColor Red
163164
exit 1
164165
}
165166
Write-Host "All Windows green guards passed." -ForegroundColor Green

‎src/foundation/subprocess.c‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313

1414
#ifdef _WIN32
1515
#include <windows.h>
16+
#include "win_utf8.h" /* cbm_utf8_to_wide — spawn the worker with a wide command line so a
17+
* non-ASCII repo path survives CreateProcess (#423/#20) */
18+
#include <stdlib.h> /* free */
1619
#else
1720
#include <errno.h>
1821
#include <fcntl.h>
@@ -235,9 +238,20 @@ static int cbm_run_win(const cbm_proc_opts_t *opts, cbm_proc_result_t *out) {
235238
out->term_signal = 0;
236239
return -1;
237240
}
241+
/* Spawn via CreateProcessW with a WIDE command line. CreateProcessA would
242+
* re-interpret our UTF-8 cmdline bytes through the ANSI code page (CP_ACP),
243+
* re-mangling a non-ASCII repo path at the parent->worker boundary — so the
244+
* worker's own wide-argv read could never recover it (#423/#20). */
245+
wchar_t *wcmd = cbm_utf8_to_wide(cmdline);
246+
if (!wcmd) {
247+
out->outcome = CBM_PROC_SPAWN_FAILED;
248+
out->exit_code = -1;
249+
out->term_signal = 0;
250+
return -1;
251+
}
238252

239253
HANDLE hlog = INVALID_HANDLE_VALUE;
240-
STARTUPINFOA si = {.cb = sizeof(si)};
254+
STARTUPINFOW si = {.cb = sizeof(si)};
241255
if (opts->log_file) {
242256
hlog = CreateFileA(opts->log_file, GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS,
243257
FILE_ATTRIBUTE_NORMAL, NULL);
@@ -249,7 +263,8 @@ static int cbm_run_win(const cbm_proc_opts_t *opts, cbm_proc_result_t *out) {
249263
}
250264

251265
PROCESS_INFORMATION pi = {0};
252-
BOOL ok = CreateProcessA(NULL, cmdline, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi);
266+
BOOL ok = CreateProcessW(NULL, wcmd, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi);
267+
free(wcmd);
253268
if (hlog != INVALID_HANDLE_VALUE) {
254269
CloseHandle(hlog);
255270
}

‎src/main.c‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ enum {
4040
#include "foundation/compat_thread.h"
4141
#include "foundation/mem.h"
4242
#include "foundation/profile.h"
43+
#include "foundation/win_utf8.h" /* cbm_wide_to_utf8 — Windows UTF-8 argv (#423/#20); no-op on POSIX */
44+
#ifdef _WIN32
45+
#include <shellapi.h> /* CommandLineToArgvW — not pulled in by windows.h under WIN32_LEAN_AND_MEAN */
46+
#endif
4347
#include "ui/config.h"
4448
#include "ui/http_server.h"
4549
#include "ui/embedded_assets.h"
@@ -593,6 +597,48 @@ static void setup_signal_handlers(void) {
593597
#endif
594598
}
595599

600+
#ifdef _WIN32
601+
/* On Windows the CRT hands main() an argv encoded in the active ANSI code page, so a
602+
* non-ASCII CLI argument (e.g. a repo path like café_日本語_repo) is mangled before the
603+
* program ever sees it — the documented `cli index_repository "<json>"` then fails with
604+
* "repo_path is required" (#423/#20). Rebuild argv from the wide command line
605+
* (GetCommandLineW → CommandLineToArgvW) and convert each element to UTF-8 so the rest
606+
* of the program receives the same UTF-8 bytes it gets on POSIX. Returns a
607+
* NULL-terminated argv and sets *out_argc, or NULL on any failure (caller then keeps
608+
* the original narrow argv). The returned block lives for the whole process (argv must
609+
* stay valid until exit), so it is intentionally never freed. */
610+
static char **cbm_win_utf8_argv(int *out_argc) {
611+
int wargc = 0;
612+
LPWSTR *wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
613+
if (!wargv) {
614+
return NULL;
615+
}
616+
if (wargc <= 0) {
617+
LocalFree(wargv);
618+
return NULL;
619+
}
620+
char **u8argv = (char **)calloc((size_t)wargc + 1, sizeof(char *));
621+
if (!u8argv) {
622+
LocalFree(wargv);
623+
return NULL;
624+
}
625+
for (int i = 0; i < wargc; i++) {
626+
u8argv[i] = cbm_wide_to_utf8(wargv[i]);
627+
if (!u8argv[i]) {
628+
for (int j = 0; j < i; j++) {
629+
free(u8argv[j]);
630+
}
631+
free(u8argv);
632+
LocalFree(wargv);
633+
return NULL;
634+
}
635+
}
636+
LocalFree(wargv);
637+
*out_argc = wargc;
638+
return u8argv; /* NULL-terminated (calloc'd wargc+1) */
639+
}
640+
#endif /* _WIN32 */
641+
596642
int main(int argc, char **argv) {
597643
/* Defense-in-depth: bind tree-sitter and sqlite3 to mimalloc so a
598644
* correct binary does not rely on the fragile MI_OVERRIDE symbol override
@@ -601,6 +647,20 @@ int main(int argc, char **argv) {
601647
* below opens sqlite early), else sqlite3_config returns SQLITE_MISUSE and
602648
* the bind is silently ignored. No-op in the test build. */
603649
cbm_alloc_init();
650+
#ifdef _WIN32
651+
/* Replace the ANSI-code-page argv the CRT handed us with a UTF-8 argv rebuilt from
652+
* the wide command line, so non-ASCII CLI arguments survive (#423/#20). Falls back
653+
* to the original argv if the wide rebuild fails. Done after cbm_alloc_init (which
654+
* must stay the very first statement) but before argv is first read below. */
655+
{
656+
int win_argc = 0;
657+
char **win_argv = cbm_win_utf8_argv(&win_argc);
658+
if (win_argv) {
659+
argc = win_argc;
660+
argv = win_argv;
661+
}
662+
}
663+
#endif
604664
/* #845: mark this process as the REAL binary so the index supervisor may
605665
* wrap index_repository in a worker subprocess. Must run before any
606666
* subcommand dispatch so MCP-server, CLI, and HTTP paths are all covered.

‎src/ui/http_server.c‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "foundation/str_util.h"
3535
#include "foundation/compat_thread.h"
3636
#include "foundation/subprocess.h" /* cbm_build_win_cmdline — shared MS-CRT arg quoting */
37+
#include "foundation/win_utf8.h" /* cbm_utf8_to_wide — CreateProcessW wide cmdline (#423/#20) */
3738

3839
#include <sqlite3/sqlite3.h>
3940
#include <yyjson/yyjson.h>
@@ -972,19 +973,30 @@ static void *index_thread_fn(void *arg) {
972973
atomic_store(&job->status, 3);
973974
return NULL;
974975
}
976+
/* Wide command line: CreateProcessA would re-mangle the UTF-8 repo path through the
977+
* ANSI code page at the spawn boundary, so a non-ASCII repo path never reaches the
978+
* worker intact (#423/#20). Convert and spawn via CreateProcessW. */
979+
wchar_t *wcmd = cbm_utf8_to_wide(cmdline);
980+
if (!wcmd) {
981+
snprintf(job->error_msg, sizeof(job->error_msg), "index command line conversion failed");
982+
atomic_store(&job->status, 3);
983+
return NULL;
984+
}
975985

976986
cbm_log_info("ui.index.spawn", "bin", bin, "log", log_file);
977987

978988
HANDLE hlog = CreateFileA(log_file, GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS,
979989
FILE_ATTRIBUTE_NORMAL, NULL);
980-
STARTUPINFOA si_proc = {.cb = sizeof(si_proc)};
990+
STARTUPINFOW si_proc = {.cb = sizeof(si_proc)};
981991
if (hlog != INVALID_HANDLE_VALUE) {
982992
si_proc.dwFlags = STARTF_USESTDHANDLES;
983993
si_proc.hStdError = hlog;
984994
si_proc.hStdOutput = hlog;
985995
}
986996
PROCESS_INFORMATION pi = {0};
987-
if (!CreateProcessA(NULL, cmdline, NULL, NULL, TRUE, 0, NULL, NULL, &si_proc, &pi)) {
997+
BOOL spawned = CreateProcessW(NULL, wcmd, NULL, NULL, TRUE, 0, NULL, NULL, &si_proc, &pi);
998+
free(wcmd);
999+
if (!spawned) {
9881000
snprintf(job->error_msg, sizeof(job->error_msg), "CreateProcess failed");
9891001
atomic_store(&job->status, 3);
9901002
if (hlog != INVALID_HANDLE_VALUE)

0 commit comments

Comments
 (0)