Skip to content

Commit d9adfeb

Browse files
committed
fix(writer): open the graph DB wide-safe for non-ASCII cache paths
cbm_writer_open used a raw ANSI-CP fopen for the hand-rolled SQLite writer - the one file-creating call on the dump chain without a UTF-8-to-wide conversion. On a non-ASCII Windows profile (C:\Users\Kovacs Janos) extract and resolve succeeded but the dump failed with pipeline.err phase=dump: fopen could not create graph.db under the profile-derived cache dir while cbm_mkdir_p (already wide) had happily created the directory around it. Routed through cbm_fopen, matching DeusData#700 (pass readers) and DeusData#973 (canonicalization); POSIX is byte-transparent. New Windows guard test_non_ascii_cache_dump.py indexes an ASCII repo into a non-ASCII CBM_CACHE_DIR and requires a query_graph readback with Function count > 0 - proving the DB was written to and reopened from the non-ASCII cache, not merely that no error surfaced. The existing non-ASCII guard only varied the repo path against an ASCII cache, which is why this class went unseen. The USERPROFILE-sourced env encoding question (Layer 2 - _environ is ANSI-CP, not UTF-8) is deliberately not bundled; it needs verification on a real Windows box and will be filed separately if the guard alone does not settle it. Closes DeusData#996 Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent db330bf commit d9adfeb

3 files changed

Lines changed: 139 additions & 1 deletion

File tree

‎internal/cbm/sqlite_writer.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2211,7 +2211,11 @@ cbm_db_writer_t *cbm_writer_open(const char *path) {
22112211
* -wal/-shm or a crashed session's WAL gets replayed on top of the
22122212
* new file at the next open (#897). */
22132213
cbm_remove_db_sidecars(path);
2214-
FILE *fp = fopen(path, "wb");
2214+
/* cbm_fopen, not raw fopen: the cache dir lives under the user profile,
2215+
* and an ANSI-CP fopen fails to create the DB on non-ASCII Windows
2216+
* profiles — the reported phase=dump failure (#996). Everything around
2217+
* this call (cbm_mkdir_p, sqlite3_open_v2 reopen) is already wide-safe. */
2218+
FILE *fp = cbm_fopen(path, "wb");
22152219
if (!fp) {
22162220
return NULL;
22172221
}

‎scripts/test-windows.ps1‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
They must stay GREEN (exit 0); a RED (exit 1) means the fix
1515
regressed and fails this runner.
1616
* test_non_ascii_path.py guards #636/#357 (fixed by #700)
17+
* test_non_ascii_cache_dump.py guards #996 (writer cbm_fopen)
1718
* test_hook_augment.py guards #618 (fixed by #619)
1819
* test_ui_drive_listing.py guards #548 (roots field)
1920
* test_cli_non_ascii_arg.py guards #423/#20 (wide-argv main())
@@ -104,6 +105,7 @@ $env:CBM_INDEX_SUPERVISOR = "0" # in-process indexing (see .DESCRIPTION)
104105
# (exit 2) and is skipped rather than failed.
105106
$guards = @(
106107
"tests\windows\test_non_ascii_path.py",
108+
"tests\windows\test_non_ascii_cache_dump.py",
107109
"tests\windows\test_hook_augment.py",
108110
"tests\windows\test_ui_drive_listing.py",
109111
"tests\windows\test_cli_non_ascii_arg.py"
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
"""Regression guard for issue #996 — dump phase must write the graph DB into a
2+
NON-ASCII cache directory on Windows.
3+
4+
The existing test_non_ascii_path.py exercises non-ASCII REPO paths against an
5+
ASCII cache, so it never covers the dump->cache write. #996's reporter (a
6+
non-ASCII %USERPROFILE%, e.g. C:\\Users\\Kovács János) saw extract/resolve
7+
succeed and `pipeline.err phase=dump`: cbm_writer_open used a raw ANSI-CP
8+
fopen for the hand-rolled SQLite writer (internal/cbm/sqlite_writer.c), the
9+
one file-creating call on the dump chain without UTF-8→wide conversion.
10+
Fixed by routing it through cbm_fopen (same pattern as #700/#973).
11+
12+
This guard indexes an ASCII repo into a NON-ASCII cache dir (CBM_CACHE_DIR is
13+
read before any USERPROFILE derivation, so it isolates the writer cleanly).
14+
GREEN is non-vacuous: the index must succeed AND a query_graph readback must
15+
count Function nodes > 0 — proving the DB was written to and reopened from
16+
the non-ASCII cache, not merely that no error surfaced.
17+
18+
Passes on Linux/macOS either way (byte-transparent UTF-8 filesystems).
19+
20+
Exit code: 0 == invariant holds, 1 == regression, 2 == setup error.
21+
22+
Usage:
23+
python test_non_ascii_cache_dump.py <path-to-codebase-memory-mcp[.exe]>
24+
"""
25+
import json
26+
import os
27+
import shutil
28+
import sys
29+
import tempfile
30+
31+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
32+
from mcp_stdio import McpServer # noqa: E402
33+
34+
MATH_TS = (
35+
"export function add(a: number, b: number): number { return a + b; }\n"
36+
"export function mul(a: number, b: number): number { return add(a, a); }\n"
37+
"export class Calc {\n"
38+
" total: number = 0;\n"
39+
" push(x: number): void { this.total = add(this.total, x); }\n"
40+
"}\n"
41+
)
42+
43+
# Mixed scripts in ONE segment — one shot covers the classes the sibling
44+
# test exercises separately (the writer either converts wide or it doesn't).
45+
NON_ASCII_CACHE_SEGMENT = "cache_café_Ωμέγα_日本語"
46+
47+
48+
def graph_function_count(server, project):
49+
resp = server.call_tool(
50+
"query_graph",
51+
{
52+
"project": project,
53+
"format": "json",
54+
"query": "MATCH (n:Function) RETURN count(n) AS c",
55+
},
56+
)
57+
text, err = McpServer.tool_text(resp)
58+
if err or not text:
59+
return 0
60+
data = json.loads(text)
61+
rows = data.get("rows") or []
62+
if not rows or not rows[0]:
63+
return 0
64+
return int(rows[0][0])
65+
66+
67+
def main():
68+
if len(sys.argv) != 2:
69+
print("usage: test_non_ascii_cache_dump.py <binary>")
70+
return 2
71+
binary = os.path.abspath(sys.argv[1])
72+
if not os.path.exists(binary):
73+
print(f"SETUP: binary not found: {binary}")
74+
return 2
75+
76+
work = tempfile.mkdtemp(prefix="cbm_996_")
77+
try:
78+
repo = os.path.join(work, "ascii_repo")
79+
os.makedirs(repo)
80+
with open(os.path.join(repo, "math.ts"), "w", encoding="utf-8") as f:
81+
f.write(MATH_TS)
82+
83+
cache = os.path.join(work, NON_ASCII_CACHE_SEGMENT)
84+
os.makedirs(cache)
85+
86+
with McpServer(binary, cache_dir=cache) as s:
87+
resp = s.call_tool("index_repository", {"repo_path": repo})
88+
text, err = McpServer.tool_text(resp)
89+
if err:
90+
print(f"FAIL: index_repository rpc error: {err}")
91+
return 1
92+
text = text or ""
93+
if '"error"' in text:
94+
print(f"FAIL: index_repository into non-ASCII cache errored: {text[:300]}")
95+
return 1
96+
if "phase" in text and "dump" in text and "error" in text.lower():
97+
print(f"FAIL: dump phase error: {text[:300]}")
98+
return 1
99+
100+
# Non-vacuous readback: the DB must exist under the non-ASCII
101+
# cache and answer queries.
102+
project = None
103+
try:
104+
project = json.loads(text).get("project")
105+
except (ValueError, AttributeError):
106+
pass
107+
if not project:
108+
# TOON-shaped success output: fall back to list_projects.
109+
lp = s.call_tool("list_projects", {})
110+
lp_text, _ = McpServer.tool_text(lp)
111+
lp_text = lp_text or ""
112+
for line in lp_text.splitlines():
113+
if "ascii_repo" in line:
114+
project = line.split(",")[0].strip().strip('"')
115+
break
116+
if not project:
117+
print("FAIL: could not determine project name after index")
118+
return 1
119+
120+
count = graph_function_count(s, project)
121+
if count < 1:
122+
print(f"FAIL: readback from non-ASCII cache found {count} Function nodes")
123+
return 1
124+
125+
print(f"OK: dump wrote and reopened graph DB under non-ASCII cache ({count} functions)")
126+
return 0
127+
finally:
128+
shutil.rmtree(work, ignore_errors=True)
129+
130+
131+
if __name__ == "__main__":
132+
sys.exit(main())

0 commit comments

Comments
 (0)