Skip to content

Commit 59d6b52

Browse files
authored
Merge pull request DeusData#2421 from Fieldnote-Echo/fix/dotd-depinfo-sniff
fix(discover): keep make/cargo dep-info .d files out of the D grammar
2 parents ba4122d + eb76d04 commit 59d6b52

5 files changed

Lines changed: 310 additions & 0 deletions

File tree

‎src/discover/discover.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ CBMLanguage cbm_disambiguate_frm(const char *path);
5858
* CBM_LANG_RESCRIPT. On read failure, defaults to CBM_LANG_RESCRIPT. */
5959
CBMLanguage cbm_disambiguate_res(const char *path);
6060

61+
/* Disambiguate .d files by reading first 4KB of content.
62+
* Returns CBM_LANG_COUNT (not source) for a make-style dependency file written
63+
* by rustc/cargo, gcc/clang -MD or CMake ("target ...: prereq ..."), otherwise
64+
* CBM_LANG_DLANG. On read failure or any doubt, defaults to CBM_LANG_DLANG. */
65+
CBMLanguage cbm_disambiguate_d(const char *path);
66+
6167
/* Disambiguate .inc files by reading first 4KB of content.
6268
* Returns CBM_LANG_OBJECTSCRIPT_ROUTINE if it looks like an ObjectScript
6369
* include (a "ROUTINE <Uppercase>" header), otherwise CBM_LANG_BITBAKE.

‎src/discover/language.c‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,6 +1436,107 @@ CBMLanguage cbm_disambiguate_res(const char *path) {
14361436
return lang_res_bytes(head, n);
14371437
}
14381438

1439+
/* A make rule's targets end at a ':' followed by whitespace or the end of the
1440+
* file, which skips drive-letter colons ("C:/", "C:\"). */
1441+
static bool is_rule_colon(const char *p, bool whole_file) {
1442+
char next = p[SKIP_ONE];
1443+
return *p == ':' && (next == ' ' || next == '\t' || next == '\r' || next == '\n' ||
1444+
(next == '\0' && whole_file));
1445+
}
1446+
1447+
/* Bytes of D whitespace at p, or 0: space, tab, vertical tab, form feed, CR
1448+
* and the UTF-8 line and paragraph separators U+2028/U+2029, all of which split
1449+
* D tokens. '\n' ends the line and is the caller's. */
1450+
static size_t d_space_len(const char *p) {
1451+
if (*p == ' ' || *p == '\t' || *p == '\v' || *p == '\f' || *p == '\r') {
1452+
return SKIP_ONE;
1453+
}
1454+
if (strncmp(p, "\xE2\x80\xA8", SLEN("\xE2\x80\xA8")) == 0 ||
1455+
strncmp(p, "\xE2\x80\xA9", SLEN("\xE2\x80\xA9")) == 0) {
1456+
return SLEN("\xE2\x80\xA8");
1457+
}
1458+
return 0;
1459+
}
1460+
1461+
/* True if the line at p is a make rule "target...: prereq..." whose targets all
1462+
* look like paths (contain '/', '\' or '.'); a backslash-escaped space stays
1463+
* inside its target. D source fails this: "public:", "@safe:", "extern(C):"
1464+
* and "import a.b : c" all have a non-path word or D punctuation before the
1465+
* colon. An attribute ('@', as in "@1.0:") or a comment between tokens
1466+
* ("public/+ +/:") is D even with a '.' or '/' before the colon, so a target
1467+
* holding either is never dep-info. */
1468+
static bool is_dep_rule_line(const char *p, bool whole_file) {
1469+
bool in_target = false;
1470+
bool path_like = false;
1471+
bool any_target = false;
1472+
for (; *p && *p != '\n'; p++) {
1473+
if (is_rule_colon(p, whole_file)) {
1474+
return in_target ? path_like : any_target;
1475+
}
1476+
size_t space = d_space_len(p);
1477+
if (space) {
1478+
if (in_target && !path_like) {
1479+
return false;
1480+
}
1481+
in_target = false;
1482+
path_like = false;
1483+
p += space - SKIP_ONE;
1484+
continue;
1485+
}
1486+
if (strchr("(){};=\"',@", *p)) {
1487+
return false;
1488+
}
1489+
if (*p == '/' && (p[SKIP_ONE] == '*' || p[SKIP_ONE] == '+' || p[SKIP_ONE] == '/')) {
1490+
return false;
1491+
}
1492+
in_target = true;
1493+
any_target = true;
1494+
path_like = path_like || *p == '/' || *p == '\\' || *p == '.';
1495+
if (*p == '\\' && p[SKIP_ONE] == ' ') {
1496+
p++; /* escaped space: part of this target */
1497+
}
1498+
}
1499+
return false;
1500+
}
1501+
1502+
/* Disambiguate .d files: shared by D source and make-style dependency files
1503+
* written by rustc/cargo (target/<profile>/deps/<crate>-<hash>.d), gcc/clang -MD
1504+
* and CMake (<object>.o.d). Those are "target: prereq ..." rules, often tens of
1505+
* KB on one line, which the D grammar parses slowly into nothing but a module
1506+
* node, so they are reported as unsupported (CBM_LANG_COUNT). Defaults to D on
1507+
* any doubt (preserves existing behaviour). `n` is the count of head bytes buf
1508+
* was made from. */
1509+
static CBMLanguage lang_d_text(const char *buf, size_t n) {
1510+
/* A colon at the end of buf only ends the rule if nothing was cut off. */
1511+
bool whole_file = n < LANG_PROBE_HEAD && strlen(buf) == n;
1512+
1513+
const char *p = buf;
1514+
if (strncmp(p, "\xEF\xBB\xBF", SLEN("\xEF\xBB\xBF")) == 0) {
1515+
p += SLEN("\xEF\xBB\xBF");
1516+
}
1517+
while (*p && isspace((unsigned char)*p)) {
1518+
p++;
1519+
}
1520+
/* A shebang or a leading D comment (including the DUB single-file
1521+
* "/+ dub.sdl:" recipe) is D; dep-info never starts that way. */
1522+
if (*p == '#' ||
1523+
(p[0] == '/' && (p[SKIP_ONE] == '/' || p[SKIP_ONE] == '*' || p[SKIP_ONE] == '+'))) {
1524+
return CBM_LANG_DLANG;
1525+
}
1526+
return is_dep_rule_line(p, whole_file) ? CBM_LANG_COUNT : CBM_LANG_DLANG;
1527+
}
1528+
1529+
CBMLanguage cbm_disambiguate_d(const char *path) {
1530+
unsigned char head[LANG_PROBE_HEAD];
1531+
size_t n = 0;
1532+
if (!lang_read_head(path, head, sizeof(head), &n)) {
1533+
return CBM_LANG_DLANG;
1534+
}
1535+
char buf[LANG_PROBE_HEAD + SKIP_ONE];
1536+
lang_head_text(buf, LANG_PROBE_HEAD, head, n);
1537+
return lang_d_text(buf, n);
1538+
}
1539+
14391540
/* Disambiguate .cls files: shared by InterSystems ObjectScript UDL, Salesforce
14401541
* Apex and Visual Basic 6 class modules (#721). ObjectScript class files begin
14411542
* with a line of the form "Class <UppercasePackage>..."; VB6 class modules
@@ -1611,6 +1712,13 @@ static const char *lang_probe_extension(const char *filename) {
16111712
return (dot && lang_name_in(dot, PROBED)) ? dot : NULL;
16121713
}
16131714

1715+
/* .d is shared by D and make-style dep-info (cargo, gcc -MD). Probe only while
1716+
* .d maps to D, so a user override to another language wins. */
1717+
static bool lang_probe_d(CBMLanguage lang, const char *filename) {
1718+
const char *dot = strrchr(filename, '.');
1719+
return lang == CBM_LANG_DLANG && dot && strcmp(dot, ".d") == 0;
1720+
}
1721+
16141722
/* "<Export generator=" in the first line-probe bytes: an ObjectScript Studio
16151723
* export, which is XML by name. */
16161724
static bool lang_objectscript_export(const unsigned char *head, size_t n) {
@@ -1633,6 +1741,9 @@ size_t cbm_language_probe_bytes_with(const cbm_userconfig_t *config, const char
16331741
if (ext) {
16341742
return strcmp(ext, ".cfc") == 0 ? LANG_PROBE_CFC : LANG_PROBE_HEAD;
16351743
}
1744+
if (lang_probe_d(lang, filename)) {
1745+
return LANG_PROBE_HEAD;
1746+
}
16361747
return lang == CBM_LANG_XML ? LANG_PROBE_LINE : 0;
16371748
}
16381749

@@ -1676,6 +1787,11 @@ CBMLanguage cbm_language_classify_with(const cbm_userconfig_t *config, const cha
16761787
/* .res: ReScript, or a binary Godot / Windows resource (#2176) */
16771788
lang = readable ? lang_res_bytes(head, head_len) : CBM_LANG_RESCRIPT;
16781789
}
1790+
} else if (lang_probe_d(lang, filename)) {
1791+
/* D, or a make-style dep-info file */
1792+
char buf[LANG_PROBE_HEAD + SKIP_ONE];
1793+
lang_head_text(buf, LANG_PROBE_HEAD, head, head_len);
1794+
lang = readable ? lang_d_text(buf, head_len) : CBM_LANG_DLANG;
16791795
}
16801796
if (lang == CBM_LANG_XML && readable && lang_objectscript_export(head, head_len)) {
16811797
return CBM_LANG_OBJECTSCRIPT_EXPORT;

‎tests/test_discover.c‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "test_framework.h"
77
#include "test_helpers.h"
88
#include "discover/discover.h"
9+
#include "discover/userconfig.h"
910
#include "discover/discover_policy_internal.h"
1011
#include "foundation/platform.h"
1112

@@ -2444,6 +2445,49 @@ TEST(shebang_oversized_first_line_unindexed) {
24442445
PASS();
24452446
}
24462447

2448+
/* ── .d: D source vs make/cargo dep-info ───────────────────────── */
2449+
2450+
/* rustc/cargo dep-info (and gcc -MD, CMake .o.d) files share .d with D source
2451+
* but are Makefile rules the D grammar parses slowly into nothing useful, so
2452+
* they must not be indexed as D. */
2453+
static const char CARGO_DEP_INFO[] = "/home/u/proj/t/debug/deps/foo-abc123.d: src/lib.rs src/a.rs\n"
2454+
"\n"
2455+
"src/lib.rs:\n"
2456+
"src/a.rs:\n";
2457+
2458+
TEST(discover_d_dep_info_not_indexed_as_dlang) {
2459+
CBMLanguage dep;
2460+
CBMLanguage src;
2461+
ASSERT(shebang_probe("cbm_disc_dotd", "t/debug/deps/foo-abc123.d", CARGO_DEP_INFO, &dep));
2462+
ASSERT(shebang_probe("cbm_disc_dotd", "app.d", "module app;\nimport std.stdio : writeln;\n",
2463+
&src));
2464+
ASSERT_EQ(dep, CBM_LANG_COUNT);
2465+
ASSERT_EQ(src, CBM_LANG_DLANG);
2466+
PASS();
2467+
}
2468+
2469+
/* The probe runs only while .d maps to D, so a user override to another
2470+
* language wins. */
2471+
TEST(discover_d_user_override_skips_dep_info_probe) {
2472+
char *cfg_dir = th_mktempdir("cbm_disc_dotd_uc");
2473+
ASSERT(cfg_dir != NULL);
2474+
ASSERT_EQ(th_write_file(TH_PATH(cfg_dir, ".codebase-memory.json"),
2475+
"{\"extra_extensions\":{\".d\":\"makefile\"}}"),
2476+
0);
2477+
cbm_userconfig_t *cfg = cbm_userconfig_load(cfg_dir);
2478+
th_cleanup(cfg_dir);
2479+
ASSERT_NOT_NULL(cfg);
2480+
2481+
cbm_set_user_lang_config(cfg);
2482+
CBMLanguage lang;
2483+
bool ok = shebang_probe("cbm_disc_dotd_uc", "foo-abc123.d", CARGO_DEP_INFO, &lang);
2484+
cbm_set_user_lang_config(NULL);
2485+
cbm_userconfig_free(cfg);
2486+
ASSERT(ok);
2487+
ASSERT_EQ(lang, CBM_LANG_MAKEFILE);
2488+
PASS();
2489+
}
2490+
24472491
/* ── Suite ─────────────────────────────────────────────────────── */
24482492

24492493
SUITE(discover) {
@@ -2539,6 +2583,10 @@ SUITE(discover) {
25392583
RUN_TEST(shebang_embedded_nul_unindexed);
25402584
RUN_TEST(shebang_oversized_first_line_unindexed);
25412585

2586+
/* .d: D source vs make/cargo dep-info */
2587+
RUN_TEST(discover_d_dep_info_not_indexed_as_dlang);
2588+
RUN_TEST(discover_d_user_override_skips_dep_info_probe);
2589+
25422590
/* Integration tests (cross-platform) */
25432591
RUN_TEST(discover_simple);
25442592
RUN_TEST(discover_wide_sibling_fanout_exceeds_initial_walk_stack);

‎tests/test_language.c‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -885,6 +885,73 @@ TEST(lang_cfc_default_on_read_fail) {
885885
PASS();
886886
}
887887

888+
/* ── .d: D source vs make/cargo dep-info ───────────────────────── */
889+
890+
/* Write content to a temp .d file and return cbm_disambiguate_d() for it. A
891+
* setup failure returns -1 so neither a D nor a COUNT expectation can pass
892+
* vacuously. */
893+
static int disambiguate_d_content(const char *content) {
894+
char path[256];
895+
snprintf(path, sizeof(path), "%s/test_lang_dotd.d", cbm_tmpdir());
896+
if (!write_probe_file(path, content)) {
897+
return -1;
898+
}
899+
CBMLanguage lang = cbm_disambiguate_d(path);
900+
remove(path);
901+
return (int)lang;
902+
}
903+
904+
TEST(lang_d_dep_info_unsupported) {
905+
/* cargo (absolute target), gcc -MD continuation, CMake object rule, Windows
906+
* drive letter (not the rule colon), escaped space (one target, not "app"),
907+
* and the GNU make manual's "foo.o foo.d : ..." (two targets, spaced colon). */
908+
ASSERT_EQ(disambiguate_d_content("/home/u/t/debug/deps/foo-abc123.d: src/lib.rs src/a.rs\n"
909+
"\nsrc/lib.rs:\n"),
910+
CBM_LANG_COUNT);
911+
ASSERT_EQ(disambiguate_d_content("foo.o: foo.c /usr/include/stdio.h \\\n"
912+
" /usr/include/features.h\n"),
913+
CBM_LANG_COUNT);
914+
ASSERT_EQ(disambiguate_d_content("CMakeFiles/x.dir/a.cpp.o: \\\n /src/a.cpp\n"),
915+
CBM_LANG_COUNT);
916+
ASSERT_EQ(disambiguate_d_content("C:/b/foo.o: C:/s/foo.c \\\r\n C:/s/foo.h\r\n"),
917+
CBM_LANG_COUNT);
918+
ASSERT_EQ(disambiguate_d_content("out/my\\ app: main.c\n"), CBM_LANG_COUNT);
919+
ASSERT_EQ(disambiguate_d_content("foo.o foo.d : foo.c defs.h\n"), CBM_LANG_COUNT);
920+
PASS();
921+
}
922+
923+
TEST(lang_d_source_stays_dlang) {
924+
ASSERT_EQ(disambiguate_d_content("module a;\n"), CBM_LANG_DLANG);
925+
ASSERT_EQ(disambiguate_d_content("import std.stdio : writeln;\n"), CBM_LANG_DLANG);
926+
ASSERT_EQ(disambiguate_d_content("@safe:\nvoid f() {}\n"), CBM_LANG_DLANG);
927+
/* A leading comment is D, even DUB's "/+ dub.sdl:" single-file recipe. */
928+
ASSERT_EQ(disambiguate_d_content("/+ dub.sdl:\n name \"hello\"\n+/\nvoid main() {}\n"),
929+
CBM_LANG_DLANG);
930+
ASSERT_EQ(disambiguate_d_content("enum E : ubyte {\n a,\n}\n"), CBM_LANG_DLANG);
931+
/* A float UDA ("@ FloatLiteral") has a '.' before its colon, alone or glued
932+
* to a keyword. */
933+
ASSERT_EQ(disambiguate_d_content("@1.0:\nvoid f() {}\n"), CBM_LANG_DLANG);
934+
ASSERT_EQ(disambiguate_d_content("nothrow@1.0:\nvoid f() {}\n"), CBM_LANG_DLANG);
935+
/* A comment between tokens puts a '/' before the colon. */
936+
ASSERT_EQ(disambiguate_d_content("public/**/:\nvoid f() {}\n"), CBM_LANG_DLANG);
937+
ASSERT_EQ(disambiguate_d_content("public/+ +/:\nvoid f() {}\n"), CBM_LANG_DLANG);
938+
ASSERT_EQ(disambiguate_d_content("public//note: x\nvoid f() {}\n"), CBM_LANG_DLANG);
939+
/* Every D whitespace splits tokens, not only space, tab and CR: form feed,
940+
* vertical tab, U+2028 and U+2029. */
941+
ASSERT_EQ(disambiguate_d_content("import\fstd.stdio : writeln;\n"), CBM_LANG_DLANG);
942+
ASSERT_EQ(disambiguate_d_content("import\vstd.stdio : writeln;\n"), CBM_LANG_DLANG);
943+
ASSERT_EQ(disambiguate_d_content("import\xE2\x80\xA8"
944+
"std.stdio : writeln;\n"),
945+
CBM_LANG_DLANG);
946+
ASSERT_EQ(disambiguate_d_content("import\xE2\x80\xA9"
947+
"std.stdio : writeln;\n"),
948+
CBM_LANG_DLANG);
949+
/* Default on doubt: empty or unreadable. */
950+
ASSERT_EQ(disambiguate_d_content(""), CBM_LANG_DLANG);
951+
ASSERT_EQ(cbm_disambiguate_d("/tmp/nonexistent_file_12345.d"), CBM_LANG_DLANG);
952+
PASS();
953+
}
954+
888955
/* --- New languages (auto-generated) --- */
889956
TEST(lang_ext_solidity) {
890957
ASSERT_EQ(cbm_language_for_extension(".sol"), CBM_LANG_SOLIDITY);
@@ -1398,6 +1465,7 @@ TEST(lang_probe_bytes_follow_the_name) {
13981465
ASSERT_EQ(cbm_language_probe_bytes("foo.inc"), 4096);
13991466
ASSERT_EQ(cbm_language_probe_bytes("Form1.frm"), 4096);
14001467
ASSERT_EQ(cbm_language_probe_bytes("App.res"), 4096);
1468+
ASSERT_EQ(cbm_language_probe_bytes("app.d"), 4096);
14011469
ASSERT_EQ(cbm_language_probe_bytes("Widget.cfc"), 16384);
14021470
ASSERT_EQ(cbm_language_probe_bytes("pom.xml"), 255);
14031471
/* An unknown name falls back to a shebang probe of the first line. */
@@ -1439,6 +1507,9 @@ TEST(lang_classify_matches_every_content_rule) {
14391507
{"p.frm", "#procedure foo\nLocal F = a;\n", CBM_LANG_FORM, cbm_disambiguate_frm},
14401508
{"q.res", "RSRC\0\0\0\1binary", CBM_LANG_COUNT, cbm_disambiguate_res},
14411509
{"r.res", "let x = 1\n", CBM_LANG_RESCRIPT, cbm_disambiguate_res},
1510+
{"dep.d", "target/debug/deps/app-0a1b2c.d: src/main.rs src/lib.rs\n", CBM_LANG_COUNT,
1511+
cbm_disambiguate_d},
1512+
{"app.d", "module app;\nimport std.stdio : writeln;\n", CBM_LANG_DLANG, cbm_disambiguate_d},
14421513
{"s.xml", "<?xml version=\"1.0\"?>\n<Export generator=\"IRIS\" version=\"26\">\n",
14431514
CBM_LANG_OBJECTSCRIPT_EXPORT, NULL},
14441515
{"t.xml", "<?xml version=\"1.0\"?>\n<project/>\n", CBM_LANG_XML, NULL},
@@ -1486,6 +1557,7 @@ TEST(lang_classify_unreadable_content_keeps_the_name_default) {
14861557
ASSERT_EQ(cbm_language_classify("a.cfc", NULL, 0, false, false), CBM_LANG_CFSCRIPT);
14871558
ASSERT_EQ(cbm_language_classify("a.frm", NULL, 0, false, false), CBM_LANG_FORM);
14881559
ASSERT_EQ(cbm_language_classify("a.res", NULL, 0, false, false), CBM_LANG_RESCRIPT);
1560+
ASSERT_EQ(cbm_language_classify("a.d", NULL, 0, false, false), CBM_LANG_DLANG);
14891561
ASSERT_EQ(cbm_language_classify("a.xml", NULL, 0, false, false), CBM_LANG_XML);
14901562
ASSERT_EQ(cbm_language_classify("run-tests", NULL, 0, false, false), CBM_LANG_COUNT);
14911563
ASSERT_EQ(cbm_language_classify("main.go", NULL, 0, false, false), CBM_LANG_GO);
@@ -1716,6 +1788,8 @@ SUITE(language) {
17161788
RUN_TEST(lang_frm_form_stays_form);
17171789
RUN_TEST(lang_res_binary_resource_unsupported);
17181790
RUN_TEST(lang_res_rescript_stays_rescript);
1791+
RUN_TEST(lang_d_dep_info_unsupported);
1792+
RUN_TEST(lang_d_source_stays_dlang);
17191793

17201794
/* Go test ports */
17211795
/* New languages */

0 commit comments

Comments
 (0)