From 69a544df777fa4ce17938ab001fa212d8d5f3633 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 11:32:18 +0900 Subject: [PATCH 01/30] Design record: the rules behind mcpp#931 and #952 to #960, their five causes, seven decisions and the round's scope (for review) --- ...ecosystem-rule-convergence-design-repro.sh | 246 ++++++ ...11-19-ecosystem-rule-convergence-design.md | 753 ++++++++++++++++++ .agents/docs/README.md | 4 +- 3 files changed, 1002 insertions(+), 1 deletion(-) create mode 100644 .agents/docs/2026-11-19-ecosystem-rule-convergence-design-repro.sh create mode 100644 .agents/docs/2026-11-19-ecosystem-rule-convergence-design.md diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design-repro.sh b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design-repro.sh new file mode 100644 index 000000000..a843283de --- /dev/null +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design-repro.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# Reproductions for 2026-11-19-ecosystem-rule-convergence-design.md. +# +# MCPP= bash [case...] +# +# Cases: 960 956 952 952s 931 931d 954 958 955 (all when none is named). Each prints what it +# observed and a line `RESULT : `. +# Nothing is asserted about a fixed engine: the script states the behaviour of +# the engine it is given. Measured with mcpp 2026.11.18.1 on Linux x86_64. +# +# 956 needs INDEX_830=, the +# commit that lists openkal-macos 0.23.5; skipped without it. +# 952 needs `unshare -Urpf` (a new pid namespace); skipped without it. +set -u + +MCPP="${MCPP:?set MCPP to the absolute path of an mcpp binary}" +JOBS="${MCPP_JOBS:-8}" +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/mcpp-issues-XXXXXX")" +echo "work directory: $ROOT ($("$MCPP" --version))" + +run() { MCPP_JOBS="$JOBS" "$MCPP" "$@"; } + +git_init() { # a publish archive is built from git objects + git -C "$1" init -q && git -C "$1" add -A && + git -C "$1" -c user.name=probe -c user.email=probe@example.invalid commit -qm init +} + +# 960: the descriptor `mcpp publish` writes, read by `xpkg parse` and by a +# consumer through a local index; then the same archive as the string form. +case_960() { + local d="$ROOT/960"; mkdir -p "$d/lib/src" "$d/index/pkgs/p" "$d/app/src" + printf '[package]\nnamespace = "probe"\nname = "sdk"\nversion = "0.1.0"\ndescription = "probe"\nlicense = "MIT"\nrepo = "https://github.com/example/probe"\n\n[targets.sdk]\nkind = "lib"\n' > "$d/lib/mcpp.toml" + printf 'export module probe.sdk;\nexport int sdk_value() { return 42; }\n' > "$d/lib/src/sdk.cppm" + git_init "$d/lib" + (cd "$d/lib" && run publish --dry-run >/dev/null 2>&1) + local lua="$d/lib/target/dist/probe.sdk.lua" tar="$d/lib/target/dist/probe.sdk-0.1.0.tar.gz" + [ -f "$lua" ] && [ -f "$tar" ] || { echo "RESULT 960: skipped: publish wrote no descriptor"; return; } + echo "-- xpkg parse of what publish wrote:" + local parsed; parsed="$(run xpkg parse "$lua" 2>&1)"; echo "$parsed" | tail -2 + cp "$lua" "$d/index/pkgs/p/" + local store="$d/app/.mcpp/.xlings/data/xpkgs/probe.sdk/0.1.0"; mkdir -p "$store" + tar -xzf "$tar" -C "$store" + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[indices]\nprobe = { path = "%s" }\n\n[dependencies.probe]\nsdk = "0.1.0"\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' "$d/index" > "$d/app/mcpp.toml" + printf 'import std;\nimport probe.sdk;\nint main() { std::println("v={}", sdk_value()); }\n' > "$d/app/src/main.cpp" + echo "-- a consumer of that descriptor:" + local inlineOut; inlineOut="$(cd "$d/app" && run build 2>&1 | tail -1)"; echo "$inlineOut" + # The string form: the field replaced, nothing else. + awk 'BEGIN{skip=0} /^ mcpp = \{/{print " mcpp = \"*/mcpp.toml\","; skip=1; next} skip&&/^ \},/{skip=0; next} !skip{print}' \ + "$lua" > "$d/index/pkgs/p/probe.sdk.lua" + echo "-- the same archive, the field as a string:" + local stringOut; stringOut="$(cd "$d/app" && run run 2>&1 | tail -1)"; echo "$stringOut" + if echo "$inlineOut" | grep -q 'missing sources' && [ "$stringOut" = "v=42" ]; then + echo "RESULT 960: reproduced" + else echo "RESULT 960: not reproduced"; fi +} + +# 956: mcpplibs.os as a build dependency under --target x86_64-macos-musl, with +# an index that lists a newer patch of a member of the platform's version group. +case_956() { + local idx="${INDEX_830:-}" + [ -n "$idx" ] && [ -f "$idx/pkgs/o/openkal-macos.lua" ] || { echo "RESULT 956: skipped: INDEX_830 is not set"; return; } + local d="$ROOT/956"; mkdir -p "$d/dep/src" "$d/build/src" + for kind in dep build; do + local table="dependencies"; [ "$kind" = build ] && table="build-dependencies" + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[indices]\nmcpplibs = { path = "%s" }\n\n[%s]\nmcpplibs.os = "0.4.4"\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' "$idx" "$table" > "$d/$kind/mcpp.toml" + printf 'import std;\nint main() { std::println("hi"); }\n' > "$d/$kind/src/main.cpp" + done + printf 'import std;\nimport mcpp;\nint main() { return 0; }\n' > "$d/build/build.mcpp" + echo "-- as a dependency (the group binds the range):" + (cd "$d/dep" && run build --target x86_64-macos-musl --configure-only 2>&1 | grep -E 'Resolved openkal-macos|^error' | sort -u) + echo "-- as a build dependency:" + local out; out="$(cd "$d/build" && run build --target x86_64-macos-musl --configure-only 2>&1)" + echo "$out" | grep -E 'Resolved openkal-(macos|linux)|^error|requested by' + echo "-- the same manifest for the host (which table of mcpplibs.os is read):" + (cd "$d/build" && run build --configure-only 2>&1 | grep -E 'Resolved openkal-(macos|linux)|^error') + if echo "$out" | grep -q 'is a version group that names'; then echo "RESULT 956: reproduced" + else echo "RESULT 956: not reproduced"; fi +} + +# 952: a build ended by SIGKILL inside a new pid namespace, continued from this +# one; then the same after `mcpp clean`; and a control in one namespace. +case_952() { + unshare -Urpf --mount-proc true 2>/dev/null || { echo "RESULT 952: skipped: no new pid namespace here"; return; } + local d="$ROOT/952"; mkdir -p "$d/seed/src" + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' > "$d/seed/mcpp.toml" + local main='import std;\n' body='int main() { int s = 0;' + for i in $(seq 0 23); do + printf 'import std;\nint f%s(int x) { std::vector v; for (int i = 0; i < x; ++i) v.push_back(std::format("{}", i)); std::ranges::sort(v); return (int)v.size(); }\n' "$i" > "$d/seed/src/u$i.cpp" + main="$main""int f$i(int);\n"; body="$body s += f$i(3);" + done + printf "$main$body"' std::println("{}", s); }\n' > "$d/seed/src/main.cpp" + pending() { local n=0 a; for a in "$1"/.mcpp/compiler-runs/*/run-*/attempts/*; do + [ -e "$a/admitted" ] && [ ! -e "$a/retired" ] && n=$((n+1)); done; echo "$n"; } + local hit="" t + for t in 0.8 1.0 1.4 1.8 2.2 2.6; do + local p="$d/ns-$t"; mkdir -p "$p"; cp -r "$d/seed/src" "$d/seed/mcpp.toml" "$p/" + (cd "$p" && unshare -Urpf --mount-proc timeout -s KILL "$t" env MCPP_JOBS="$JOBS" "$MCPP" build >/dev/null 2>&1) + if [ "$(pending "$p")" -gt 0 ]; then hit="$p"; echo "-- killed after ${t}s: $(pending "$p") admitted attempts without retirement"; break; fi + done + [ -n "$hit" ] || { echo "RESULT 952: skipped: no kill landed during a compile"; return; } + echo "-- the next build, from this pid namespace:" + local out; out="$(cd "$hit" && run build 2>&1 | tail -1)"; echo "$out" + echo "-- after mcpp clean:" + (cd "$hit" && run clean 2>&1 | tail -1; run build 2>&1 | tail -1) + echo "-- control: killed and continued in one namespace:" + local c="$d/control" ok="" + for t in 0.8 1.0 1.4 1.8; do + mkdir -p "$c-$t"; cp -r "$d/seed/src" "$d/seed/mcpp.toml" "$c-$t/" + (cd "$c-$t" && timeout -s KILL "$t" env MCPP_JOBS="$JOBS" "$MCPP" build >/dev/null 2>&1) 2>/dev/null + if [ "$(pending "$c-$t")" -gt 0 ]; then sleep 2; (cd "$c-$t" && run build 2>&1 | tail -1); ok=1; break; fi + done + [ -n "$ok" ] || echo "(no kill landed during a compile)" + if echo "$out" | grep -q 'occupied or unknown'; then echo "RESULT 952: reproduced" + else echo "RESULT 952: not reproduced"; fi +} + +# 931: a file a link flag names as the value of an option changes; the program +# is relinked or it is not. Control: the same file named as a response file. +case_931() { + local d="$ROOT/931" + link_stamp() { stat -c %Y "$1"/target/*/*/bin/app 2>/dev/null | head -1; } + probe() { # + mkdir -p "$1/src"; printf '{ main; };\n' > "$1/list.txt" + local flag; flag="$(printf -- "$2" "$1/list.txt")" + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[build]\nldflags = ["%s"]\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' "$flag" > "$1/mcpp.toml" + printf 'int helper() { return 1; }\nint main() { return helper() - 1; }\n' > "$1/src/main.cpp" + (cd "$1" && run build >/dev/null 2>&1) || { echo "build-failed"; return; } + local before; before="$(link_stamp "$1")"; sleep 1.1 + printf '{ main; helper; };\n' > "$1/list.txt" + (cd "$1" && run build >/dev/null 2>&1) + [ "$(link_stamp "$1")" = "$before" ] && echo "not relinked" || echo "relinked" + } + local asOption; asOption="$(probe "$d/option" '-Wl,--dynamic-list=%s')" + echo "-- the file as the value of an option (-Wl,--dynamic-list=): $asOption" + local asRsp; asRsp="$(probe_rsp "$d/rsp")" + echo "-- control, the file as a response file (@): $asRsp" + [ "$asOption" = "not relinked" ] && echo "RESULT 931: reproduced" || echo "RESULT 931: not reproduced" +} +probe_rsp() { + mkdir -p "$1/src"; printf -- '-Wl,--as-needed\n' > "$1/args.rsp" + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[build]\nldflags = ["@%s"]\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' "$1/args.rsp" > "$1/mcpp.toml" + printf 'int main() { return 0; }\n' > "$1/src/main.cpp" + (cd "$1" && run build >/dev/null 2>&1) || { echo "build-failed"; return; } + local before; before="$(stat -c %Y "$1"/target/*/*/bin/app | head -1)"; sleep 1.1 + printf -- '-Wl,--no-as-needed\n' > "$1/args.rsp" + (cd "$1" && run build >/dev/null 2>&1) + [ "$(stat -c %Y "$1"/target/*/*/bin/app | head -1)" = "$before" ] && echo "not relinked" || echo "relinked" +} + +# 954: an abi table stated once at a virtual workspace root; the warning meant +# for a dependency's own table is counted. +case_954() { + local d="$ROOT/954"; mkdir -p "$d/a/src" "$d/b/src" + printf '[workspace]\nmembers = ["a", "b"]\n\n[workspace.target.'"'"'cfg(linux)'"'"'.abi]\nexceptions = true\n' > "$d/mcpp.toml" + printf '[package]\nname = "a"\nversion = "0.1.0"\n\n[targets.a]\nkind = "lib"\n' > "$d/a/mcpp.toml" + printf 'export module a;\nexport int a_value() { return 1; }\n' > "$d/a/src/a.cppm" + printf '[package]\nname = "b"\nversion = "0.1.0"\n\n[dependencies]\na = { path = "../a" }\n\n[targets.b]\nkind = "bin"\nmain = "src/main.cpp"\n' > "$d/b/mcpp.toml" + printf 'import a;\nint main() { return a_value() - 1; }\n' > "$d/b/src/main.cpp" + local out; out="$(cd "$d" && run build --workspace --configure-only 2>&1)" + echo "$out" | grep -E 'declares \[target|^error' | sort | uniq -c + local n; n="$(echo "$out" | grep -c 'declares \[target')" + [ "$n" -gt 0 ] && echo "RESULT 954: reproduced ($n warnings, no member has an abi table)" || echo "RESULT 954: not reproduced" +} + +# 958: a compile command of more than 512 argument words. +case_958() { + local d="$ROOT/958"; mkdir -p "$d/src" + local defs="" i + for i in $(seq 0 519); do defs="$defs\"PROBE_D$i=1\", "; done + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[build]\ndefines = [%s]\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' "${defs%, }" > "$d/mcpp.toml" + printf 'int main() { return PROBE_D519 - 1; }\n' > "$d/src/main.cpp" + local out; out="$(cd "$d" && run build 2>&1 | grep -E '^cannot start|Finished' | cut -c1-200)"; echo "$out" + if echo "$out" | grep -qi 'out of memory'; then echo "RESULT 958: reproduced (520 defines)" + else echo "RESULT 958: not reproduced"; fi +} + +# 955: where a plugin's own host program is visible. `plug` has a library and +# a program `steps`; its library asks for the program's path. +case_955() { + local d="$ROOT/955"; mkdir -p "$d/plug/src" "$d/plug/tools" "$d/fw/src" "$d/direct/src" "$d/through/src" + printf '[package]\nname = "plug"\nversion = "0.1.0"\n\n[targets.plug]\nkind = "lib"\n\n[targets.steps]\nkind = "bin"\nmain = "tools/steps_main.cpp"\n' > "$d/plug/mcpp.toml" + printf 'export module plug;\nimport std;\nimport mcpp;\nexport void plug_declare() {\n const char* p = mcpp::dep_bin("plug", "steps");\n mcpp::warning(std::format("steps=[{}]", p ? p : "(null)").c_str());\n}\n' > "$d/plug/src/plug.cppm" + printf 'int main() { return 0; }\n' > "$d/plug/tools/steps_main.cpp" + printf '[package]\nname = "fw"\nversion = "0.1.0"\n\n[build-dependencies]\nplug = { path = "../plug", tools = ["steps"], host-module = true }\n\n[targets.fw]\nkind = "lib"\n' > "$d/fw/mcpp.toml" + printf 'export module fw;\nimport mcpp;\nimport plug;\nexport void fw_declare() { plug_declare(); }\n' > "$d/fw/src/fw.cppm" + printf '[package]\nname = "direct"\nversion = "0.1.0"\n\n[build-dependencies]\nplug = { path = "../plug", tools = ["steps"], host-module = true }\n\n[targets.direct]\nkind = "bin"\nmain = "src/main.cpp"\n' > "$d/direct/mcpp.toml" + printf 'import mcpp;\nimport plug;\nint main() { plug_declare(); return 0; }\n' > "$d/direct/build.mcpp" + printf '[package]\nname = "through"\nversion = "0.1.0"\n\n[build-dependencies]\nfw = { path = "../fw" }\n\n[targets.through]\nkind = "bin"\nmain = "src/main.cpp"\n' > "$d/through/mcpp.toml" + printf 'import mcpp;\nimport fw;\nint main() { fw_declare(); return 0; }\n' > "$d/through/build.mcpp" + printf 'int main() { return 0; }\n' | tee "$d/direct/src/main.cpp" > "$d/through/src/main.cpp" + local a b + a="$(cd "$d/direct" && run build 2>&1 | grep -o 'steps=\[[^]]*\]')" + b="$(cd "$d/through" && run build 2>&1 | grep -o 'steps=\[[^]]*\]')" + echo "-- the edge that names the tool: ${a:-no answer}" + echo "-- one package away, through a framework that names it: ${b:-no answer}" + if [ -n "$a" ] && [ "$a" != "steps=[]" ] && [ "$b" = "steps=[]" ]; then + echo "RESULT 955: reproduced (the path reaches the declaring package only; elsewhere dep_bin answers an empty string)" + else echo "RESULT 955: not reproduced"; fi +} + +# 952s: two consecutive container jobs. A build is killed in one fresh pid +# namespace and continued in another fresh one; the namespaces' names are +# printed, because the kernel gives a later namespace an earlier one's number. +case_952s() { + unshare -Urpf --mount-proc true 2>/dev/null || { echo "RESULT 952s: skipped: no new pid namespace here"; return; } + local d="$ROOT/952s"; mkdir -p "$d/seed/src" + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' > "$d/seed/mcpp.toml" + local main='import std;\n' body='int main() { int s = 0;' i + for i in $(seq 0 23); do + printf 'import std;\nint f%s(int x) { std::vector v; for (int i = 0; i < x; ++i) v.push_back(std::format("{}", i)); std::ranges::sort(v); return (int)v.size(); }\n' "$i" > "$d/seed/src/u$i.cpp" + main="$main""int f$i(int);\n"; body="$body s += f$i(3);" + done + printf "$main$body"' std::println("{}", s); }\n' > "$d/seed/src/main.cpp" + printf '#!/bin/sh\nreadlink /proc/self/ns/pid > ns-a.txt\nMCPP_JOBS=%s timeout -s KILL 1.0 "$1" build >/dev/null 2>&1\nexit 0\n' "$JOBS" > "$d/kill.sh" + printf '#!/bin/sh\nreadlink /proc/self/ns/pid > ns-b.txt\nMCPP_JOBS=%s "$1" build 2>&1 | tail -1 > result.txt\nexit 0\n' "$JOBS" > "$d/continue.sh" + local refused=0 trials=0 seen="" t + for t in 1 2 3 4; do + local p="$d/t$t"; mkdir -p "$p"; cp -r "$d/seed/src" "$d/seed/mcpp.toml" "$p/" + (cd "$p" && unshare -Urpf --mount-proc sh "$d/kill.sh" "$MCPP") + local n=0 a; for a in "$p"/.mcpp/compiler-runs/*/run-*/attempts/*; do [ -e "$a/admitted" ] && [ ! -e "$a/retired" ] && n=$((n+1)); done + [ "$n" -gt 0 ] || continue + (cd "$p" && unshare -Urpf --mount-proc sh "$d/continue.sh" "$MCPP") + trials=$((trials+1)); seen="$seen $(cat "$p/ns-b.txt")" + echo "-- job A in $(cat "$p/ns-a.txt"), job B in $(cat "$p/ns-b.txt"): $(cut -c1-70 "$p/result.txt")" + grep -q 'occupied or unknown' "$p/result.txt" && refused=$((refused+1)) + done + local distinct; distinct="$(echo $seen | tr ' ' '\n' | sort -u | wc -l)" + echo "-- $trials second jobs ran in $distinct distinct namespace names" + [ "$trials" -gt 0 ] && [ "$refused" = "$trials" ] && echo "RESULT 952s: reproduced ($refused of $trials refused)" || echo "RESULT 952s: not reproduced ($refused of $trials refused)" +} + +# 931d: what the linker itself reports. The link is given --dependency-file +# through the project's own flags; the file names what the linker read. +case_931d() { + local d="$ROOT/931d"; mkdir -p "$d/src"; printf '{ main; };\n' > "$d/list.txt" + printf '[package]\nname = "app"\nversion = "0.1.0"\n\n[build]\nldflags = ["-Wl,--dynamic-list=%s", "-Wl,--dependency-file=%s"]\n\n[targets.app]\nkind = "bin"\nmain = "src/main.cpp"\n' "$d/list.txt" "$d/link.d" > "$d/mcpp.toml" + printf 'int main() { return 0; }\n' > "$d/src/main.cpp" + (cd "$d" && run build >/dev/null 2>&1) + [ -f "$d/link.d" ] || { echo "RESULT 931d: not reproduced: the linker wrote no dependency file"; return; } + echo "-- files named by the linker's dependency file: $(tr ' \\' '\n\n' < "$d/link.d" | grep -c '^/')" + echo "-- the option's file among them: $(grep -c "$d/list.txt" "$d/link.d") line(s); a library found by search among them: $(grep -c 'libc\.so\|libc\.a\|libstdc++' "$d/link.d") line(s)" + grep -q "$d/list.txt" "$d/link.d" && echo "RESULT 931d: reproduced (the linker names the option's file and the libraries it found)" || echo "RESULT 931d: not reproduced" +} + +[ $# -gt 0 ] || set -- 960 956 952 952s 931 931d 954 958 955 +for c in "$@"; do echo; echo "==== $c"; "case_$c"; done diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md new file mode 100644 index 000000000..81575ab9e --- /dev/null +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md @@ -0,0 +1,753 @@ +--- +subject: ecosystem +status: active +--- + +# 生态规则收敛方案:2026.11.18.1 之后十个 issue 的五类成因、七项决定与实施范围 + +**日期:** 2026-11-19 · **版本:** v3(待评审,未实施。用户决定:全部在本轮实施,每个仓库一个拉取请求,提交作为检查点, +从成因解决而不留变通。第 15 节为七项决定的依据,第 16 节为两轮自评,第 17 节为已定事项与仍需决定的三项) · **基线:** mcpp 2026.11.18.1(`3b886163`)、openkal 0.25.3、openkal-linux 0.26.3(`363ccbb`)、 +openkal-macos 0.23.5(`9dbc36f`,索引中为 0.23.4)、openkal-platform 0.29.2(引擎默认)与 0.30.1、mcpplibs.os 0.4.4、 +mcpp.plugins 0.26.0、mcpp-index `3ee5f08`、xlings `4f5df092` · **前文:** +[第二部分之后的补充方案](2026-11-17-ninja-cascade-publish-supplement-design.md)、 +[编译尝试的生命期与输出恢复](2026-11-06-compiler-attempt-recovery.md) · **复现:** +[`2026-11-19-ecosystem-rule-convergence-design-repro.sh`](2026-11-19-ecosystem-rule-convergence-design-repro.sh) + +## 1. 来源、范围与结论 + +本文处理 mcpp#931、#952 至 #960 十个 issue 中属于通用基础设施的部分。每一项先核对 `origin/main` 的实现与相关仓库的 +当前提交,再判定是否纳入;纳入的项给出原因与方案。#953 是一份遗留项清单,逐条判定,只有下表列出的条目在本文范围内。 + +| 来源 | 判定 | 性质 | 章节 | +|---|---|---|---| +| #960 | 纳入,范围大于报告 | `mcpp publish` 写出的描述符,任何读取方都不接受 | 3 | +| #956 | 纳入,原因与报告不同 | 版本组的精确约束不进入合并;build-only 包的条件表按目标求值 | 4 | +| #957 | 纳入 | 可达性谓词之前有一条按系统名写的特例 | 5 | +| #931 | 纳入,无新接口 | 链接的输入由标志推断,已是同一类问题的第三次;链接器自己的报告没有被使用 | 6 | +| #954 | 纳入 | 条件行不带来源,继承的行与成员自己的行不可分 | 7 | +| #959 | 纳入 | 引擎独有的事实(链接单元的对象)没有通道;规划已知的文本没有写出方式 | 8 | +| #955 | 不按原样采纳,问题纳入 | 多步动作的运行器归插件;引擎补一条让插件携带自身程序的通道 | 8 | +| #958 | 纳入,严重程度高于报告 | openkal 的两个实现对向量条数设界;普通工程的编译命令即可触及 | 9 | +| #952 | 纳入,第 1 项请求不按原样采纳 | 单元的观察把“从此处不可观察”与“暂时未知”合为一种回答;进程空间的标识不精确 | 10 | +| #953 · Termux 应用沙箱 | 纳入 | `__compiler-cap` 的多条返回路径不输出原因;能力缺失时没有规定的行为 | 10 | +| #953 · 评论一(unit admission 被拒) | 纳入 | 六种条件一条消息,拒绝不可重试 | 10 | +| #953 · 特性并集不同的 43 条边 | 纳入 | 同一输出路径对应两种配置 | 11 | +| #953 · Windows LLVM 载荷的 lld 不带 zlib | 纳入 | 同一工具链载荷在不同宿主上能力不同 | 11 | +| #953 · e2e 1033 与 986 的间歇失败 | 只纳入证据的留存 | 第一次出现时没有留下可供诊断的状态 | 12 | + +已部分实现、本文只补缺口的三处:#956 的目标图请求自 2026.11.7.1 起已由版本组约束(SPEC-013 §7.5);#931 的整词绝对路径与 +`@file` 自 2026.10.31.1 与 2026.11.2.2 起已是链接输入;#959 所需的“复制一个文件”自 2026.9.13.1 起由 `${mcpp.self} stage` 提供。 + +### 1.1 实测 + +以 mcpp 2026.11.18.1 在 Linux x86_64 上实测,脚本见文首(九例)。其余各项的依据是源码,逐项写明。 + +| 项 | 做法 | 结果 | +|---|---|---| +| #960 | `mcpp publish --dry-run` 写出描述符;`mcpp xpkg parse` 读取它;放入本地索引,载荷取自同一次发布的归档,消费者构建 | 两处均报 `synthesised manifest missing sources (mcpp segment must declare sources = { ... })`;仅把字段改为 `mcpp = "*/mcpp.toml"` 后,消费者构建并输出 `v=42` | +| #956 | 索引取列出 openkal-macos 0.23.5 的提交(mcpp-index `333be93`);`--target x86_64-macos-musl --configure-only` | `mcpplibs.os` 在 `[dependencies]`:`openkal-macos =0.23.4`,通过。在 `[build-dependencies]`:`>=0.22.0, <0.24.0 → v0.23.5`,随后 `=0.23.4 → v0.23.4`,没有 `Merged` 一行,拒绝,消息与报告逐字相同 | +| #956 | 同一份清单(build 依赖),不带 `--target` | 请求的是 `openkal-linux`;带 `--target x86_64-macos-musl` 时请求的是 `openkal-macos`,没有 `openkal-linux` | +| #956 | 当前索引,交叉到 `x86_64-macos-musl`,构建程序 `import mcpplibs.os;`,完整构建 | 构建成功;`mcpp.lock` 记录 `openkal-macos 0.23.4`,没有构建程序实际链接的 `openkal-linux` | +| #952 | 在新的 pid 命名空间中构建并以 SIGKILL 结束(8 个已准入而未退役的尝试);从另一个命名空间继续 | `previous compiler unit is occupied or unknown`,每次构建相同;`mcpp clean` 之后仍相同。对照:同一命名空间内结束并继续,2.9–3.5 s 后构建完成 | +| #952 | 连续两个容器作业:在一个新的命名空间中结束构建,在另一个新的命名空间中继续,四次 | 四次均拒绝。四个不同的第二命名空间的名字相同(`pid:[4026533469]`):内核把先前的编号给了后来的命名空间 | +| #931 | `[build] ldflags = ["-Wl,--dynamic-list=<绝对路径>"]`,构建后改写该文件再构建 | 程序未重新链接。对照:同一文件以 `@<绝对路径>` 给出时重新链接 | +| #931 | 同一工程的链接另加 `-Wl,--dependency-file=<文件>`(GNU ld 2.42.1;ld.lld 23.1.3 另以手工命令核对) | 链接器写出 make 格式的依赖文件,列出 70 个它读取的文件:选项点名的文件、链接脚本、版本脚本,以及按搜索找到的库 | +| #954 | 虚拟根只写 `[workspace.target.'cfg(linux)'.abi] exceptions = true`,两个成员均无 `abi` 表 | 每个成员一条 `declares [target..abi]` 警告 | +| #958 | `[build] defines` 写 520 项,在终端中 `mcpp build` | `cannot start '…/g++' (error 12): Out of memory`,第一个编译步骤失败 | +| #955 | 插件包 `plug` 带库与程序 `steps`;库以 `mcpp::dep_bin("plug", "steps")` 取路径 | 在清单中写 `tools = ["steps"], host-module = true` 的包的构建程序里得到工具存储中的路径;经一个声明了它的框架到达的应用的构建程序里得到空串,没有诊断 | + +## 2. 判定的依据 + +一项请求纳入方案须同时满足三条:它在一个以上的工程或平台上出现(通用);现有机制不能表达它(必要);它的解法符合 AGENTS.md +第 2、3 节的约束。不满足第三条而问题本身成立的,改用符合约束的解法(#955、#952 的第 1 项、#931 的两个候选接口)。 + +十个 issue 的原因归为五类,方案按类给出,而不是逐个修补: + +| 类 | 表现 | 涉及 | +|---|---|---| +| 一个事实有两个定义 | 写出方与读取方各自定义描述符的形式;没有规范条款,也没有往返测试 | #960 | +| 一条规则带着例外 | 约束改写了请求却没有进入合并;谓词之前有按系统名写的分支;规则只覆盖了它所述对象的一部分写法 | #956、#957、#931、#954 | +| 引擎的事实没有通道 | 只有引擎知道的事实不能进入动作,于是规则在事后猜测 | #959 | +| 工具的逻辑进入引擎 | 把会随工具变化的词汇放进引擎 | #955 | +| 不可观察被当作可以等待 | 一种不会改变的回答被等待后拒绝;拒绝不写出对象与出路 | #952、#953、#958 | + +## 3. 描述符的 `mcpp` 字段:两种形式与发布的写出(#960) + +### 3.1 现状 + +- **写出方。** `emit_xpkg`(`src/pm/publisher.cppm`)写出 `mcpp = { schema, language, import_std, modules, deps, [runtime], manifest = "mcpp.toml" }`。 +- **读取方。** 六处按字段的种类分支:`src/build/prepare/graph_load.cpp` 的布局核对(795)、安装前的子依赖(916)、读清单(1298)、 + 补丁根(1456),`src/scaffold/create.cppm`(91),`src/cli/cmd_xpkg.cppm`(196)。表形式一律交给 `synthesize_from_xpkg_lua`, + 它要求 `sources`;`manifest` 不在 `kKnownXpkgKeys`(`modules/manifest/src/xpkg.cppm`)中。 +- **结果。** `mcpp publish` 与 `mcpp emit xpkg` 写出的描述符不能被解析、不能作为依赖、不能作为模板包(1.1 节)。#960 报告的是其中第三个。 +- **未被发现的原因。** mcpp-index 中带 `mcpp` 字段的描述符 74 个为字符串形式、194 个为带 `sources` 的表形式,没有一个带 `manifest` 键; + `tests/unit/test_xpkg_emit.cpp` 断言写出方输出 `manifest = "mcpp.toml"`;e2e 772 以 `path` 依赖消费发布的归档,不经描述符。 +- **规范。** `mcpp` 字段的形式只在 mcpp-index 的文档中以 Form A、Form B 描述,mcpp 的规范没有条款。 + +### 3.2 方案 + +1. **条款。** SPEC-001 增加一节“描述符的 `mcpp` 字段”: + - **指针形式** `mcpp = ""`:载荷带有清单。glob 相对版本目录,恰好匹配一个 `mcpp.toml`,它所在的目录是包根。字段缺省时依次 + 尝试 `mcpp.toml` 与 `*/mcpp.toml`。 + - **内联形式** `mcpp = { ... }`:载荷不带清单,该表即清单;键的集合是 `kKnownXpkgKeys`,`sources` 必须给出。 + - 表中指向载荷内清单的写法不是第三种形式,`manifest` 键不存在。Form A、Form B 作为两种形式的别名保留。 +2. **写出方。** 发布的归档就是包的源码树,其顶层目录内有清单(工作区成员为规范化后的清单),因此 `emit_xpkg` 写出 + `mcpp = "*/mcpp.toml"`。`xpm..deps`(安装期的边)不变。`modules`、`deps` 与 `runtime.requirements` 不再写入描述符 + (决定 D6,依据见 15.6)。 +3. **一个读取函数。** `descriptor_manifest(lua, 版本目录, 名字, 版本, 平台)` 返回清单、包根与形式,取代上述六处分支。 + `mcpp new --template` 由包根查找 `templates/`,拒绝的条件是没有该目录,消息写出查找的目录,不再以形式为由拒绝。 +4. **往返测试。** 单元测试:`emit_xpkg` 的输出经 `mcpp xpkg parse` 的同一函数读取成功。e2e:发布一个带模板的成员,以本地索引 + 消费它,构建并运行;`mcpp new --template <包>:<模板>` 创建工程并构建。`test_xpkg_emit.cpp` 中对 `manifest` 键的断言改为对指针形式的断言。 + +### 3.3 兼容 + +指针形式是已发布的每个 mcpp 都读取的形式(索引中 74 个条目)。此前由 `mcpp publish` 生成的描述符没有进入过索引, +不需要迁移。文档:docs/11 与 docs/zh/11 写明 `mcpp publish` 写出的字段。 + +## 4. 版本解析:版本组的约束与 build-only 包的条件表(#956) + +### 4.1 原因 + +报告把原因写为“范围请求与精确请求分别解析”。实测表明目标图的范围请求已解析为组中版本,失败只发生在 `mcpplibs.os` +经构建依赖进入图时(1.1 节)。原因是两条,彼此独立: + +- **F1 版本组的精确约束不进入合并。** `step4b_pin_to_platform_group`(`src/build/prepare/platform.cpp:1026`)把请求的版本改写为 + `=<组中版本>`,并跳过 build-only 的请求(SPEC-013 §7.5:只为本机构建的包不受约束)。同一个包的第二个请求到达时, + `step4b_identity_version_merge`(`src/build/prepare/graph.cpp:697`)合并的是双方**书写时**的约束(`item.originalConstraint`), + 改写后的精确约束不在其中。于是 build-only 的边先取得区间内最新的 0.23.5,平台的层随后到达,合并认为已有版本满足双方书写的约束而保留它 + (实测没有 `Merged` 一行),遍历之后的断言再把它拒绝。通用的合并规则已经存在,版本组绕过了它。 +- **F2 build-only 包的条件表按目标求值。** 交叉到 `x86_64-macos-musl` 时,一个只经构建依赖到达的包读取的是 + `[target.macos.dependencies]`,而它为构建机器构建(SPEC-013 的“宿主图”)。主遍因此解析了任何构建都不使用的包, + 而构建程序实际链接的 openkal-linux 不在 `mcpp.lock` 中(1.1 节第四行):交叉构建时锁没有记录宿主一侧的构建内容。 + +F1 是 #956 的直接原因;F2 使它在宿主与目标不是同一系统时也出现,并且是锁的完整性缺口。任何平台在组中列出、 +又被某个构建期的库以范围请求的成员都会遇到;插件全部依赖 `mcpplibs.os`,所以带插件的工程在 hosted 行上都在其中。 + +### 4.2 方案 + +1. **R1 版本组的约束进入合并。** 版本组对一个成员的精确约束是该请求的约束,随请求进入解析表的记录与此后的每次合并。 + 同一个包的全部请求由一个它们都容许的版本满足:`>=0.22.0, <0.24.0` 与组的 `=0.23.4` 给出 0.23.4(输出一行 `Merged`), + 与请求到达的先后无关。没有这样的版本时拒绝,消息列出每个请求者与其约束(`ResolvedRecord::constraintsBy` 已有记录, + 当前只写出第一个)。只为本机构建、始终没有进入目标图的包不受约束,SPEC-013 §7.5 的条文不变;遍历之后的检查保留为断言。 +2. **R2 条件表的求值行。** 包为哪一行构建,它的 `[target..*]` 就按哪一行选择,依赖表在内。只经构建依赖到达的包按宿主行求值, + 主遍与宿主图自身的准备给出同一组请求,`mcpp.lock` 记录构建程序实际链接的包。写入 SPEC-004 的解析轴一节与 SPEC-013 §2 的“宿主图”定义。 + +v1 的做法(对 build-only 的请求也提前改写为组中版本)撤回:它让目标平台的版本组约束了只为本机构建的包,与 §7.5 的条文相反, +而 R1 用已有的合并规则得到同样的结果。 + +### 4.3 验证与生态规则 + +- e2e(本地索引,一个带 `[platform.versions] m = "1.0.0"` 的平台夹具,索引列出 `m` 的 1.0.0 与 1.0.1):build-only 的库以 `^1.0` + 请求 `m`,目标图也使用 `m`,结果为 1.0.0 并输出 `Merged`,两种到达顺序各一例;build-only 的库以 `=1.0.1` 请求时拒绝并列出两个请求者; + build-only 且目标图不使用 `m` 时为 1.0.1;build-only 的包带两个系统的条件依赖表时,交叉构建读取宿主的那一个,锁记录它。 +- 复现脚本的 956 一项不再复现。 +- “成员的补丁不列入索引”这一临时规则随修正撤销(见 14.3)。 +- R2 改变主遍为 build-only 包解析出的包:交叉构建时 `mcpp.lock` 记录的是宿主一侧的成员。升级后的处理见 17.1 的 U5。 + +## 5. 执行可达性的单一谓词(#957) + +`unreachable_execution`(`src/build/execute.cppm:1525`)的注释写明“一个谓词适用于每一行”,而谓词之前有一个分支: +宿主不是 macOS 且三元组属于 Apple 时一律拒绝,不看行是否声明了运行器。其后的通用谓词(行的执行系统不是本机系统且没有声明运行器时拒绝) +已经给出同样的答案,并在有运行器时放行。该分支在引擎中点名了一个系统(AGENTS.md 第 2 节)。 + +**方案。** 删除该分支。Apple 的行与其余的行一样由通用谓词判定:运行器的 `hosts` 含本机时可达,否则拒绝,消息由通用分支给出 +(执行系统取自行的数据,已含 `run_under` 与运行器两条出路);`other_hosts_note` 照常附加。 + +**使这一类不再进入。** `check_system_names.py` 的词表含宿主谓词(`is_macos` 等),不含三元组的系统谓词,所以该分支的 `t->is_apple()` +不被计数。词表增加三元组的系统谓词(`is_apple`、`is_ios`、`is_ios_simulator` 与同类),基线取当前计数;本项删除的分支使 +`is_macos` 与 `is_apple` 的基线各降一。 + +**验证。** 单元测试:一行 Apple 三元组、一个 `hosts` 含 Linux 的运行器,谓词放行;无运行器时拒绝且消息写出执行系统。 +e2e:平台夹具定义一行 Apple 三元组并声明一个回显参数的运行器,`mcpp run` 在 Linux 上调用它。docs/04(`run_under` 一节, +“Apple artifact on a host that is not macOS, with no `run_under`, is refused”)按新行为改写(中英文)。PureDarwin 的运行器包是实验性包, +不属于本方案。 + +## 6. 链接标志点名的文件是链接的输入(#931) + +### 6.1 现状 + +`src/build/flags.cppm`(`ldFileInputs`)已有规则:链接标志中以绝对路径点名的现有文件,是携带该标志的每个链接的隐式输入。 +它识别两种写法:整个词是路径;`@<文件>` 及该文件列出的路径。选项的值不在其中(`--js-library=<文件>`、 +`-Wl,--version-script=<文件>`、`-Wl,-force_load,<文件>`、`/DEF:<文件>`),实测见 1.1 节。构建程序重新运行后给出相同的标志, +链接命令不变,于是链接被视为最新。 + +这是同一类问题的第三次:2026.10.31.1 补了整词的绝对路径(mcpp#869),2026.11.2.2 补了响应文件(mcpp#883),本次是选项的值。 +三次的共同点是链接的输入由引擎从标志中推断。编译边没有这类问题,因为编译器自己报告它读了什么(依赖文件)。 + +### 6.2 方案 + +链接边的输入与编译边一样由两部分组成:规划时已知的,与工具自己报告的。 + +**B 链接器的报告。** 链接器能以 make 格式写出它读取的文件时,链接边带上该依赖文件(ninja 的 `depfile` 与 `deps = gcc`)。 +GNU ld(2.35 起)与 ld.lld 以 `--dependency-file=<文件>` 提供,实测见 1.1 节;它覆盖标志推断不可能覆盖的输入: +按 `-l` 搜索找到的库、链接脚本中 `INPUT` 引入的文件。哪个链接器提供、以哪个选项,是链接模型 +(`modules/toolchain-model/src/linkmodel.cppm`)中链接器的一个事实,引擎的规则只读取它。 +ld64 的 `-dependency_info` 是另一种格式,wasm-ld、lld-link 与 link.exe 在帮助中没有对应的选项(LLVM 23.1.3),这些链接器只有 A。 + +**A 规划时已知的。** 现有规则不变,补全“点名”的位置。它覆盖链接器不读取而由驱动读取的文件(emcc 的 `--js-library`、`--pre-js`), +以及不提供报告的链接器。对每个链接词取候选:整个词;`=` 之后的部分;以 `,` 分隔的每一段;MSVC 链接方言下以 `/` 开头的词中 +第一个 `:` 之后的部分。候选是绝对路径,且指向一个现有的普通文件或本次规划中某个动作声明的输出时,它是隐式输入。 +目录(`-L`、`-rpath`、`--sysroot`)不是普通文件,不受影响;`-fuse-ld=<绝对路径>` 这类指向程序的值成为输入,程序变化时重新链接,符合规则的本意。 +多认一个文件的代价是该文件变化时多一次链接,少认一个的代价是陈旧的产物,所以候选取宽。 + +不增加接口。报告的两个候选都不采纳:`mcpp::link_input` 一名已由 SPEC-013 的平台陈述使用,含义是“链接进每个映像的文件”; +把构建程序的 `rerun_if_changed` 文件计为链接输入,会使构建程序的任何输入变化都重新链接它的全部消费者。 + +### 6.3 文档与验证 + +docs/30 中“写在 `[build].ldflags` 里的预构建对象不被跟踪,修改后报告 `ninja: no work to do`”一段与 2026.10.31.1 起的实现不符, +随本项改写为两条规则及其边界:只有 A 的链接器上,相对路径与规划时尚不存在、也不是任何动作输出的文件不是输入。 +e2e:三种写法(整词、`@file`、选项的值)各改写一次文件,程序重新链接,`mcpp:link-script` 的脚本一并断言; +在提供报告的链接器上,改写一个只经 `-L`、`-l` 找到的静态库,程序重新链接;wasm32-emscripten 上改写 `--js-library` 的文件,程序重新链接。 + +## 7. 条件行的来源(#954) + +`inherit_workspace_layer_rows`(`src/project.cppm`)把 `[workspace.target.]` 的行插入每个成员的条件行之前,行不带来源。 +警告的条件(`src/build/prepare/features.cpp:514`)是“不是根包且声明了 abi”,继承的行因此被当作成员自己的声明。 + +**方案。** 条件行携带来源(陈述它的清单与层:成员、工作区层),与平台字段的 `origin` 同一种记录。警告只对来源为该包自身清单的 +`abi` 行发出。工作区层的行是根的陈述,不产生警告。成员自己写的表在它作为依赖到达时照常警告。 +来源同时进入 `mcpp self env` 与机器输出中该值的出处,与其余带来源的值一致。 + +**验证。** e2e(`tests/e2e/workspace/`):虚拟根陈述 `abi`,两个成员,零条警告;成员自带 `abi` 表并被另一成员依赖,一条警告,点名该成员。 +docs/07 写明虚拟根在 `[workspace.target..abi]` 陈述 abi。 + +## 8. 动作:引擎提供事实,步骤的逻辑属于插件(#959、#955) + +### 8.1 现状 + +动作的命令是一个 argv,没有 shell;`${mcpp.*}` 是封闭的词汇(八个记号),只在 argv、输入、输出、环境与工作目录中展开。 +引擎自身提供一条文件命令 `${mcpp.self} stage`。mcpp.plugins 0.25.0 的 `tools-steps` 以一组可移植操作描述多步动作, +后端是 `cmake -P`;0.26.0 在一天之后为 `glob` 增加了 `exclude`,此前已有 #955 的表中没有的 `append` 与 `copy_tree` 的 `if_exists`。 + +宿主工具的通道实测如下(1.1 节末行):`[build-dependencies]` 的边接受 `tools = [...]`(要同时导入其库须写 `host-module = true`), +程序为本机构建并进入全机的工具存储;路径只到达**写出该边的那个包**的构建程序。插件的库在另一个包的构建程序中运行时, +`mcpp::dep_bin` 回答空串。插件因此不能携带自己的程序,这是它们改用 `cmake -P`、shell 脚本与 JDK 的原因。 + +### 8.2 `${mcpp.objects:}`(采纳,D2) + +链接单元由哪些对象组成只有引擎知道;规则在事后遍历 `obj/` 并按路径排除,是在猜测引擎的事实。可对照的做法是 CMake 的 +`$`。 + +| 项 | 规定 | +|---|---| +| 名字的解析 | 与 `${mcpp.target_file:}` 相同,未知与有歧义的名字同样拒绝 | +| 展开为 | 构建目录中一个清单文件的绝对路径(正斜杠)。引擎在写出规划时写它,字节不变时不写 | +| 内容 | 该单元的链接边作为显式输入的对象文件(静态并入的依赖的对象与标准库模块的对象在内,库与框架不在内),按链接顺序,每行一个绝对路径,UTF-8,`\n` 结尾 | +| 依赖 | 清单与其中每个对象是该动作的隐式输入 | +| 角色 | `artifact` 与 `check`。`source`、`prepare`、`object` 中使用时拒绝并写出角色:它们先于或进入清单所依据的编译与链接 | +| 第二种拼写 | `${mcpp.objects_rsp:}`:同一清单按响应文件的引号规则写出,供以 `@file` 读取的工具(`ar`、`llvm-ar`、`lib.exe`)。按行读取的工具(Apple `libtool -filelist`)用前一种。两种格式不能合一:构建目录的路径含空格时(Windows 的用户目录常见),按行的清单不能被 `@file` 正确切分,带引号的清单不能被按行读取 | +| 协议 | 协议 20。`import mcpp;` 提供 `mcpp::objects_of(name)` 与 `mcpp::objects_rsp_of(name)` 返回记号文本,使旧引擎上的构建程序在编译时失败,而不是把未展开的记号交给工具 | + +### 8.3 `${mcpp.self} write`(采纳,D2)与这一族命令的判据 + +`${mcpp.self} write --output [--append ]... -- ...` 写出各行与每个 `--append` 文件的内容,写在别处后改名, +字节不变时不动输出。`--append` 的文件由动作声明为输入。 + +引擎自身的命令以一条判据为界,而不是逐个列举:**一个输出,内容完全由声明的输入与 argv 决定,重复执行结果相同**。 +`stage`(放置一个文件)与 `write`(写出规划已知的文本)满足;删除、建目录、通配、按顺序运行多个程序不满足,它们属于步骤程序(8.4)。 +判据写入 SPEC-007,此后对这一族的增补以它裁决。 + +### 8.4 多步动作的运行器归插件(#955 不按原样采纳,D1) + +#955 请求引擎解释一组操作。结论是运行器为插件包自己的宿主程序,依据见 15.1。引擎欠的是 8.1 实测出的那条通道: +**插件携带它自己的程序**。方案是让 `tools` 这个键出现在特性上,含义不变: + +```toml +# mcpp.plugins 的清单 +[features.tools-steps] +sources = ["tools/steps.cppm"] +tools = ["steps"] # 本包的一个 kind = "bin" 目标;特性激活时为构建机器构建 +``` + +| 项 | 规定 | +|---|---| +| 含义 | 一条边激活带 `tools` 的特性,效果等同于该边写了 `tools = ["steps"]`:程序为本机构建,进入全机的工具存储(键为源码、宿主工具链、特性与依赖闭包) | +| 可见范围 | 图中链接了该包的宿主模块且该特性激活的**每个**构建程序,都得到 `MCPP_DEP__BIN_`;经框架到达的应用不需要在自己的清单中写出工具名 | +| 未声明的工具 | `mcpp::dep_bin` 对没有任何边或特性声明的工具不再回答空串:构建程序以 `build-program/undeclared-tool` 报告,写出包、工具名与应当写的那一行 | +| 动作 | 工具是使用它的动作的输入,字节变化时动作重新运行(docs/30 已有的规则) | +| 位置 | 只在 `[features.]`。没有特性的包以 `default` 所含的特性陈述。`tools` 一个键、一个含义、两个位置(边与特性),符合 AGENTS.md 第 3 节“一个概念一个拼写” | + +写入 SPEC-007 与 docs/30、docs/31。**插件一侧(mcpp-plugins,不属于引擎的改动):** `tools-steps` 的后端改为 `steps <步骤文件>`; +`dist-apple` 的 `#!/bin/sh` 动作改用步骤;成员表中 `"xim:cmake"` 只留在以 CMake 为工具的成员上。 + +### 8.5 验证 + +e2e:一个动作以 `${mcpp.objects:app}` 归档程序的对象,修改一个源文件后归档重新生成,未修改时不运行;构建目录的路径含空格时 +`objects_rsp` 被 `llvm-ar @file` 正确读取;`${mcpp.self} write` 的输出含展开后的绝对路径,重复构建不改写文件; +复现脚本的 955 一项在特性声明 `tools` 后两处都得到路径,Windows 宿主在内;未声明的工具得到诊断。 + +## 9. 进程启动的大小界限(#958) + +### 9.1 现状 + +- openkal-linux `src/process.cpp:33` 与 openkal-macos `src/process.cpp:35`:`kMaxEntries = 512`,参数向量与环境各自超过即返回 + `kal_err_no_memory`。两个向量都按实测总量分配,该常量不保护任何定长缓冲。openkal-windows 的 `process.cpp` 没有该常量。 +- openkal 规范 §7.6 规定实现“原样传递向量”,没有授权条数的界限。内核报告的 `E2BIG` 在 openkal-linux 的转换表中落入默认分支 + `kal_err_io`。 +- **影响面。** 界限同时约束参数向量:`[build] defines` 写 520 项的工程在终端中即不能编译(1.1 节)。依赖多的工程的包含目录与宏定义 + 合计达到这个数量是正常情况,所以它不限于报告中的 Xcode 环境(546 个变量)。消息为内存不足,不指向原因。 + +### 9.2 方案 + +1. **openkal 规范。** §7.6 增加:实现不得对向量与环境的条数或总量设置低于其环境的界限;环境因大小拒绝的启动报告为 + `kal_err_no_space`。一致性套件(`tests/cts`)增加三例:2,000 个参数的启动成功;2,000 个环境变量、约 200 KiB 的启动成功; + 超过环境界限的启动报告 `kal_err_no_space`(Linux 以单个超过 128 KiB 的字符串构造,macOS 以总量超过 `ARG_MAX` 构造)。 +2. **实现。** openkal-linux 与 openkal-macos 删除 `kMaxEntries`,`E2BIG` 转换为 `kal_err_no_space`;openkal-windows 把命令行与环境块的 + 长度拒绝转换为同一值;其余实现以套件核对。 +3. **mcpplibs.os。** 启动失败的错误带上两个向量的条数与字节数。 +4. **mcpp。** 启动失败的消息写出它们:`cannot start '<程序>': the request is larger than the system admits (N arguments, A bytes; M variables, E bytes)`, + 原因标识 `spawn-too-large`(docs/50)。e2e:复现脚本的 958 一项构建成功。 + +mcpp 是否应当为编译器裁剪环境是另一个问题(“默认封闭”一条),本文不处理。 + +## 10. 编译运行状态:观察的回答与恢复(#952,#953 的两项) + +### 10.1 现状 + +- **单元与引用。** Linux 上一个单元是一个进程组,引用为 `L2;<组>;<出生>;;` + (openkal-linux `src/process.cpp:2025` 起)。引用的来源与调用者不同时,`kal_process_job_open` 返回 `kal_err_not_supported`, + `kal_process_job_observe` 回答 `UNKNOWN`;组号现在指向另一个出生时刻的进程时也回答 `UNKNOWN`。 +- **引擎。** `observed()`(`src/build/schedule/compiler_transaction.cppm:305`)把打开失败与 `UNKNOWN` 合并为 `unknown`; + `recover_generation`(623)对未决的尝试等待 5 s 后以 `previous compiler unit is occupied or unknown` 拒绝。 + 调用它之前,`RunScope::begin`(`detach_codegen.cppm:655`)已经取得旧一代的 `owner.lock` 并确认各 actor 的锁空闲; + 这些锁在 Linux 上是打开文件描述的锁(`F_OFD_SETLK`),属于文件系统而不属于进程空间。 +- **状态的位置。** `<工作目录>/.mcpp/compiler-runs/<输出目录的散列>/`,有意置于 `target/` 之外,使锁不随清理消失; + `mcpp clean` 因此不触及它。 +- **实测。** 在一个 pid 命名空间中被结束的构建,从另一个命名空间永远不能继续,`mcpp clean` 之后仍然如此;连续两个容器作业的情形 + 四次均拒绝(1.1 节)。容器与沙箱每次启动都是新的命名空间,CI 的容器作业复用工作目录、开发容器重建、代理的沙箱会话都在其中。 +- **命名空间的名字不是它的身份。** 引用中的进程空间取自 `/proc/self/ns/pid` 的链接文本(`pid:[<编号>]`),而内核把结束的命名空间的编号 + 给后来的命名空间(实测:四个不同的命名空间同名)。名字相同时实现认为是同一来源,并按该编号下的进程组回答, + 回答的是另一个空间中无关的进程。 + #952 的现场(进程全部不存在而失败持续一天)与此相符;是否为同一条路径没有确立,首次中断的原因在报告中未知。 +- **准入的拒绝。** 所有者对六种条件都回答 `REFUSED`(`serve()`,485–513),控制方把任何拒绝视为最终结果并输出 + `compiler-run owner refused unit admission`。 +- **无声的返回。** `run_capped`(`detach_codegen.cppm:1158`)中 `owned_unit::create()` 失败、`admitted` 记录写入失败等路径返回 1 或 2 而不输出。 + Termux 应用沙箱中 `__compiler-cap` 无消息失败与此一致;怀疑的原因是读取 `boot_id` 被拒绝,使单元不能声明,未在沙箱中测量。 + 同一函数已有不声明单元、只受内核许可约束的分支(所有者不受管理时);#953 的探针测得在一次构建之外直接调用 + `__compiler-cap` 在该沙箱中成功。 + +### 10.2 原因 + +观察有三种事实:单元为空、单元被占用、从此处不可能观察。模型只有“空”与“未知”两种出路,而“不可能观察”不会随等待改变。 +另有两种情形当前回答未知,而它们可以证明为空;进程空间的标识不精确,使“是否同一来源”本身可能答错。 +能否声明单元是环境的性质,当前在每次编译失败时才被发现。 + +### 10.3 方案 + +**A. openkal 规范(可观察单元一节)与各实现。** + +| 情形 | 回答 | 依据 | +|---|---|---| +| 引用产生于本机的上一次启动 | `EMPTY` | 上一次启动的进程都已结束 | +| 标识单元的进程号现在属于另一个出生时刻的进程 | `EMPTY` | POSIX:存在以某进程号为组号的进程组时,该进程号不被重新分配 | +| 引用产生于同一次启动的另一个进程空间 | `kal_process_job_open` 返回 `kal_err_not_supported`(现状),规范写明这是对该调用者不变的回答 | 调用者无从枚举另一空间的进程 | +| 进程空间的标识 | 规范:引用**必须**精确标识其进程空间,实现**不得**以另一空间的进程回答。openkal-linux 的引用在命名空间的编号之外记入该空间 1 号进程的出生时刻(格式 `L3`;`L2` 的引用按不可观察处理) | 编号会被复用(实测);同一次启动内编号与 1 号进程的出生时刻不会同时相同 | +| 能否声明单元 | `KAL_PROCESS_PROP_JOB` 只在实现能够产生引用的环境中声明(启动标识、进程空间与出生时刻均可读) | 条款 6.1:能力的缺失是使用者得知的事实 | +| 其余不能判定的情形 | `UNKNOWN`,含义限定为暂时 | — | + +**B. mcpplibs.os。** `unit_observer::open` 的错误区分“不可观察”(`not_supported`)与其余失败;提供对 `KAL_PROCESS_PROP_JOB` 的查询。 + +**C. 引擎的恢复(D3)。** 未退役的尝试有三种结果: + +- 空:恢复,与现状相同。 +- 被占用或暂时未知:在界限内等待,之后拒绝。 +- 不可观察:不等待。此时旧一代的所有者锁与全部 actor 锁已经空闲,这是跨进程空间仍然成立的证据: + 该代的控制方、辅助进程与状态写入者在任何共享此目录的进程空间中都已结束。该尝试按被放弃的尝试处理,未完成的输出走同一条作废路径, + 并经 `diag::degraded("compiler-run/unobservable")` 报告一次,写出代的目录与影响:在那个进程空间中比其控制方活得更久的编译器后代不会被发现。 + 其中任一把锁被持有时照常拒绝,消息写明另一个进程空间中的 mcpp 正在构建该配置。 + +残留的情形是:另一个仍在运行的进程空间中,控制方被强行结束而编译器后代仍在写同一个输出。当次构建中该文件可能有两个写入者; +此后的后果有界:迟到的写入使输出的身份与退役证书不符,下一次构建按已有的证书检查作废并重编该单元(依据 `cached_result` 的实现)。 + +**D. 清理。** `mcpp clean`(含 `--stale`、`--older-than`)在移除一个配置的输出目录时,在构建所用的同一把闸锁之下退役该配置的各代 +(`run-*`),锁文件保留;所有者或 actor 的锁被持有时该配置不清理,消息写出正在进行的构建。输出已被移除,旧一代没有可保护的对象, +退役不需要观察。 + +**E. 拒绝的消息。** 写出代的目录、未决尝试的个数、每种状态的个数与出路: + +``` +error: 3 compiler units of an interrupted build of this configuration may still be running (2 occupied, 1 unknown) + state: <工作目录>/.mcpp/compiler-runs/<散列>/run-<…> + When no compiler of this configuration is running: mcpp clean, then build again. +``` + +原因标识 `compiler-run-occupied`。 + +**F. 准入。** 所有者的回答带上条件:`REFUSED <标识>`,标识取自 `outside-generation`、`record-unreadable`、`actor-outside`、 +`actor-not-held`、`controller-ended`、`unit-unobservable`、`owner-stopping`;控制方把它写进消息。`record-unreadable` +在启动宽限内重试,与未得到回答的登记相同;其余为最终结果。所有者与控制方是同一次构建中的同一个程序,没有版本间的兼容问题。 + +**G. 返回路径。** 引擎内部的辅助命令(`__compiler-cap`、`__action`、`stage`、`write`)以非零状态结束时必有一行写出步骤与原因; +130(被取消)除外。由事务套件的故障矩阵断言:每个注入的故障对应的 stderr 非空且含该步骤的名字。 + +**H. 单元不能声明的环境(D4)。** 一次构建在开始时查询该性质一次。环境不声明它时,整次构建的编译走已有的不声明单元的分支, +经 `diag::degraded("compiler-run/no-unit")` 报告一次,影响同 C;声明了而 `owned_unit::create()` 失败是错误,拒绝并写出原因。 +这样行为由环境的性质决定,不由某一次调用的失败决定。Termux 的原因在 G 落地后由 `ci-fresh-install` 的输出直接给出; +是否确为 `boot_id` 在此之前不下结论,方案不依赖它。 + +### 10.4 两个不采纳的做法 + +- **#952 的第 1 项**(控制方与单元的进程都不存在时视为已放弃):组长不存在不能证明没有仍在写入的后代,这是 2026-11-06 的记录中明确拒绝的推断。 + A 给出了可以证明的两种情形,C 给出了不可观察时的规定行为,二者覆盖该请求的现场。 +- **以继承的文件锁作为单元的见证**(辅助进程持有一把锁并让编译器的进程树继承其描述,锁空闲即单元为空,与进程空间、启动无关): + 它在任何命名空间都可判定,但编译器启动的常驻进程会继承该描述并一直持有(`launcher` 为 sccache 时的服务进程、Windows 的 `mspdbsrv`), + 单元永远显示为被占用;进程组的做法在这一点上没有问题。 + +### 10.5 验证 + +- openkal 的套件:上一次启动的引用(以改写引用中的启动标识构造)回答 `EMPTY`;另一个进程空间的引用,`open` 返回 `not_supported`; + 与当前空间同名而 1 号进程出生时刻不同的引用,`open` 返回 `not_supported`;不能产生引用的环境不声明该性质。 +- e2e(Linux,要求可创建 pid 命名空间,否则 `SKIP`):复现脚本的 952 与 952s 两项改为断言:继续的构建成功并输出一次 `degraded`; + 旧一代的 actor 锁被持有时拒绝;`mcpp clean` 之后 `.mcpp/compiler-runs/<散列>/` 下没有 `run-*`。 +- 事务套件:G 的故障矩阵;F 的每种条件各一例;H 以注入 `not_supported` 构造。 + +### 10.6 产物的事务化发布(N1) + +10.3 的 A 至 H 修正的是观察的模型。需要观察的原因在更下面一层:**编译器直接写已发布的路径**。对象以 `-o $out` 写出; +GCC 的 BMI 由编译器自己写 `<名>.gcm~` 并改名到位,clang 的 BMI 直接写到最终路径(`detach_codegen.cppm` 开头的记录)。 +GCC 的策略又有意让代码生成在 ninja 的边结束之后继续。因此一个比控制方活得更久的编译器能够改写已发布的产物, +“被占用则拒绝”与 10.3 C 的残留情形都由此而来。 + +**做法。** 编译器只写该次尝试私有的路径,发布由引擎的辅助进程以改名完成:BMI 在它完整时,对象、依赖文件与状态在编译器结束时。 +残留的编译器只能写它自己的私有文件。恢复即丢弃未发布的尝试,不需要以观察保护输出;单元只用于取消时结束进程与并发上限的记账, +在那里答错的后果是多占一个并发名额。 + +**已有的基础。** 每个单元的 BMI 路径已由模块映射给出(`ModuleScope`,`src/build/plan.cppm:399`:GCC 读 `-fmodule-mapper=`, +clang 与 MSVC 读参数文件);辅助进程持有编译器的 argv;对象截断已在执行器中把旧对象改名保留再比较。 + +**前提,在该检查点内对每个编译器测量。** + +| 前提 | 内容 | +|---|---| +| P1 | GCC 16.1 以该次尝试的映射把自己的 BMI 写到私有路径,辅助进程发布后导入者读取;mcpp 自身的构建与测试通过,冷构建的总时间与当前之差在重复测量的波动之内 | +| P2 | clang 的 `-fmodule-output=` 与 `-o`、MSVC 的 `/ifcOutput` 与 `/Fo` 指向私有路径,结果相同 | +| P3 | 带着输出路径的附属产物逐个核对:覆盖率记录、分离的调试信息、PDB、时间跟踪。列出 mcpp 实际产生的那些,各自或随对象一同发布,或其内嵌的路径以编译器的映射选项写为最终路径 | + +**前提不成立的编译器**保持就地写入并按 10.3 C 处理;“产物可重定向”是工具链模型中编译器的一个事实,引擎的规则只读取它。 +**成立的编译器**上,10.3 C 的报告与残留情形、E 的拒绝不再出现;D、F、G、H 仍然需要。 + +## 11. #953 中的两项 + +### 11.1 同一输出路径的两种配置(D5) + +一个配置的构建目录由它的谱系命名(编译器家族、目标、标准库、C++ 标准、旗标、基座),特性不在其中。docs/07 规定: +一次命令内,多个成员共用的包按该次选择请求的特性的并集编译一次;并且写明测试的规划会因开发依赖改变一个包的特性。 +因此同一个目录中,同一个对象路径在不同的命令下对应不同的命令行: + +| 触发 | 例 | +|---|---| +| 构建与测试交替 | 开发依赖为某个共用的包多激活一个特性:`mcpp test` 之后的 `mcpp build` 把它编回去 | +| 选择交替 | `-p X`、`-p Y` 与 `--workspace` 对同一个包给出不同的并集 | + +31 个成员的工作区上测得两个成员之间 43 条这样的边(2026-11-17 记录 10.6 节)。产物是正确的(ninja 按命令行重建),重复的是工作, +并且经模块的 BMI 传到导入者。 + +**方案。** 一个包实例的对象目录由它的配置命名:`obj/<包>@<摘要>/`。摘要覆盖决定其命令的因素:该包激活的特性与宏、 +作为根的成员对它的声明,以及它所依赖的各包的摘要(导入者的命令含被导入包的 BMI 路径,所以摘要必须传递)。 +摘要与谱系一样只含配置,不含版本与编译器的构建号:升级不改变路径。同一模块名的两个实例的 BMI 由已有的模块映射区分 +(每个单元的映射给出它所见的 BMI 路径,GCC 的映射不回落到按名字的默认位置),不需要新的机制。两种配置并存,交替的命令不再重新编译; +`mcpp clean --stale` 按构建记录回收不再被引用的实例目录,`mcpp clean --report` 按实例列出体积。 + +**验收的测量。** 实施前在同一工作区上记录 `mcpp build` 与 `mcpp test` 交替三轮、以及 `-p X` 与 `-p Y` 交替三轮时重新运行的边数, +实施后同样测量;预期第二轮起为零。磁盘的增量以 `clean --report` 给出。 + +### 11.2 Windows 的 LLVM 载荷不读压缩的调试段 + +Windows 的 LLVM 载荷中的 lld 构建时未带 zlib,不能读取带 `ELFCOMPRESS_ZLIB` 段的对象;FreeBSD 基座以解压其链接输入绕过(修订 2), +用户自己的对象仍不可读。mcpp 自身的 dev 构建在 Linux 上以 `-gz=zlib` 编译(实测的命令行),所以这类对象不是少数情形。 +规则:一个工具链载荷在每个宿主上读取同样的输入。xim-pkgindex 的 LLVM 载荷在 Windows 上以 zlib 与 zstd 构建 lld, +载荷的 CI 在每个宿主上链接一个带 zlib 压缩段与一个带 zstd 压缩段的对象。之后 e2e 1116 在 Windows 宿主上不再跳过, +FreeBSD 基座包恢复上游的字节(修订 3,撤销修订 2 对链接输入的解压)。 + +## 12. 拒绝与失败的可诊断性 + +AGENTS.md 第 2 节已规定:拒绝写出原因与出路并带原因标识,少做不得无声。本轮的违反之处及其归属: + +| 处 | 违反 | 归属 | +|---|---|---| +| 启动失败报内存不足 | 原因错误 | 9 | +| `occupied or unknown` 不写目录与出路 | 无对象、无出路 | 10.3 E | +| 准入被拒,六种条件一条消息 | 原因不可分 | 10.3 F | +| `run_capped` 无声返回 | 少做而无声 | 10.3 G | +| `dep_bin` 对未声明的工具回答空串 | 少做而无声 | 8.4 | +| 模板包以描述符的形式为由被拒 | 写的不是缺少的东西 | 3.2 | +| 版本组冲突只写一个请求者 | 原因不全 | 4.2 R1 | + +间歇失败(e2e 1033 的 ok 映像线程在 CI 上三次以状态 137 超时,986 的一次 `SIGSEGV`)的共同点是第一次出现时没有留下状态。 +补一条规定:**超时先请求结束,再终止。** mtest 在超时时向测试程序发送结束请求并等待一段宽限,记录进程树与各进程的状态,然后终止; +okrun 收到结束请求时输出映像中每个任务正在等待的 openkal 调用。两个 issue 本身不在本文中解决:没有可据以设计的原因。 + +## 13. 规范、文档与测试的落点 + +| 项 | 规范 | 用户文档(中英文) | 测试 | +|---|---|---|---| +| 3 | SPEC-001 新节 | docs/11 | 单元往返;e2e 发布、消费、模板 | +| 4 | SPEC-004 解析轴;SPEC-013 §2、§7.5 | docs/35、docs/05 | e2e 本地索引五例 | +| 5 | SPEC-016 §6 | docs/04 | 单元谓词;e2e 夹具运行器;`check_system_names.py` 的词表与基线 | +| 6 | SPEC-004 §8(两条规则) | docs/30、docs/04 | e2e 三种写法、搜索到的库、emscripten 一例 | +| 7 | SPEC-004 来源 | docs/07 | e2e 工作区两例 | +| 8 | SPEC-007(记号、命令族的判据、特性的 `tools`) | docs/30、docs/31 | e2e 五例 | +| 9 | openkal SPEC §7.6 | docs/50(原因标识) | openkal 套件三例;e2e 一例 | +| 10 | openkal SPEC 可观察单元(回答、进程空间的标识、性质);SPEC-011 | docs/09(清理)、docs/50 | openkal 套件;e2e 命名空间两例;事务套件 | +| 11.1 | SPEC-018(构建目录的记录) | docs/07、docs/09 | e2e 两种交替 | + +每一项的 CHANGELOG 条目与规范条款的实现状态在同一个拉取请求中更新。 + +## 14. 各仓库的改动、拉取请求与检查点 + +### 14.1 原则 + +- **每个仓库一个拉取请求。** 拉取请求内的提交是检查点:每个提交自身可构建并通过它引入的测试,提交信息陈述该检查点的内容; + 规范条款、中英文文档与测试随它所属的检查点提交(AGENTS.md 第 1 节)。 +- **依赖而不复制。** 下游的拉取请求在上游发布之前以 `git` 依赖指向上游拉取请求的分支,最后一个检查点改为版本依赖。 +- **前提在拉取请求内测量。** 只能在真实 runner 上得到的事实(macOS 与 Windows 的单元回答、Termux 的性质、Windows 的 lld) + 由相应拉取请求自身的 CI 测量,结果追加到本文的状态行;不另开探针拉取请求。 +- **合并与发布按依赖顺序。** 下表的序号相同的仓库彼此独立。 + +### 14.2 仓库、拉取请求与检查点 + +| 序 | 仓库 | 检查点(按顺序) | +|---|---|---| +| 1 | openkal(0.26.0) | ① §7.6 的大小条款与套件三例 ② 可观察单元:回答、进程空间的标识、性质,套件四例 ③ 版本与变更记录 | +| 2 | openkal-linux | ① 删除 `kMaxEntries`,`E2BIG` 转换为 `kal_err_no_space` ② 引用 `L3`;上一次启动、进程号被重新分配回答 `EMPTY` ③ 性质按环境声明 ④ 规范依赖 `^0.26.0` 与版本 | +| 2 | openkal-macos | ① 删除界限与转换 ② 单元的回答与进程空间的标识 ③ 版本(未列入索引的 0.23.5 的改动随它列入) | +| 2 | openkal-windows | ① 命令行与环境块的长度拒绝的转换 ② 上一次启动的回答 ③ 版本 | +| 2 | 其余实现(openkal-freebsd、-uefi、-dos、-svr4、-emscripten),各一个 | ① 套件的新用例所要求的改动(没有进程接口的实现没有) ② 版本 | +| 3 | openkal-musl、openkal-llvm-runtime,各一个 | 精确钉住的级联与版本 | +| 3 | openkal-image(`okld`、`okrun`、工具) | ① okrun 收到结束请求时输出每个任务正在等待的调用(12) ② 级联与版本 | +| 4 | mcpplibs.os | ① 启动失败带两个向量的条数与字节数 ② 不可观察的区分、性质的查询 ③ 成员的范围、引擎下界与版本 | +| 4 | mtest | ① 超时先请求结束、宽限、记录进程树(12) ② 版本 | +| 5 | openkal-platform | ① 版本组点名新的成员 ② 版本 | +| 6 | mcpp | 见下 | +| 7 | mcpp-plugins | ① `steps` 程序与 `tools-steps` 的后端 ② `dist-apple` 的动作改用步骤 ③ 成员表中移除不以 CMake 为工具的 `xim:cmake` ④ 文档与版本,引擎下界为第 6 项 | +| 7 | xim-pkgindex | ① Windows 的 LLVM 载荷以 zlib 与 zstd 构建 lld,载荷 CI 的两例 ② FreeBSD 基座修订 3 | + +mcpp 的拉取请求的检查点: + +| 检查点 | 内容 | 节 | +|---|---|---| +| 1 | 本设计记录与复现脚本 | — | +| 2 | 描述符:一个读取函数、指针形式的写出、往返测试 | 3 | +| 3 | 解析:版本组的约束进入合并,build-only 包按宿主行求值,锁 | 4 | +| 4 | 可达性的单一谓词;系统名检查的词表与基线 | 5 | +| 5 | 链接的输入:链接器的报告与标志的补全 | 6 | +| 6 | 条件行的来源 | 7 | +| 7 | `${mcpp.objects:}`、`${mcpp.objects_rsp:}`、`${mcpp.self} write`,协议 20 | 8.2、8.3 | +| 8 | 特性上的 `tools`;未声明工具的诊断 | 8.4 | +| 9 | 包实例的对象目录,含修正前后的测量 | 11.1 | +| 10 | 编译运行状态:恢复、清理、准入的条件、返回路径 | 10.3 C–G | +| 10b | 产物的事务化发布:先测量 P1 至 P3,再实现(N1 采纳时) | 10.6 | +| 11 | 新的 mcpplibs.os:启动失败的消息、不可观察的区分、性质 | 9.2 之 4,10.3 B、H | +| 12 | 发布:版本、CHANGELOG、默认平台、依赖改为版本依赖(`mcpp.toml` 与 `version.cppm` 在这一个提交中) | — | + +检查点 2 至 10b 不依赖新的 openkal,先于上游的发布完成;11 与 12 在第 1 至 5 项发布之后。 + +**mcpp-index 是这一原则的结构性例外。** 索引的条目随被列出的发布产生:家族、mcpplibs.os、mtest 与平台的条目须在 mcpp 的检查点 12 +之前列出(发布构建以版本依赖取它们),mcpp.plugins 的条目须在 mcpp 发布之后(它的引擎下界)。两批不能合为一个拉取请求; +xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生,不计入。 + +**版本(D7)。** openkal 规范第 8 条规定:每个实现都必须满足的要求推进次版本。两处条款都是这样的要求,所以是 0.26.0。实现随之进入新的次版本, +不在 mcpplibs.os 0.4.4 的范围(`<0.26.0`、`<0.27.0`、`<0.24.0`)之内:已发布的引擎看不到这些版本,4.1 的失败不会因本轮的发布在它们上出现。 + +### 14.3 本轮撤销的变通 + +与这十个 issue 相关、当前存在于生态中的变通,逐项对应消除它的章节。一项变通在本轮之后仍被需要,即说明对应的章节没有从成因解决。 + +| 变通 | 所在 | 消除它的章节 | 撤销 | +|---|---|---|---| +| 成员的补丁不列入索引(mcpp-index#831 的回退,openkal-platform#37 的搁置) | mcpp-index、openkal-platform | 4 | 规则撤销 | +| 以 tag 加 `override = true` 取版本组的成员 | 工程 | 4 | 可删除 | +| `run_under = ["env"]` | 工程 | 5 | 可删除 | +| 手工删除已链接的程序以取得新的链接输入 | 工程 | 6 | 不再需要 | +| 以裁剪后的环境启动 mcpp | 工程 | 9 | 可删除 | +| 手工删除 `.mcpp/compiler-runs/<散列>/` 与配置目录 | 工程、CI | 10 | 不再需要 | +| 把 `mcpp publish` 的描述符手工改为字符串形式 | 发布方 | 3 | 不再需要 | +| `cmake -P` 后端、`#!/bin/sh` 动作、JDK 上的辅助程序 | mcpp-plugins | 8.4 | 14.2 第 7 项 | +| 规则在事后遍历 `obj/` 并按路径排除;以 shell 的 `printf` 写出含绝对路径的文件 | 规则包 | 8.2、8.3 | 可改用记号 | +| FreeBSD 基座修订 2 解压其链接输入 | xim-pkgindex | 11.2 | 修订 3 | +| e2e 1116 在 Windows 宿主上跳过 | mcpp | 11.2 | 不再跳过 | + +工程与规则包一侧的各项由其仓库在升级后删除并以自己的 CI 核对,不属于 14.2 的拉取请求。 + +## 15. 七项决定的结论与依据 + +每项列出相关的视角、两种做法在该视角下的情况与结论。视角取自:mcpp 的设计规范、openkal 的规范、通用架构、简洁、语义与一致性、 +跨平台、稳定与兼容、开发体验、扩展与演进、行业做法。 + +### 15.1 D1 多步动作的运行器:插件自己的宿主程序 + +| 视角 | 插件的宿主程序(采纳) | 引擎解释步骤词汇 | +|---|---|---| +| mcpp 规范 | 引擎增加的是机制(特性携带工具),步骤的词汇留在数据与插件中 | 引擎持有一种会随工具变化的词汇,违反“机制在引擎” | +| 扩展与演进 | 词汇随插件的版本演进 | issue 提出后一天词汇已变(`exclude`);每次变化对应引擎发布、协议号与每个使用者的引擎下界 | +| 简洁 | 一个已有的键多一个位置 | 十余个操作、列表变量、通配的语义各需规范条款 | +| 语义与一致性 | 构建逻辑只有一种语言:宿主上的 C++ 程序 | 引擎内出现第二种带变量与条件的语言 | +| 跨平台 | 标准库与 mcpplibs.os,每个宿主相同;没有 shell | 同样可移植 | +| 开发体验 | 每台机器、每个插件版本与宿主工具链构建一次,进入全机存储;不再安装 `xim:cmake` | 零构建成本 | +| 行业做法 | Bazel 与 Buck 的动作运行在执行配置下构建出的工具;Cargo 的 `build.rs` 是编译出的宿主程序 | CMake 的 `-P` 是这种做法:脚本语言由构建工具自身携带,其词汇随工具的版本冻结 | +| 通用性 | 通道对所有带辅助程序的规则包成立(代码生成、反射、打包) | 只解决步骤 | + +结论:引擎解释词汇只在首次构建的几秒上占优,其余视角相反。并且 8.1 的实测表明通道的缺口是插件改用外部工具的原因, +补上它同时消除三种变通(shell 脚本、JDK 上的 Java 程序、`cmake -P`)。 + +### 15.2 D2 `${mcpp.objects:}` 两种拼写与引擎内的 `write` + +| 视角 | 采纳的做法 | 另一种做法 | +|---|---|---| +| mcpp 规范 | 对象清单是引擎的事实,以封闭词汇中的一个记号给出 | 规则遍历 `obj/` 并排除:猜测引擎的事实,增量依赖也不成立 | +| 语义 | `write` 是纯边:一个输出、内容由输入与 argv 决定、可重复,与 `stage` 同类 | 交给步骤程序:规则包为写两行文本要带上运行器,且规划已知的内容被放进一个带副作用的步骤 | +| 稳定与兼容 | 命令族以判据为界,判据写入规范;此后不逐个争论 | 不设判据地增加命令,会逐步回到 15.1 被否掉的做法 | +| 跨平台 | 两种拼写分别对应按行读取与 `@file` 读取;构建目录含空格时二者不可合一 | 单一拼写在含空格的路径上对一类工具失效 | +| 行业做法 | CMake `$`、`file(GENERATE)`;Bazel `ctx.actions.write`、`use_param_file` | — | + +结论:两个记号与 `write` 都采纳。 + +### 15.3 D3 不可观察的单元:锁空闲时恢复并报告 + +| 视角 | 恢复并报告(采纳) | 保持拒绝,只改消息 | +|---|---|---| +| 语义 | “不可观察”是不变的事实,等待它没有意义;按可得的最强证据(锁)决定 | 把不变的回答当作暂时的 | +| 稳定性 | 容器中被中断的构建在下一个容器中继续 | 每次都要手工清理;CI 的容器作业复用工作目录时每次失败(实测) | +| 安全 | 残留情形见 10.3 C:当次构建可能有两个写入者,此后由退役证书发现 | 没有残留情形 | +| mcpp 规范 | 少做经 `diag::degraded()` 报告,写出影响 | 符合,但把通常已经结束的构建当作仍在运行 | +| openkal 规范 | 条款 6.1:能力的缺失是使用者得知的事实,使用者据此行动 | — | +| 行业做法 | Cargo、Bazel、ninja 以文件锁判断上一次构建已结束;mcpp 在可观察时多一层,不可观察时与它们相同 | 没有工具在这种情形下拒绝 | + +结论:恢复并报告。以继承的锁作为见证的做法另行评估后否掉(10.4)。它是就地写入之下的规定行为;就地写入本身由 10.6 处理。 + +### 15.4 D4 单元不能声明的环境:降级并报告 + +| 视角 | 降级并报告(采纳) | 拒绝 | +|---|---|---| +| 通用性 | mcpp 在应用沙箱一类的环境中可用 | Termux 应用沙箱自 2026.11.7.1 起不能构建 | +| 简洁 | 走 `run_capped` 中已有的分支,不新增路径;由环境的性质在构建开始时决定一次 | — | +| 语义与一致性 | 与 D3 同一条规定:观察能力缺失时按许可与锁运行,并写明 | 两处行为不一致 | +| 安全 | 失去的是对比控制方活得更久的编译器后代的发现,即引入单元之前的行为 | 没有损失,也没有构建 | + +结论:降级并报告;依据是环境不声明 `KAL_PROCESS_PROP_JOB`,声明了而失败照常拒绝。 + +### 15.5 D5 同一路径的两种配置:配置计入对象目录 + +| 视角 | 实例目录带配置摘要(采纳) | 全工作区取并集 | +|---|---|---| +| 语义 | 不带测试的构建不被测试的依赖配置;成员单独构建的结果不随无关成员变化 | 为消除“构建与测试交替”必须把开发依赖的特性并入普通构建,发布的库随之改变 | +| 一致性 | 目录已由配置的谱系命名,实例目录用同一条原则 | 版本在工作区内是统一的,特性也统一,这一点一致 | +| 稳定与兼容 | 路径方案变化,升级后一次全量重编译;摘要不含版本,此后升级不变 | 路径不变 | +| 开发体验 | 交替的命令不再重编;磁盘随并存的配置增长 | 不重编;`-p X` 的规划须遍历全部成员的图 | +| 简洁 | 对象路径的每个读取方都要改(规划、编译数据库、清理、报告) | 改动小 | +| 行业做法 | Cargo 的产物按单元的元数据散列并存(含特性与依赖的散列),解析器 v2 停止把开发依赖的特性并入普通构建;Bazel 的输出目录按配置散列 | Cargo 的旧解析器;`cargo-hakari` 以人工的方式逼近它 | + +结论:配置计入对象目录。并集的做法在“构建与测试交替”上只能以改变产物为代价,不能采纳。该项的改动面是本方案中最大的;它在本轮实施(17.1 的 U1),验收为 11.1 的测量。 + +### 15.6 D6 发布的描述符:指针形式,不带摘要 + +| 视角 | 指针形式(采纳) | 保留 `modules`、`deps` 并让读取方接受第三种形式 | +|---|---|---| +| 语义与一致性 | 清单是依赖与模块的唯一陈述 | 一个事实两处陈述 | +| 正确性 | — | 描述符的 `deps` 在结构上不能携带目标轴的依赖(mcpp-index 的文档以 huxerui 为例:`emit xpkg` 写出空的 `deps`,36 项闭包手工抄入);它不是清单的忠实摘要 | +| 稳定与兼容 | 已发布的每个 mcpp 都读取 | 需要新引擎才能读取 | +| 读取方 | 引擎之外没有读取者:xlings(`4f5df092`)与 mcpp-index 的工具中均未找到对 `mcpp` 表的读取 | — | +| 生态演进 | 将来若需要不下载即解析,摘要应由索引从已发布的清单生成并带目标轴,而不是由发布方手写三块平台表 | 现在的形状承担不了该用途 | + +结论:指针形式。 + +### 15.7 D7 openkal 的两处条款:规范的次版本 + +| 视角 | 0.26.0 与家族级联(采纳) | 补丁版本 | +|---|---|---| +| openkal 规范 | 第 8 条:每个实现都必须满足的要求推进次版本;两处条款都是 | 违反第 8 条 | +| 稳定与兼容 | 新版本在 mcpplibs.os 0.4.4 的范围之外,已发布的引擎不受影响 | 落在范围之内;已发布的引擎在 hosted 行上因 4.1 的缺口失败,#830 的情形重演 | +| 代价 | 一次家族级联(版本组中十三个包、平台、mcpplibs.os、索引) | 只发两个实现 | +| 生态演进 | 版本号如实表达“实现须重新符合” | 版本号掩盖了要求的变化 | + +结论:次版本。512 的界限若单独看可以作为补丁(§7.6 已要求原样传递),随同一次发布走,不单独发补丁。 + +## 16. 自评 + +### 16.1 第一轮(v1 至 v2):按视角复核 + +| 视角 | 发现 | 调整 | +|---|---|---| +| 语义与一致性 | v1 的 4.2 让目标平台的版本组提前约束 build-only 的请求,与 SPEC-013 §7.5 的条文相反 | 改为约束进入已有的合并;重读代码并实测确认合并丢失了该约束 | +| 生态基础设施 | v1 把 #958 的参数向量一侧写为推断 | 实测:520 个宏定义的工程不能编译;严重程度上调,套件增加参数一例 | +| 通用架构 | v1 的 8.4 假定宿主工具通道不够用,未运行 | 实测:边上的 `tools` 可用,路径不越过声明它的包;`dep_bin` 无声返回空串 | +| 稳定与兼容 | v1 未说明交叉构建时锁不含宿主一侧的包 | 实测并写入 4.1 F2 | +| 语义 | v1 的 11.1 把问题只写成选择交替,摘要未说明须传递 | 补入构建与测试交替;摘要含依赖的摘要;并集的做法因改变产物而否掉 | +| 开发体验 | 第 12 节的清单漏了 `dep_bin` | 补入 | + +### 16.2 第二轮(v2 至 v3):是否从成因解决 + +逐节问一句:这一节消除的是成因,还是这一次的表现。 + +| 节 | v2 的状态 | 判定 | 调整 | +|---|---|---|---| +| 3 | 一个定义、一个读取函数、往返测试 | 成因 | — | +| 4 | 约束进入通用的合并;条件表按构建的行求值 | 成因 | — | +| 5 | 删除特例 | 特例是成因,但同类分支仍可无声进入:现有的系统名检查不计三元组的谓词 | 词表增加三元组的系统谓词 | +| 6 | 补全“点名”的位置 | 表现:这是同一类问题的第三次修补 | 增加规则 B,链接器自己的报告;规则 A 保留给驱动读取的文件与不提供报告的链接器,二者都是长期规则 | +| 7 | 条件行带来源 | 成因 | — | +| 8 | 引擎的事实一个记号;插件携带自己的程序 | 成因 | — | +| 9 | 规范条款与实现 | 成因 | — | +| 10 A | 两种可证明为空的情形 | 不完整:进程空间以名字识别,而名字被复用(实测) | 引用精确标识其进程空间,写入规范;实现记入 1 号进程的出生时刻 | +| 10 H | 调用失败后降级 | 表现:能力的缺失在每次编译失败时才被发现 | 由 `KAL_PROCESS_PROP_JOB` 在构建开始时决定一次;实现只在能产生引用的环境声明它 | +| 10 C | 不可观察时按锁恢复并报告 | 表现:它规定了就地写入之下的行为,而需要观察的原因是编译器直接写已发布的路径 | 增加 10.6,产物的事务化发布,前提在检查点内测量;列为 N1 | +| 11.1 | 配置计入对象目录 | 成因 | — | +| 11.2 | 载荷带 zlib 与 zstd | 成因;但生态中已有的绕过未撤销 | FreeBSD 基座恢复上游的字节 | +| 12 | 消息与测试 | 成因在各节;超时的证据留存是新增的机制 | — | +| 全文 | — | 没有列出本轮之后应当消失的变通 | 14.3:十一项变通逐项对应章节,作为“是否从成因解决”的验收 | + +结合 openkal 规范的复核:版本由第 8 条决定(15.7);大小的条款是对 §7.6“原样传递”的补全;“不可观察”与“不能声明”都落在条款 6.1 +(缺失是使用者得知的事实)之下,前者是 `open` 的不变回答,后者是性质;引用的格式是实现的私有内容,规范只规定它必须精确标识进程空间。 +结合 mcpp 规范的复核:每一项的规则在引擎,事实在数据或规范(链接器是否提供报告、特性携带的工具、行的执行系统); +没有新增按系统名的分支;新增的键(特性上的 `tools`)沿用已有键的名字与含义;每一处少做都经 `diag::degraded()` 或拒绝写出。 + +复核后仍存在的弱点:11.1 的收益没有在目标工作区上按新的两种交替测量(检查点 9 的验收即为此);10.6 的三个前提都未测量, +`detach_codegen.cppm` 与 `compiler_transaction.cppm` 只读了与本文相关的部分;10.3 C 的残留情形只有论证没有构造的用例; +第 5、9 节各有一项只读了源码;规则 B 在 macOS 的系统链接器与 Windows 上不可用。 + +## 17. 已定事项与仍需决定的三项 + +### 17.1 已定 + +| 编号 | 事项 | 结果 | +|---|---|---| +| U1 | 11.1 是否在本轮实施 | 本轮(用户,2026-11-19) | +| U2 | openkal 0.26.0 的家族级联 | 本轮(用户,2026-11-19) | +| U6 | xim-pkgindex 的 LLVM 载荷、mtest 的超时、okrun 的任务状态 | 本轮(用户,2026-11-19) | +| — | 实施方式 | 每个仓库一个拉取请求,提交作为检查点;从成因解决,不留变通(用户,2026-11-19) | +| U3 | #955 不按原样采纳,mcpp-plugins 文档中的相应一句改写 | 按建议;评审时未提出异议即如此 | +| U4 | 10.3 C 的残留情形 | 就地写入的编译器上按建议接受并报告;N1 采纳且前提成立的编译器上不存在 | +| U5 | `mcpp.lock` 的迁移 | 按建议:第一次构建改写锁并输出一行说明;`--locked` 在此之前拒绝并写出原因与 `mcpp update` | +| U7 | `tools` 出现在 `[features.]` | 按建议 | +| U8 | `${mcpp.objects:}`、`${mcpp.objects_rsp:}`、`${mcpp.self} write` 的拼写 | 按建议 | +| U9 | 原因标识与诊断键 | 按建议 | + +### 17.2 仍需决定 + +| 编号 | 事项 | 建议 | 不采纳建议时 | +|---|---|---|---| +| N1 | 产物的事务化发布是否列为本轮的一项(10.6) | 列入,作为 mcpp 拉取请求的检查点 10b:本轮已经改动对象与 BMI 的路径(11.1)与恢复(10.3),同一处不必再开一次;它消除的是需要观察的原因。它关闭的情形没有被观察到,三个前提未测量,这是把它单列为一项决定的原因 | 不列入:10.3 的 A 至 H 独立成立,10.6 整节与检查点 10b 删除,10.3 C 的报告与残留情形保留 | +| N2 | mcpp-index 的两个拉取请求(14.2) | 接受这一结构性例外 | 无可替代的做法:两批条目之间隔着 mcpp 的发布 | +| N3 | 规则 B 是否为 ld64 增加对 `-dependency_info` 的转换(6.2) | 不增加:需要在链接边外再包一层引擎的命令;该链接器上规则 A 生效,尚无规则 A 覆盖不到的报告 | 增加一个把该格式转为 make 格式的引擎内部命令,链接边经它运行 | + +## 18. 未验证事项 + +- #952 的现场是否经由 10.1 的命名空间路径;首次中断的原因。 +- Termux 应用沙箱中不能声明单元的原因(10.3 H 不依赖它)。 +- 平台的层发出的请求其书写时的约束是什么,没有打印;4.1 F1 的依据是合并读取 `originalConstraint` 的源码与实测中 `Merged` 一行的缺失。 +- 11.1 的两种交替在目标工作区上重新运行的边数(检查点 9 的测量即为此);Windows 宿主的 ELF 目标是否同样带 `-gz=zlib`(11.2)。 +- 10.3 C 的残留情形没有构造的用例。 +- 10.6 的 P1 至 P3 未测量;`detach_codegen.cppm`(1,221 行)与 `compiler_transaction.cppm`(651 行)只读了与本文相关的部分, + 事务化发布对恢复逻辑的简化程度是设计上的预期。 +- 规则 B:ld.lld 的依赖文件以手工命令核对(动态列表、版本脚本、链接脚本均被列出),未经 mcpp 的链接边运行;mold 未核对; + ninja 对链接边的 `depfile` 与 mcpp 的快速路径的配合未运行(6.2)。 +- 命名空间编号被复用时,“同名而不同”的两个空间在同一时刻不会并存,这一点依据内核的分配方式,未构造(10.3 A)。 +- Apple 各行的执行系统在行数据中的取值未逐行核对(5)。 +- `mcpp:link-script` 的脚本当前是否为链接的输入(6.3 一并断言)。 +- openkal-macos 与 openkal-windows 的单元引用如何记录启动与进程空间,未读(10.3 A 以条款约束,各实现以套件核对)。 +- 文件锁跨 pid 命名空间可见的结论依据 Linux 本地文件系统上 `F_OFD_SETLK` 的语义;NFS 上未验证(10.3 C)。 +- mtest 0.9.0 的超时是否已先发送结束请求(12)。 +- xlings 与 mcpp-index 的工具不读取描述符中的 `mcpp` 表,依据是对两个仓库的检索,未运行(15.6)。 +- 14.2 中其余实现的仓库(openkal-freebsd、-uefi、-dos、-svr4)是否实现进程接口,未读。 +- #957 未在本机复现,依据为源码与报告中的输出。 diff --git a/.agents/docs/README.md b/.agents/docs/README.md index 366dd4201..d59005332 100644 --- a/.agents/docs/README.md +++ b/.agents/docs/README.md @@ -18,7 +18,7 @@ superseded_by: 2026-09-07-....md # when status is superseded --- ``` -365 records. +366 records. ## By subject @@ -90,6 +90,7 @@ Records that declare one. Everything else is listed by date below. ### ecosystem +- [生态规则收敛方案:2026.11.18.1 之后十个 issue 的五类成因、七项决定与实施范围](2026-11-19-ecosystem-rule-convergence-design.md) — active - [生态架构优化方案:四个平面的收敛、已合入项的纠正与新系统的准入(mcpp × openkal × xlings)](2026-11-12-ecosystem-architecture-optimization-design.md) — active - [测试与依赖解析补充方案:mcpp、mcpp test、mtest、mcpp-ide](2026-11-10-test-and-resolution-supplement-design.md) — active - [生态顶层方案:#885 收尾之后的工作流、原则与依赖](2026-11-08-ecosystem-top-level-design.md) — active @@ -204,6 +205,7 @@ Records that declare one. Everything else is listed by date below. ### 2026-11 +- [生态规则收敛方案:2026.11.18.1 之后十个 issue 的五类成因、七项决定与实施范围](2026-11-19-ecosystem-rule-convergence-design.md) — active - [第二部分之后的补充方案:带修正的 ninja、模块重编译的级联、工作区发布与本轮遗留项](2026-11-17-ninja-cascade-publish-supplement-design.md) — active - [生态缺口补充方案(第二部分):文件的角色、运行文件、清单书写与第一部分的遗留项](2026-11-16-ecosystem-gap-supplement-part2-design.md) — active - [生态缺口补充方案:一个 C 程序走遍全部目标所暴露的通用问题](2026-11-15-ecosystem-gap-supplement-design.md) — active From 0ca2a18ac9867ea518b873a0fc8e1a3aca246289 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 12:52:32 +0900 Subject: [PATCH 02/30] A descriptor's mcpp field has two forms, read by one function; publish writes the pointer form (mcpp#960) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mcpp publish and mcpp emit xpkg write mcpp = "*/mcpp.toml": the archive is the package's source tree and its manifest states the modules and the dependencies - mcpp::pm::descriptor_manifest decides the form once for the build's layout check, the pre-install dependencies, the manifest, the patch root, mcpp new --template and mcpp xpkg parse - a table that names a manifest is refused with the line to write in its place - mcpp new --template refuses a package for the templates/ directory it lacks - SPEC-001 §3.5, docs/11 in both languages, the round trip as a unit test and as e2e index::local/960 --- CHANGELOG.md | 19 ++++++ docs/11-publishing-a-library.md | 20 ++++++ docs/specs/package-identity.md | 30 ++++++++- docs/zh/11-publishing-a-library.md | 15 +++++ modules/manifest/src/xpkg.cppm | 12 ++++ src/build/prepare/graph_load.cpp | 100 ++++++----------------------- src/cli/cmd_xpkg.cppm | 8 +-- src/pm/descriptor.cpp | 88 +++++++++++++++++++++++++ src/pm/descriptor.cppm | 58 +++++++++++++++++ src/pm/publisher.cppm | 66 ++++--------------- src/publish/pipeline.cppm | 4 +- src/scaffold/create.cppm | 37 +++-------- src/scaffold/template.cppm | 3 +- tests/e2e/cli/templates.cpp | 19 ++++-- tests/e2e/index/listing.cpp | 9 ++- tests/e2e/index/local.cpp | 49 ++++++++++++++ tests/unit/test_xpkg_emit.cpp | 83 +++++++++++++++--------- 17 files changed, 411 insertions(+), 209 deletions(-) create mode 100644 src/pm/descriptor.cpp create mode 100644 src/pm/descriptor.cppm diff --git a/CHANGELOG.md b/CHANGELOG.md index c0b60af03..5f191accf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ > Each `## []` section is that release's notes. Entries are written in English > from 2026.9.28.3 on; earlier entries remain as written. +## [2026.11.19.1] - 2026-11-19 + +### Fixed + +- `mcpp publish` and `mcpp emit xpkg` wrote a descriptor that nothing read: a table of + modules, version dependencies and run-time requirements with `manifest = "mcpp.toml"`. + `mcpp xpkg parse` refused it for its missing `sources`, a consumer could not depend on + it, and `mcpp new --template` refused it as an inline recipe (mcpp#960). A descriptor's + `mcpp` field has two forms (SPEC-001 §3.5): the pointer form names the manifest inside + the payload, the inline form is the manifest of a payload that carries none. Both + commands write the pointer form, `mcpp = "*/mcpp.toml"`; the manifest alone states the + modules and the dependencies. The six places that read the field read it through one + function (`mcpp::pm::descriptor_manifest`), and a table that names a manifest is refused + with the line to write in its place. +- `mcpp new --template ` refuses a package for the directory it lacks, + `templates/` beside its manifest, and names the directory it looked for. It refused an + inline descriptor for its form, and found the package root by a rule of its own that + ignored the descriptor's pointer. + ## [2026.11.18.1] - 2026-11-18 ### Fixed diff --git a/docs/11-publishing-a-library.md b/docs/11-publishing-a-library.md index 1cedea216..b91263f11 100644 --- a/docs/11-publishing-a-library.md +++ b/docs/11-publishing-a-library.md @@ -196,6 +196,26 @@ refuses a `path` dependency outside the package that is not a workspace member, and an inherited include directory outside the member's directory, because the archive contains neither. +### The descriptor `mcpp publish` writes *(mcpp 2026.11.19.1+)* + +A descriptor's `mcpp` field has two forms (SPEC-001 §3.5). The pointer form, +`mcpp = ""`, names the manifest inside the payload; the inline form, +`mcpp = { sources = ..., ... }`, is the manifest of a payload that carries none. +`mcpp publish` and `mcpp emit xpkg` write the pointer form: + +```lua +mcpp = "*/mcpp.toml", +``` + +The archive is the package's source tree under one top directory, and its +manifest is in it, so the descriptor states where the manifest is and repeats +nothing the manifest states: modules, dependencies and run-time requirements +are read from `mcpp.toml`. `mcpp xpkg parse`, a consumer's build and +`mcpp new --template` read the field through one function, so a descriptor that +was written is a descriptor that is read. Before this release the two commands +wrote a table with `manifest = "mcpp.toml"`, which is neither form; such a +descriptor is refused with the line to write in its place. + ### Names, the tag and several members *(mcpp 2026.11.17.2+)* | Item | Rule | diff --git a/docs/specs/package-identity.md b/docs/specs/package-identity.md index 8fdaf93ee..722f24b69 100644 --- a/docs/specs/package-identity.md +++ b/docs/specs/package-identity.md @@ -5,12 +5,12 @@ | **规范编号** | SPEC-001 | | **标题** | 包身份(`package.namespace` / `package.name`)、`[dependencies]` 选择器与匹配机制 | | **状态** | **评审中(Review)** —— 已实现 | -| **版本** | 1.7 | -| **最后修改** | 2026-11-09 | +| **版本** | 1.9 | +| **最后修改** | 2026-11-19 | | **最低实现版本** | 描述符身份:mcpp **0.0.106**;精确 selector:mcpp **2026.8.10.1**(xlings >= 0.4.69) | | **作者/维护** | mcpp-community | | **相关设计文档** | `.agents/docs/2026-06-20-package-resolution-architecture.md` §4
`.agents/docs/2026-06-26-identity-first-resolution-no-filename.md`
`.agents/docs/2026-07-25-issue278-descriptor-name-form-canonicalization-design.md`
`.agents/docs/2026-07-25-name-namespace-bidirectional-verification-report.md`
`.agents/docs/2026-07-25-name-namespace-canonical-implementation-spec.md` | -| **相关 issue** | [mcpp#278](https://github.com/mcpp-community/mcpp/issues/278)
[xlings#381](https://github.com/openxlings/xlings/issues/381) —— 索引键缺命名空间维度(§3.3) | +| **相关 issue** | [mcpp#278](https://github.com/mcpp-community/mcpp/issues/278)
[mcpp#960](https://github.com/mcpp-community/mcpp/issues/960) —— 发布写出的描述符形式(§3.5)
[xlings#381](https://github.com/openxlings/xlings/issues/381) —— 索引键缺命名空间维度(§3.3) | ## 规范用语 @@ -145,6 +145,29 @@ e2e `163_identity_first_resolution.sh` 锁住:身份为 `(acme, widget)` 的描 | `(compat, zlib)` | `compat.zlib.lua`, `zlib.lua` | | `(∅, imgui)` | `imgui.lua`, `compat.imgui.lua` | +### 3.5 描述符的 `mcpp` 字段 + +> **`mcpp` 字段陈述清单在何处,有且只有两种形式。** + +| 形式 | 写法 | 含义 | +|---|---|---| +| 指针形式(Form A) | `mcpp = ""` | 载荷带有清单。glob 相对版本目录,**必须**恰好匹配一个 `mcpp.toml`;该文件所在的目录是包根 | +| 内联形式(Form B) | `mcpp = { ... }` | 载荷不带清单,该表即清单。键的集合是封闭的(`kKnownXpkgKeys`),`sources` **必须**给出;包根是版本目录 | + +- 字段缺省时按指针形式处理,依次尝试 `mcpp.toml` 与 `*/mcpp.toml`。 +- 表中指向载荷内清单的写法**不是**第三种形式:`manifest` 键不存在。带 `manifest` 而没有 `sources` 的表被拒绝, + 消息写出指针形式的写法。 +- 读取方**必须**经同一个函数(`mcpp::pm::descriptor_manifest`)取得清单与包根。构建的布局核对、安装前的子依赖、 + 读清单、补丁根,`mcpp new --template` 与 `mcpp xpkg parse` 都经由它;包根之下的 `templates/` 是模板的位置, + 一个包因缺少该目录被拒绝,不因其描述符的形式被拒绝。 +- `mcpp publish` 与 `mcpp emit xpkg` **必须**写出读取方读取的形式。发布的归档是包的源码树,顶层目录内有清单, + 因此写出 `mcpp = "*/mcpp.toml"`;模块、依赖与运行期要求只由清单陈述,不写入描述符。 + +**已实现**(mcpp 2026.11.19.1)。此前 `mcpp publish` 写出带 `modules`、`deps`、`runtime` 与 `manifest = "mcpp.toml"` 的表, +六处读取方各自按字段的种类分支,没有一处读取它(mcpp#960)。单元测试 `XpkgEmit/WhatIsWrittenIsRead` +把写出方的输出交回读取函数;e2e `index::listing/06 emit xpkg` 与 `index::published/…` 断言发布的描述符可解析、 +可作为依赖、可作为模板包。 + --- ## 4. 消费侧规范:`[dependencies]` 书写文法 @@ -552,6 +575,7 @@ mcpp 侧的使用: | 版本 | 日期 | 变更 | |---|---|---| +| 1.9 | 2026-11-19 | 新增 §3.5:描述符 `mcpp` 字段的两种形式、一个读取函数、发布写出指针形式(mcpp 2026.11.19.1,mcpp#960) | | 1.8 | 2026-11-09 | 新增 §5.5:git 引用至多一个 ref、缺省分支、命令行的字符串形式(mcpp 2026.11.9.1) | | 1.5 | 2026-09-27 | 新增 §10.1:版本位的回答与 xlings 一致(安装记录优先,其次按 xlings 版本文法选择;已记录而缺失的载荷不算已安装),mcpp#712、#716;新增 §10.2:打包修订 `revision` 的含义与 mcpp 侧的三处使用(mcpp 2026.9.27.1,xlings 2026.9.27.1) | | 1.7 | 2026-10-17 | §10.2:新增 `floor`,区分可用与当前;工具链、运行时与宿主工具按使用/供给两种意图解析,使用命令不因修订号替换可用载荷;工具链载荷的修订号进入驱动身份;替换失败的语义。§10.3 第 3 条把修订号一致移出完整性(mcpp 2026.10.17.1,xlings 2026.10.17.1,libxpkg 0.0.65) | diff --git a/docs/zh/11-publishing-a-library.md b/docs/zh/11-publishing-a-library.md index f0e27d702..3da0726e9 100644 --- a/docs/zh/11-publishing-a-library.md +++ b/docs/zh/11-publishing-a-library.md @@ -170,6 +170,21 @@ acme.net = { path = "../net", version = "0.3.0" } # 按所写发布 兄弟成员的清单没有版本时拒绝。`mcpp publish` 同样拒绝包外、且不是工作空间成员的 `path` 依赖,以及位于成员目录之外的继承头文件目录,因为归档中二者都不存在。 +### `mcpp publish` 写出的描述符 *(mcpp 2026.11.19.1+)* + +描述符的 `mcpp` 字段有两种形式(SPEC-001 §3.5)。指针形式 `mcpp = ""` 指向载荷内的清单; +内联形式 `mcpp = { sources = ..., ... }` 是不带清单的载荷的清单。`mcpp publish` 与 `mcpp emit xpkg` +写出指针形式: + +```lua +mcpp = "*/mcpp.toml", +``` + +归档是包的源码树,位于一个顶层目录之下,清单在其中,所以描述符只陈述清单的位置,不重复清单陈述的内容: +模块、依赖与运行期要求从 `mcpp.toml` 读取。`mcpp xpkg parse`、消费方的构建与 `mcpp new --template` +经同一个函数读取该字段,因此写出的描述符就是被读取的描述符。此前两条命令写出带 `manifest = "mcpp.toml"` +的表,它不属于任何一种形式;这样的描述符被拒绝,消息写出应当替换成的那一行。 + ### 名字、tag 与多个成员 *(mcpp 2026.11.17.2+)* | 项 | 规则 | diff --git a/modules/manifest/src/xpkg.cppm b/modules/manifest/src/xpkg.cppm index 02681bc63..9a7180fb1 100644 --- a/modules/manifest/src/xpkg.cppm +++ b/modules/manifest/src/xpkg.cppm @@ -2563,6 +2563,18 @@ synthesize_from_xpkg_lua(std::string_view luaContent, // Validate minimum if (m.modules.sources.empty()) { + // A TABLE THAT NAMES A MANIFEST IS NEITHER FORM (SPEC-001 §3.5; + // mcpp#960). `mcpp publish` wrote one until 2026.11.19.1: a summary + // and `manifest = "mcpp.toml"`. What is missing is not `sources` --- + // the payload has a manifest --- so the refusal names the form such a + // payload takes. + if (std::ranges::find(m.xpkgUnknownKeys, std::string("manifest")) != m.xpkgUnknownKeys.end()) + return std::unexpected(ManifestError{ + "the `mcpp` table names a manifest (`manifest = ...`) and states no `sources`: a table is " + "the inline form, which is the manifest itself. A payload that carries its manifest is " + "named by the pointer form, `mcpp = \"*/mcpp.toml\"` (`mcpp publish` writes it from " + "2026.11.19.1; regenerate the descriptor, or replace the table with that line)", + m.sourcePath, 0, 0}); return std::unexpected(ManifestError{ "synthesised manifest missing sources (mcpp segment must declare `sources = { ... }`)", m.sourcePath, 0, 0}); diff --git a/src/build/prepare/graph_load.cpp b/src/build/prepare/graph_load.cpp index 17084c3af..df7d223e8 100644 --- a/src/build/prepare/graph_load.cpp +++ b/src/build/prepare/graph_load.cpp @@ -44,6 +44,7 @@ import mcpp.pm.mangle; import mcpp.pm.compat; import mcpp.pm.dep_spec; import mcpp.pm.dependency_selector; +import mcpp.pm.descriptor; import mcpp.pm.lock_io; import mcpp.pm.patch; import mcpp.ui; @@ -788,32 +789,8 @@ step4a_load_version_dep_locate(PrepareState& state, LoadVersionDepCtx& ctx) { auto installedLayoutMatchesIndex = [&](const std::filesystem::path& verRoot) -> bool { if (!ctx.luaContent) return false; - auto field = mcpp::manifest::extract_mcpp_field(*ctx.luaContent); - if (field.kind == mcpp::manifest::McppField::StringPath) { - return !mcpp::modgraph::expand_glob(verRoot, field.value).empty(); - } - if (field.kind == mcpp::manifest::McppField::TableBody) { - auto dm = mcpp::manifest::synthesize_from_xpkg_lua( - *ctx.luaContent, ctx.shortName, ctx.version, *state.targetPlatform); - if (!dm) return false; - for (auto const& [generatedPath, _] : dm->buildConfig.generatedFiles) { - if (!generatedPath.empty()) return true; - } - for (auto const& glob : dm->modules.sources) { - if (!glob.empty() && glob.front() == '!') continue; - if (!mcpp::modgraph::expand_glob(verRoot, glob).empty()) { - return true; - } - } - return false; - } - - for (auto pat : { "mcpp.toml", "*/mcpp.toml" }) { - if (!mcpp::modgraph::expand_glob(verRoot, pat).empty()) { - return true; - } - } - return false; + return mcpp::pm::descriptor_payload_present( + *ctx.luaContent, verRoot, ctx.shortName, ctx.version, *state.targetPlatform); }; // THE DESCRIPTOR'S REVISION IS PART OF WHAT IS INSTALLED (#524 A, @@ -912,8 +889,7 @@ step4a_load_version_dep_fetch(PrepareState& state, LoadVersionDepCtx& ctx) { } if (!ctx.installed) { if (ctx.luaContent) { - auto field = mcpp::manifest::extract_mcpp_field(*ctx.luaContent); - if (field.kind == mcpp::manifest::McppField::TableBody) { + if (mcpp::pm::descriptor_is_inline(*ctx.luaContent)) { auto depManifest = mcpp::manifest::synthesize_from_xpkg_lua( *ctx.luaContent, shortName, version, *state.targetPlatform); if (!depManifest) { @@ -1253,8 +1229,6 @@ step4a_load_version_dep_read_manifest(PrepareState& state, LoadVersionDepCtx& ct } if (!ctx.luaContent) return std::unexpected(with_index_cause(std::format( "dependency '{}': index entry not found in local clone", depName))); - auto field = mcpp::manifest::extract_mcpp_field(*ctx.luaContent); - // 0.0.6+: read explicit namespace from xpkg lua if present. auto luaNs = mcpp::manifest::extract_xpkg_namespace(*ctx.luaContent); @@ -1286,48 +1260,18 @@ step4a_load_version_dep_read_manifest(PrepareState& state, LoadVersionDepCtx& ct effRoot = mcppToml.parent_path(); return {}; }; - if (field.kind == mcpp::manifest::McppField::StringPath) { - auto matches = mcpp::modgraph::expand_glob(verRoot, field.value); - if (matches.empty()) return std::unexpected(std::format( - "dependency '{}': mcpp pointer '{}' did not match any " - "file under '{}'", depName, field.value, verRoot.string())); - if (matches.size() > 1) return std::unexpected(std::format( - "dependency '{}': mcpp pointer '{}' matched {} files " - "(expected exactly one)", depName, field.value, matches.size())); - if (auto r = loadFrom(matches.front()); !r) return std::unexpected(r.error()); - } else if (field.kind == mcpp::manifest::McppField::TableBody) { - auto dm = mcpp::manifest::synthesize_from_xpkg_lua( - *ctx.luaContent, shortName, version, *state.targetPlatform); - if (!dm) return std::unexpected(std::format( - "dependency '{}': {}", depName, dm.error().format())); - warn_unknown_xpkg_keys(*dm, depName); - manifest = std::move(*dm); + // The descriptor's two forms are decided in one place (SPEC-001 §3.5): + // a manifest inside the payload, or the table itself. + auto stated = mcpp::pm::descriptor_manifest( + *ctx.luaContent, verRoot, shortName, version, *state.targetPlatform); + if (!stated) return std::unexpected(std::format( + "dependency '{}': {}", depName, stated.error())); + if (stated->form == mcpp::pm::DescriptorManifest::Form::Inline) { + warn_unknown_xpkg_keys(*stated->manifest, depName); + manifest = std::move(*stated->manifest); // effRoot stays as verRoot - } else { - std::vector matches; - for (auto pat : { "mcpp.toml", "*/mcpp.toml" }) { - matches = mcpp::modgraph::expand_glob(verRoot, pat); - if (!matches.empty()) break; - } - // Name the directory actually searched. `` was a literal - // placeholder, so the message could not distinguish "the package - // is Form B and you forgot the mcpp field" from "the verdir mcpp - // resolved is not this package's at all" — the second is what a - // cross-namespace install_path hit produces, and it sent this - // investigation down the wrong path for a while. - if (matches.empty()) return std::unexpected(std::format( - "dependency '{}': index entry has no `mcpp = ...` field, " - "and no mcpp.toml was found at '{}/mcpp.toml' or " - "'{}/*/mcpp.toml' — add an explicit `mcpp = \"\"` " - "or `mcpp = {{ ... }}` block to the .lua descriptor. " - "(If that directory belongs to a DIFFERENT package, the " - "install step resolved the wrong verdir.)", - depName, verRoot.string(), verRoot.string())); - if (matches.size() > 1) return std::unexpected(std::format( - "dependency '{}': default mcpp.toml lookup matched {} " - "files; pin one with explicit `mcpp = \"\"`.", - depName, matches.size())); - if (auto r = loadFrom(matches.front()); !r) return std::unexpected(r.error()); + } else if (auto r = loadFrom(stated->manifestPath); !r) { + return std::unexpected(r.error()); } // Propagate lua-level namespace into the loaded manifest when // the manifest itself doesn't carry one (Form A descriptors @@ -1449,15 +1393,11 @@ std::expected phase4a_graph_load(PrepareState& state) { std::filesystem::path packageRoot; if (!ctx.luaContent) ctx.luaContent = step4a_read_lua_content(state, ctx); if (ctx.luaContent) { - const auto field = mcpp::manifest::extract_mcpp_field(*ctx.luaContent); - std::vector found; - if (field.kind == mcpp::manifest::McppField::StringPath) - found = mcpp::modgraph::expand_glob(*ctx.installed, field.value); - else if (field.kind != mcpp::manifest::McppField::TableBody) - for (auto pat : { "mcpp.toml", "*/mcpp.toml" }) - if (found = mcpp::modgraph::expand_glob(*ctx.installed, pat); !found.empty()) break; - if (found.size() == 1) - packageRoot = std::filesystem::relative(found.front().parent_path(), *ctx.installed); + // An inline descriptor's root is the version directory itself. + if (!mcpp::pm::descriptor_is_inline(*ctx.luaContent)) + if (auto stated = mcpp::pm::descriptor_manifest( + *ctx.luaContent, *ctx.installed, shortName, version, *state.targetPlatform)) + packageRoot = std::filesystem::relative(stated->packageRoot, *ctx.installed); } if (packageRoot == ".") packageRoot.clear(); const auto patchAt = (packageRoot / patchRoot).lexically_normal(); diff --git a/src/cli/cmd_xpkg.cppm b/src/cli/cmd_xpkg.cppm index 50e0af27a..264d98287 100644 --- a/src/cli/cmd_xpkg.cppm +++ b/src/cli/cmd_xpkg.cppm @@ -16,6 +16,7 @@ import std; import mcpplibs.mwrap.libc; import mcpplibs.cmdline; import mcpp.manifest; +import mcpp.pm.descriptor; import mcpp.toolchain.platform; import mcpp.ui; import mcpp.wire; @@ -188,12 +189,11 @@ export int cmd_xpkg_parse(const mcpplibs::cmdline::ParsedArgs& parsed) { return 1; } - // Form A descriptors carry no `mcpp = {}` table — build info comes - // from the fetched source's own mcpp.toml. Nothing further to parse. + // The pointer form (Form A; SPEC-001 §3.5) carries no table: build info + // comes from the manifest in the payload. Nothing further to parse. // The xpm version tables are independent of that segment and are the // resolver's source of truth, so both spellings publish them here. - auto field = mcpp::manifest::extract_mcpp_field(lua); - if (field.kind != mcpp::manifest::McppField::TableBody) { + if (!mcpp::pm::descriptor_is_inline(lua)) { if (asJson) { emit_xpkg(std::format( "{{\"namespace\":\"{}\",\"name\":\"{}\",\"versions\":{}," diff --git a/src/pm/descriptor.cpp b/src/pm/descriptor.cpp new file mode 100644 index 000000000..0e395881f --- /dev/null +++ b/src/pm/descriptor.cpp @@ -0,0 +1,88 @@ +// mcpp.pm.descriptor --- the two forms of a descriptor's `mcpp` field, decided +// once (SPEC-001 §3.5). +// +// An implementation unit: with these definitions in the interface, GCC 16.1 +// ended with an internal error (a segmentation fault) at every import of a +// module that reaches it (`import mcpp.cli;` in src/main.cpp). +module mcpp.pm.descriptor; + +import std; +import mcpp.manifest; +import mcpp.modgraph.scanner; // expand_glob +import mcpp.toolchain.platform; + +namespace mcpp::pm { + +bool descriptor_is_inline(std::string_view lua) { + return mcpp::manifest::extract_mcpp_field(lua).kind == mcpp::manifest::McppField::TableBody; +} + +bool descriptor_payload_present(std::string_view lua, const std::filesystem::path& versionDir, + std::string_view name, std::string_view version, + const mcpp::toolchain::platform::PlatformKey& platform) { + const auto field = mcpp::manifest::extract_mcpp_field(lua); + if (field.kind == mcpp::manifest::McppField::StringPath) + return !mcpp::modgraph::expand_glob(versionDir, field.value).empty(); + if (field.kind == mcpp::manifest::McppField::TableBody) { + auto inlined = mcpp::manifest::synthesize_from_xpkg_lua(lua, name, version, platform); + if (!inlined) return false; + for (auto const& [generatedPath, _] : inlined->buildConfig.generatedFiles) + if (!generatedPath.empty()) return true; + for (auto const& glob : inlined->modules.sources) { + if (!glob.empty() && glob.front() == '!') continue; + if (!mcpp::modgraph::expand_glob(versionDir, glob).empty()) return true; + } + return false; + } + for (auto pattern : { "mcpp.toml", "*/mcpp.toml" }) + if (!mcpp::modgraph::expand_glob(versionDir, pattern).empty()) return true; + return false; +} + +std::expected +descriptor_manifest(std::string_view lua, const std::filesystem::path& versionDir, + std::string_view name, std::string_view version, + const mcpp::toolchain::platform::PlatformKey& platform) { + const auto field = mcpp::manifest::extract_mcpp_field(lua); + DescriptorManifest out; + if (field.kind == mcpp::manifest::McppField::TableBody) { + auto inlined = mcpp::manifest::synthesize_from_xpkg_lua(lua, name, version, platform); + if (!inlined) return std::unexpected(inlined.error().format()); + out.form = DescriptorManifest::Form::Inline; + out.packageRoot = versionDir; + out.manifest = std::move(*inlined); + return out; + } + std::vector matches; + if (field.kind == mcpp::manifest::McppField::StringPath) { + matches = mcpp::modgraph::expand_glob(versionDir, field.value); + if (matches.empty()) return std::unexpected(std::format( + "mcpp pointer '{}' did not match any file under '{}'", field.value, versionDir.string())); + if (matches.size() > 1) return std::unexpected(std::format( + "mcpp pointer '{}' matched {} files under '{}' (expected exactly one)", + field.value, matches.size(), versionDir.string())); + } else { + for (auto pattern : { "mcpp.toml", "*/mcpp.toml" }) { + matches = mcpp::modgraph::expand_glob(versionDir, pattern); + if (!matches.empty()) break; + } + // The directory searched is named: a version directory that belongs to + // another package reads the same as a descriptor without the field, + // and only the path tells the two apart. + if (matches.empty()) return std::unexpected(std::format( + "the index entry has no `mcpp = ...` field, and no mcpp.toml was found at " + "'{}/mcpp.toml' or '{}/*/mcpp.toml'. A payload that carries its manifest is named by " + "`mcpp = \"\"`; one that carries none states it inline, `mcpp = {{ sources = ... }}`. " + "(If that directory belongs to a different package, the install resolved the wrong " + "version directory.)", versionDir.string(), versionDir.string())); + if (matches.size() > 1) return std::unexpected(std::format( + "the default mcpp.toml lookup matched {} files under '{}'; name one with `mcpp = \"\"`", + matches.size(), versionDir.string())); + } + out.form = DescriptorManifest::Form::Pointer; + out.manifestPath = matches.front(); + out.packageRoot = out.manifestPath.parent_path(); + return out; +} + +} // namespace mcpp::pm diff --git a/src/pm/descriptor.cppm b/src/pm/descriptor.cppm new file mode 100644 index 000000000..a36508eb0 --- /dev/null +++ b/src/pm/descriptor.cppm @@ -0,0 +1,58 @@ +// mcpp.pm.descriptor — the manifest a descriptor states (SPEC-001 §3.5). +// +// A descriptor's `mcpp` field has two forms. The pointer form, +// `mcpp = ""`, says the payload carries its manifest: the glob is +// relative to the version directory and matches exactly one `mcpp.toml`, whose +// directory is the package's root; with no field the two conventional places +// are tried. The inline form, `mcpp = { ... }`, says the payload carries none +// and the table is the manifest. +// +// Six places read the field, each with a branch of its own on its kind, and +// the writer (`mcpp publish`) wrote a table that named a manifest inside the +// payload: a third form none of the six read (mcpp#960). The forms are decided +// here once, so a reader asks for the manifest and its root, and a reader that +// needs only one of them cannot disagree with another about which form it met. +export module mcpp.pm.descriptor; + +import std; +import mcpp.manifest; +import mcpp.toolchain.platform; + +export namespace mcpp::pm { + +struct DescriptorManifest { + enum class Form { Pointer, Inline }; + Form form = Form::Pointer; + // The package's root: the directory of the manifest the pointer names, or + // the version directory of an inline descriptor. + std::filesystem::path packageRoot; + // Pointer: the manifest inside the payload. The caller loads it, since + // what a manifest inherits depends on where the caller found the package. + std::filesystem::path manifestPath; + // Inline: the table, read as a manifest for `platform`. + std::optional manifest; +}; + +// Whether the field is the inline form. The one question a reader may ask +// before a payload exists: an inline descriptor states its dependencies in the +// index, and a pointer one states them in a manifest not yet fetched. +bool descriptor_is_inline(std::string_view lua); + +// Whether the payload at `versionDir` holds what the descriptor names: asked +// of an installed payload before it is used, since one that does not goes back +// to the provider. A pointer is answered by any match --- two matches are a +// descriptor's error, reported when the manifest is read, and reinstalling +// does not repair them. +bool descriptor_payload_present(std::string_view lua, const std::filesystem::path& versionDir, + std::string_view name, std::string_view version, + const mcpp::toolchain::platform::PlatformKey& platform); + +// The manifest `lua` states for the payload installed at `versionDir`. The +// error names what was looked for and where, without the package's name: the +// caller says whose descriptor it is. +std::expected +descriptor_manifest(std::string_view lua, const std::filesystem::path& versionDir, + std::string_view name, std::string_view version, + const mcpp::toolchain::platform::PlatformKey& platform); + +} // namespace mcpp::pm diff --git a/src/pm/publisher.cppm b/src/pm/publisher.cppm index 01322e12b..12211653a 100644 --- a/src/pm/publisher.cppm +++ b/src/pm/publisher.cppm @@ -7,10 +7,8 @@ module; export module mcpp.pm.publisher; import std; -import mcpp.pack.host_requirements; // J: one derivation, two projections import mcpp.manifest; import mcpp.diag; -import mcpp.modgraph.graph; import mcpp.run; // `cfgpred::os_only_platforms` (#630 item 7) — the standalone half of // `mcpp.build.prepare`, carrying no dependency on the rest of it (see that @@ -34,7 +32,6 @@ struct ReleaseInfo { // Generate the xpkg Lua content for a package. std::string emit_xpkg(const mcpp::manifest::Manifest& manifest, - const mcpp::modgraph::Graph& graph, const ReleaseInfo& release); // Convenience: synthesize a placeholder ReleaseInfo for `mcpp emit xpkg --version V` @@ -203,7 +200,6 @@ std::string platform_deps_block( } // namespace std::string emit_xpkg(const mcpp::manifest::Manifest& manifest, - const mcpp::modgraph::Graph& graph, const ReleaseInfo& release) { std::string out; @@ -319,57 +315,17 @@ std::string emit_xpkg(const mcpp::manifest::Manifest& manifest, + platform_block(release.version, release.windows) + " },\n"; out += " },\n\n"; - out += " mcpp = {\n"; - out += " schema = \"0.1\",\n"; - out += std::format(" language = {},\n", lua_escape(manifest.language.standard)); - out += std::format(" import_std = {},\n", manifest.language.importStd ? "true" : "false"); - - // Module list (from scanner) - out += " modules = {\n"; - for (auto& u : graph.units) { - if (!u.provides) continue; - // Skip partition-only units: their logical name contains ':' - if (u.provides->logicalName.find(':') != std::string::npos) continue; - out += std::format(" {},\n", lua_escape(u.provides->logicalName)); - } - out += " },\n"; - - // Dependencies (excluding dev-dependencies). Path-based deps are - // local-only and intentionally not exposed in the published xpkg - // descriptor; only version-based deps are emitted. - out += " deps = {\n"; - for (auto& [k, v] : manifest.dependencies) { - if (v.isPath() || v.version.empty()) continue; - out += std::format(" [{}] = {},\n", lua_escape(k), lua_escape(v.version)); - } - out += " },\n"; - - // What the TARGET machine must provide. - // - // THE SAME DERIVATION `mcpp pack` USES. A tarball can only DESCRIBE these - // (its HOST-REQUIREMENTS file); a descriptor can have them RESOLVED, by - // the xlings on the machine that installs the package. Two projections of - // one fact — so they come from one function. Deriving them separately is - // how they drift, and a drifted list is undetectable: each side looks - // reasonable on its own. - if (auto hostReqs = mcpp::pack::host_requirements_of(manifest.runtimeConfig); - !hostReqs.empty()) { - out += " runtime = {\n"; - out += " requirements = {\n"; - for (auto const& req : hostReqs) { - out += std::format( - " {{ kind = \"capability\", value = {}, " - "phase = \"run\", required = {}, discovery = {} }},\n", - lua_escape(req.capability), - req.required ? "true" : "false", - lua_escape(req.discovery)); - } - out += " },\n"; - out += " },\n"; - } - - out += " manifest = \"mcpp.toml\",\n"; - out += " },\n"; + // THE POINTER FORM (SPEC-001 §3.5). The archive is the package's source + // tree under one top-level directory, and its manifest is in it, so the + // descriptor says where the manifest is and states nothing the manifest + // states. Until 2026.11.19.1 this wrote a table --- the modules, the + // version dependencies, the run-time requirements, and `manifest = + // "mcpp.toml"` --- which no reader took for either form: `xpkg parse`, a + // consumer and `mcpp new --template` each refused what `mcpp publish` had + // just written (mcpp#960). The table could not have been a faithful + // summary either: its `deps` has no target axis, so a package whose + // dependencies are conditional published an empty list. + out += " mcpp = \"*/mcpp.toml\",\n"; out += "}\n"; return out; } diff --git a/src/publish/pipeline.cppm b/src/publish/pipeline.cppm index d4dcff9cf..416ac2f7f 100644 --- a/src/publish/pipeline.cppm +++ b/src/publish/pipeline.cppm @@ -77,7 +77,7 @@ export int emit_xpkg_to(std::string version, const std::filesystem::path& output } auto release = mcpp::publish::placeholder_release(version); - auto lua = mcpp::publish::emit_xpkg(*m, scan.graph, release); + auto lua = mcpp::publish::emit_xpkg(*m, release); if (output.empty()) { std::print("{}", lua); @@ -200,7 +200,7 @@ std::expected publish_one(const std::filesystem::path& root, // 4. Build release info + emit xpkg.lua. auto release = mcpp::publish::make_release_info(pkg.version, url, out.sha); - out.lua = mcpp::publish::emit_xpkg(*m, scan.graph, release); + out.lua = mcpp::publish::emit_xpkg(*m, release); { std::ofstream os(out.descriptor); os << out.lua; diff --git a/src/scaffold/create.cppm b/src/scaffold/create.cppm index 016f0527e..5f94c312c 100644 --- a/src/scaffold/create.cppm +++ b/src/scaffold/create.cppm @@ -10,6 +10,7 @@ import mcpp.fetcher.progress; import mcpp.manifest; import mcpp.pm.dep_spec; import mcpp.pm.dependency_selector; +import mcpp.pm.descriptor; import mcpp.pm.index_route; import mcpp.toolchain.platform; import mcpp.pm.resolver; @@ -84,16 +85,6 @@ fetch_template_package(const mcpp::scaffold::TemplateSpec& spec) { } auto lua = std::move(found.hit->lua); - // Form B packages are inline build recipes with no source tree, hence no - // physical templates directory. Report the provider error rather than - // pretending that a different namespace might satisfy the selector. - auto field = mcpp::manifest::extract_mcpp_field(lua); - if (field.kind == mcpp::manifest::McppField::TableBody) { - return std::unexpected(std::format( - "template package '{}' is an inline build recipe and ships no " - "source template payload", selector)); - } - const std::string& ns = coordinate.namespace_; const std::string& shortName = coordinate.shortName; const auto constraint = spec.version @@ -144,23 +135,15 @@ fetch_template_package(const mcpp::scaffold::TemplateSpec& spec) { "package '{}@{}' install path missing after fetch", fq, version)); } - // Package root = the directory holding mcpp.toml (tarballs usually wrap - // everything in a single top-level directory). - std::filesystem::path root = *installed; - if (!std::filesystem::exists(root / "mcpp.toml")) { - std::error_code ec; - for (auto& e : std::filesystem::directory_iterator(root, ec)) { - if (e.is_directory() - && std::filesystem::exists(e.path() / "mcpp.toml")) { - root = e.path(); - break; - } - } - } - if (!std::filesystem::exists(root / "mcpp.toml")) { - return std::unexpected(std::format( - "package '{}@{}' has no mcpp.toml", selector, version)); - } + // The package's root is where its descriptor says its manifest is + // (SPEC-001 §3.5), and its templates are the directory `templates/` there. + // A package is refused for the directory it lacks (`list_templates`), + // which is what a template needs, and not for the form of its descriptor. + auto stated = mcpp::pm::descriptor_manifest( + lua, *installed, shortName, version, mcpp::toolchain::platform::HostPlatform::current()); + if (!stated) return std::unexpected(std::format( + "template package '{}@{}': {}", selector, version, stated.error())); + std::filesystem::path root = stated->packageRoot; return FetchedTemplatePackage{ .root = std::move(root), .id = coordinate, diff --git a/src/scaffold/template.cppm b/src/scaffold/template.cppm index 615c17242..c9a950377 100644 --- a/src/scaffold/template.cppm +++ b/src/scaffold/template.cppm @@ -242,7 +242,8 @@ std::expected, std::string> list_templates(const std::filesystem::path& packageRoot) { auto dir = packageRoot / "templates"; if (!std::filesystem::exists(dir)) { - return std::unexpected("package ships no templates/ directory"); + return std::unexpected(std::format( + "package ships no templates: there is no directory '{}'", dir.string())); } std::vector out; std::error_code ec; diff --git a/tests/e2e/cli/templates.cpp b/tests/e2e/cli/templates.cpp index b709e457f..a57f93938 100644 --- a/tests/e2e/cli/templates.cpp +++ b/tests/e2e/cli/templates.cpp @@ -24,7 +24,11 @@ void configure_mirror(const fs::path& dir, const s::options& o) { (void)s::mcpp({"self", "config", "--mirror", s::env("MCPP_E2E_TOOLCHAIN_MIRROR").value_or("GLOBAL")}, dir, o); } -std::string descriptor(std::string_view ns, std::string_view shortname, std::string_view ver, std::string_view mcpp_value = "\"mcpp.toml\"") { +// The pointer is the one `mcpp publish` writes: the manifest under the archive's top directory, +// which is where every payload below is laid out. `mcpp new` reads it as a build does +// (SPEC-001 §3.5); a pointer that named no file was accepted here while it found the package's +// root by a rule of its own. +std::string descriptor(std::string_view ns, std::string_view shortname, std::string_view ver, std::string_view mcpp_value = "\"*/mcpp.toml\"") { return std::format(R"(package = {{ spec = "1", namespace = "{0}", name = "{1}", description = "exact template fixture {0}.{1}", @@ -84,9 +88,11 @@ E2E templates{"templates", [] { ["1.0.0"] = { url = "https://example.invalid/x.tar.gz", sha256 = "0" }, }, }, - mcpp = "src/*.cppm", + mcpp = "*/mcpp.toml", } )"); + // The descriptor is in the form `mcpp publish` writes (SPEC-001 §3.5): a pointer to the + // manifest under the archive's top directory, which is where the templates are found. const auto xpkgs = home / "registry/data/xpkgs"; for (auto [ver, marker] : {std::pair{"1.0.0", "old"}, {"1.2.0", "new"}, {"2.0.0-rc.1", "rc"}}) { const auto root = xpkgs / "mcpplibs-x-tpl-demo" / ver / std::format("tpl-demo-{}", ver); @@ -197,8 +203,11 @@ self = {{ features = ["alpha", "beta"] }} expect(fatal(failed(r))) << "L7: bare selector crossed into compat\n" << s::text(r); expect(fatal(e2e::contains(s::text(r), "(mcpplibs, tpl-repro)"))) << "L7: bare miss did not name exact identity\n" << s::text(r); r = run({"new", "app7", "--template", "compat.tpl-repro"}, o); - expect(fatal(failed(r))) << "L7: Form B template must fail\n" << s::text(r); - expect(fatal(s::contains_nocase(s::text(r), "inline build recipe"))) << "L7: wrong provider error for Form B\n" << s::text(r); + // An inline descriptor is refused for what a template needs and it lacks, the directory + // `templates/` of its payload, and not for its form (mcpp#960). + expect(fatal(failed(r))) << "L7: a package without templates must fail\n" << s::text(r); + expect(fatal(e2e::contains(s::text(r), "ships no templates") && e2e::contains(s::text(r), "templates'"))) + << "L7: the refusal does not name the directory it looked for\n" << s::text(r); // 8. A foreign namespace with the same short name. r = run({"new", "app-acme", "--template", "acme.tpl-demo"}, o); @@ -304,7 +313,7 @@ self = {{ features = ["alpha", "beta"] }} macosx = {{ ["{2}"] = {{ url = "https://example.invalid/{1}.tar.gz", sha256 = "{0}-digest" }} }}, windows = {{ ["{2}"] = {{ url = "https://example.invalid/{1}.zip", sha256 = "{0}-digest" }} }}, }}, - mcpp = "mcpp.toml", + mcpp = "*/mcpp.toml", }} )", ns, shortname, version)); const auto root = home / "registry/data/xpkgs" / store / version / std::format("{}-{}", shortname, version); diff --git a/tests/e2e/index/listing.cpp b/tests/e2e/index/listing.cpp index a509c2c36..6ff3a2ffd 100644 --- a/tests/e2e/index/listing.cpp +++ b/tests/e2e/index/listing.cpp @@ -54,17 +54,20 @@ void pin_and_unpin(const s::project& p, const s::options& o) { E2E listing{"listing", [] { "06 emit xpkg"_test = [] { - // `emit xpkg` writes a Lua entry from the manifest: the header, the name, the module, the - // manifest pointer and import_std; `--output` writes the file, which lua loads. + // `emit xpkg` writes a Lua entry from the manifest: the header, the name and the + // pointer to the manifest in the archive (SPEC-001 §3.5); `--output` writes the file, + // which lua loads, and `xpkg parse` reads what was written (mcpp#960). auto p = s::project::scaffold("emit-test"); p.write("src/foo.cppm", "export module emit-test;\nimport std;\nexport int answer() { return 42; }\n"); auto r = p.mcpp({"emit", "xpkg"}); expect(fatal(e2e::succeeded(r))) << s::text(r); - for (std::string_view w : {"AUTO-GENERATED", "name = \"emit-test\"", "\"emit-test\"", "manifest = \"mcpp.toml\"", "import_std = true"}) + for (std::string_view w : {"AUTO-GENERATED", "name = \"emit-test\"", "mcpp = \"*/mcpp.toml\""}) expect(fatal(e2e::contains(r.out, w))) << "missing " << w << ": " << r.out; r = p.mcpp({"emit", "xpkg", "--output", "myout.lua"}); expect(fatal(e2e::succeeded(r))) << s::text(r); expect(fatal(p.exists("myout.lua"))) << "--output didn't write file"; + r = p.mcpp({"xpkg", "parse", "myout.lua"}); + expect(fatal(e2e::succeeded(r))) << "the descriptor `emit xpkg` wrote does not parse: " << s::text(r); if (auto lua = s::on_path("lua")) { auto l = e2e::run({.argv = {lua->string(), "-e", "loadfile('myout.lua')()"}, .cwd = p.dir()}); expect(fatal(l.started && l.status == 0)) << "Lua failed to load output: " << l.err; diff --git a/tests/e2e/index/local.cpp b/tests/e2e/index/local.cpp index a78a012ad..06fc8b3c4 100644 --- a/tests/e2e/index/local.cpp +++ b/tests/e2e/index/local.cpp @@ -192,6 +192,55 @@ exit 0 expect(fatal(s::has_line(s::text(r), R"(Cached compat\.widget v1\.0\.0 \((index [^,)]+, )?[0-9]+ units?\))"))) << "the cached namespaced dependency should be reported as compat.widget: " << s::text(r); }; + "960 what publish writes is what a consumer reads"_test[s::gcc][s::fresh_sandbox] = [] { + // mcpp#960, SPEC-001 §3.5. `mcpp publish` writes the pointer form of the descriptor's + // `mcpp` field; `mcpp xpkg parse` reads the file it wrote; a project that names the + // package through a local index builds against the published archive and runs. The + // table the command wrote before, which names a manifest, is refused with the line to + // write in its place. + const auto tmp = s::case_dir(); + (void)s::case_home(s::home::inherited); + const auto git = s::on_path("git"); + if (!git) skip("git is not installed"); + s::project lib(tmp / "lib", { + {"mcpp.toml", "[package]\nnamespace = \"probe\"\nname = \"sdk\"\nversion = \"0.1.0\"\ndescription = \"probe\"\nlicense = \"MIT\"\nrepo = \"https://github.com/example/probe\"\n\n[targets.sdk]\nkind = \"lib\"\n"}, + {"src/sdk.cppm", "export module probe.sdk;\nexport int sdk_value() { return 42; }\n"}}); + for (std::vector args : {std::vector{"init", "-q", "-b", "main", "."}, {"add", "-A"}, + {"-c", "user.email=e2e@example.invalid", "-c", "user.name=e2e", "commit", "-qm", "init"}}) { + auto g = s::run(*git, args, lib.dir()); + expect(fatal(e2e::succeeded(g))) << "git failed: " << s::text(g); + } + auto r = lib.mcpp({"publish", "--dry-run"}, inherited()); + expect(fatal(e2e::succeeded(r))) << "publish failed: " << s::text(r); + const auto lua = lib.at("target/dist/probe.sdk.lua"); + const auto archive = lib.at("target/dist/probe.sdk-0.1.0.tar.gz"); + expect(fatal(fs::is_regular_file(lua) && fs::is_regular_file(archive))) << "publish wrote no descriptor or no archive: " << s::text(r); + const auto written = s::read_file(lua); + expect(fatal(e2e::contains(written, "mcpp = \"*/mcpp.toml\""))) << "the descriptor is not in the pointer form:\n" << written; + expect(fatal(!e2e::contains(written, "manifest ="))) << "the descriptor names a manifest in a table:\n" << written; + r = lib.mcpp({"xpkg", "parse", lua.string()}, inherited()); + expect(fatal(e2e::succeeded(r))) << "the descriptor publish wrote does not parse: " << s::text(r); + + s::project index(tmp / "local-index", {{"pkgs/p/probe.sdk.lua", written}}); + s::project app(tmp / "project/app", { + {"src/main.cpp", "import std;\nimport probe.sdk;\nint main() { std::println(\"v={}\", sdk_value()); }\n"}, + {"mcpp.toml", std::format("[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[indices]\nprobe = {{ path = \"{}\" }}\n\n[dependencies.probe]\nsdk = \"0.1.0\"\n\n[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n", index.dir().generic_string())}}); + fs::create_directories(app.at(".mcpp/.xlings/data/xpkgs/probe.sdk/0.1.0")); + s::tar_extract(archive, app.at(".mcpp/.xlings/data/xpkgs/probe.sdk/0.1.0")); + r = app.mcpp({"run"}, inherited()); + expect(fatal(e2e::succeeded(r))) << "a consumer of the published descriptor does not build: " << s::text(r); + expect(fatal(e2e::contains(r.out, "v=42"))) << s::text(r); + + // What the command wrote before: a summary and `manifest = "mcpp.toml"`. + auto before = written; + const std::string_view pointer = "mcpp = \"*/mcpp.toml\","; + before.replace(before.find(pointer), pointer.size(), "mcpp = { schema = \"0.1\", language = \"c++23\", import_std = true, manifest = \"mcpp.toml\" },"); + s::project older(tmp / "older", {{"probe.sdk.lua", before}}); + r = older.mcpp({"xpkg", "parse", "probe.sdk.lua"}, inherited()); + expect(fatal(r.started && r.status != 0)) << "a table that names a manifest was accepted: " << s::text(r); + expect(fatal(e2e::contains(s::text(r), "mcpp = \"*/mcpp.toml\""))) << "the refusal does not state the line to write: " << s::text(r); + }; + "60 stale xpkg cache reinstall"_test[s::gcc][s::fresh_sandbox] = [] { // A payload directory without its marker is installed again (through the interface, // never direct), mcpp writes no marker into the payload, and the program runs on the diff --git a/tests/unit/test_xpkg_emit.cpp b/tests/unit/test_xpkg_emit.cpp index dc250f4fb..cce301701 100644 --- a/tests/unit/test_xpkg_emit.cpp +++ b/tests/unit/test_xpkg_emit.cpp @@ -2,7 +2,8 @@ import std; import mcpplibs.mtest; import mcpp.manifest; -import mcpp.modgraph.graph; +import mcpp.pm.descriptor; +import mcpp.toolchain.platform; import mcpp.invocation; import mcpp.publish.xpkg_emit; import mcpp.diag; @@ -27,37 +28,66 @@ mcpp::manifest::Manifest minimal_manifest() { return m; } -mcpp::modgraph::Graph minimal_graph() { - mcpp::modgraph::Graph g; - mcpp::modgraph::SourceUnit u; - u.path = "/x/hello.cppm"; - u.provides = mcpp::modgraph::ModuleId{"mcpplibs.hello"}; - g.units.push_back(u); - return g; -} - } // namespace UTest contains_required_fields{"XpkgEmit/ContainsRequiredFields", [] { auto m = minimal_manifest(); - auto g = minimal_graph(); auto rel = placeholder_release("0.1.0"); - auto out = emit_xpkg(m, g, rel); + auto out = emit_xpkg(m, rel); expect(ne(out.find("AUTO-GENERATED"), std::string::npos)); expect(ne(out.find(" name = \"hello\""), std::string::npos)); - expect(ne(out.find("import_std = true"), std::string::npos)); - expect(ne(out.find("\"mcpplibs.hello\""), std::string::npos)); // module - expect(ne(out.find("manifest = \"mcpp.toml\""), std::string::npos)); - expect(ne(out.find("[\"mcpplibs.primitives\"]"), std::string::npos)); // dep + // The pointer form (SPEC-001 §3.5): where the manifest is, and nothing the + // manifest states. + expect(ne(out.find(" mcpp = \"*/mcpp.toml\",\n"), std::string::npos)) << out; + expect(eq(out.find("manifest ="), std::string::npos)) << out; + expect(eq(out.find("modules ="), std::string::npos)) << out; + expect(eq(out.find("mcpplibs.primitives"), std::string::npos)) << out; +}}; + +// mcpp#960: what `mcpp publish` writes is what every reader reads. The +// descriptor goes back through the one function the build, `xpkg parse` and +// `mcpp new --template` read it with, over a payload laid out as the archive is. +UTest what_is_written_is_read{"XpkgEmit/WhatIsWrittenIsRead", [] { + auto m = minimal_manifest(); + const auto out = emit_xpkg(m, placeholder_release("0.1.0")); + const auto field = mcpp::manifest::extract_mcpp_field(out); + expect(fatal(field.kind == mcpp::manifest::McppField::StringPath)) << out; + expect(eq(field.value, std::string("*/mcpp.toml"))); + + const auto versionDir = std::filesystem::temp_directory_path() + / std::format("mcpp-xpkg-emit-{}", std::chrono::steady_clock::now().time_since_epoch().count()); + const auto top = versionDir / "hello-0.1.0"; + std::filesystem::create_directories(top); + { std::ofstream(top / "mcpp.toml") << "[package]\nname = \"hello\"\nversion = \"0.1.0\"\n"; } + const auto stated = mcpp::pm::descriptor_manifest( + out, versionDir, "hello", "0.1.0", mcpp::toolchain::platform::HostPlatform::current()); + expect(fatal(stated.has_value())) << (stated ? std::string{} : stated.error()); + expect(stated->form == mcpp::pm::DescriptorManifest::Form::Pointer); + expect(stated->manifestPath == top / "mcpp.toml") << stated->manifestPath.string(); + expect(stated->packageRoot == top) << stated->packageRoot.string(); + expect(!mcpp::pm::descriptor_is_inline(out)); + expect(mcpp::pm::descriptor_payload_present( + out, versionDir, "hello", "0.1.0", mcpp::toolchain::platform::HostPlatform::current())); + + // The table `mcpp publish` wrote before: neither form, and the refusal + // says which form a payload with a manifest takes. + const std::string before = "package = { name = \"hello\", mcpp = { schema = \"0.1\", " + "language = \"c++23\", manifest = \"mcpp.toml\" } }"; + const auto refused = mcpp::pm::descriptor_manifest( + before, versionDir, "hello", "0.1.0", mcpp::toolchain::platform::HostPlatform::current()); + expect(fatal(!refused.has_value())); + expect(ne(refused.error().find("mcpp = \"*/mcpp.toml\""), std::string::npos)) << refused.error(); + + std::error_code ec; + std::filesystem::remove_all(versionDir, ec); }}; UTest rejects_lua_injection{"XpkgEmit/RejectsLuaInjection", [] { auto m = minimal_manifest(); m.package.description = "evil\"injection"; - auto g = minimal_graph(); - auto out = emit_xpkg(m, g, placeholder_release("0.1.0")); + auto out = emit_xpkg(m, placeholder_release("0.1.0")); // The escaped form must be present, NOT the raw quote. expect(ne(out.find("evil\\\"injection"), std::string::npos)); @@ -65,12 +95,11 @@ UTest rejects_lua_injection{"XpkgEmit/RejectsLuaInjection", [] { UTest escapes_control_characters{"XpkgEmit/EscapesControlCharacters", [] { auto m = minimal_manifest(); - auto g = minimal_graph(); // Description with backslash, CR, NUL, BEL, DEL — all must be escaped. m.package.description = std::string("a\\b\rc") + std::string(1, '\0') + std::string("d\x07") + std::string(1, '\x7f') + "e"; - auto out = emit_xpkg(m, g, placeholder_release("0.1.0")); + auto out = emit_xpkg(m, placeholder_release("0.1.0")); // Backslash → doubled expect(ne(out.find("a\\\\b"), std::string::npos)); @@ -123,14 +152,13 @@ UTest xpkg_emit_names_of_a_release{"XpkgEmit/TheNamesAPublishedPackageIsFiledUnd // (SPEC-001 §3.2), whichever of the two the manifest's name is written as. UTest xpkg_emit_name_is_one_segment{"XpkgEmit/TheNameIsOneSegment", [] { auto m = minimal_manifest(); - auto g = minimal_graph(); m.package.namespace_ = "mlibs"; m.package.name = "math"; - auto out = emit_xpkg(m, g, placeholder_release("0.1.0")); + auto out = emit_xpkg(m, placeholder_release("0.1.0")); expect(ne(out.find("namespace = \"mlibs\""), std::string::npos)) << out; expect(ne(out.find(" name = \"math\""), std::string::npos)) << out; m.package.name = "mlibs.math"; - out = emit_xpkg(m, g, placeholder_release("0.1.0")); + out = emit_xpkg(m, placeholder_release("0.1.0")); expect(ne(out.find(" name = \"math\""), std::string::npos)) << out; }}; @@ -187,9 +215,8 @@ UTest long_bracket_sequence_in_value_is_harmless{"XpkgEmit/LongBracketSequenceIn // ASCII and pass through; what matters is the surrounding quotes // are intact and no escape is needed for them). auto m = minimal_manifest(); - auto g = minimal_graph(); m.package.description = "trick: ]==] more stuff"; - auto out = emit_xpkg(m, g, placeholder_release("0.1.0")); + auto out = emit_xpkg(m, placeholder_release("0.1.0")); expect(ne(out.find("\"trick: ]==] more stuff\""), std::string::npos)); }}; @@ -228,10 +255,9 @@ version = "0.1.0" )"; auto m = mcpp::manifest::parse_string(src); if (auto&& failed = expect(fatal(m.has_value())); !failed) failed << (m ? std::string{} : m.error().format()); - auto g = minimal_graph(); mcpp::diag::reset(); - auto out = emit_xpkg(*m, g, placeholder_release("0.1.0")); + auto out = emit_xpkg(*m, placeholder_release("0.1.0")); expect(ne(block(out, "linux = {").find("xim:gtk4"), std::string::npos)) << out; @@ -255,10 +281,9 @@ version = "0.1.0" )"; auto m = mcpp::manifest::parse_string(src); if (auto&& failed = expect(fatal(m.has_value())); !failed) failed << (m ? std::string{} : m.error().format()); - auto g = minimal_graph(); mcpp::diag::reset(); - auto out = emit_xpkg(*m, g, placeholder_release("0.1.0")); + auto out = emit_xpkg(*m, placeholder_release("0.1.0")); expect(eq(block(out, "linux = {").find("xim:gtk4"), std::string::npos)); expect(eq(block(out, "macosx = {").find("xim:gtk4"), std::string::npos)); From 42d99cc4634cf1df4fa567df9227052863f541e7 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 12:52:32 +0900 Subject: [PATCH 03/30] The version group's constraint enters the merge; a package's tables are read by the row it is built for (mcpp#956) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a request the platform's version group binds carries the group's version into the resolution record and every merge, so a build program's request of the same package is answered by one version in whichever order the two arrive; a refusal lists every request and its requester (SPEC-013 §7.5) - a package reached only through build dependencies reads its [target..dependencies] by this machine's row; a package both sides ask for makes each side's requests, and mcpp.lock records both (SPEC-004 §4.8) - the build programs' graph is given a project's local index by its resolved path, where a relative one was copied as written - e2e openkal::platforms/956 (two cases), docs/05 and docs/35 in both languages --- CHANGELOG.md | 18 ++++ docs/05-dependencies.md | 9 ++ docs/35-platforms.md | 9 ++ docs/specs/manifest-semantics.md | 24 ++++- docs/specs/platforms.md | 14 ++- docs/zh/05-dependencies.md | 6 ++ docs/zh/35-platforms.md | 5 + src/build/prepare/features.cpp | 4 +- src/build/prepare/graph.cpp | 111 ++++++++++++++------- src/build/prepare/graph_edges.cpp | 6 +- src/build/prepare/graph_requests.cpp | 143 +++++++++++++++++++++++++++ src/build/prepare/host_graph.cpp | 9 +- src/build/prepare/platform.cpp | 20 +++- src/build/prepare/state.cppm | 44 ++++++++- tests/e2e/openkal/platforms.cpp | 90 +++++++++++++++++ 15 files changed, 467 insertions(+), 45 deletions(-) create mode 100644 src/build/prepare/graph_requests.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f191accf..25ee34219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,24 @@ modules and the dependencies. The six places that read the field read it through one function (`mcpp::pm::descriptor_manifest`), and a table that names a manifest is refused with the line to write in its place. +- A build that takes a library through `[build-dependencies]` was refused for a version + the resolver itself had chosen: `openkal-macos 0.23.5 is in this build, and the platform + ... is a version group that names openkal-macos 0.23.4` (mcpp#956). The group's version + was written into the request and went no further: a second request of the package was + merged with the ranges as written. The group's constraint is now a constraint of the + request and enters the resolution record and every merge, so the requests of a package + are answered by one version in whichever order they arrive (SPEC-013 §7.5). A refusal + lists every request and its requester. +- A package reached only through build dependencies is a module of a build program, built + for this machine, and its `[target..dependencies]` were read by the build's + target. They are read by the row the package is built for (SPEC-004 §4.8): crossing to + another system no longer resolves the target's implementation of a layer for a build + program's library, and `mcpp.lock` records the package the build program links. A + package both sides ask for makes each side's requests. +- The build programs' graph looked for a project's local index beside itself: + `[indices] x = { path = "" }` was copied as written into a manifest in another + directory, and a build dependency from that index was `no package found for exact + selector`. The path is written as the project's manifest resolves it. - `mcpp new --template ` refuses a package for the directory it lacks, `templates/` beside its manifest, and names the directory it looked for. It refused an inline descriptor for its form, and found the package root by a rule of its own that diff --git a/docs/05-dependencies.md b/docs/05-dependencies.md index 64972cae8..65d501e50 100644 --- a/docs/05-dependencies.md +++ b/docs/05-dependencies.md @@ -542,6 +542,15 @@ target and one for the build program. Both have the version the lock records. Their features are activated separately: a feature requested only by a `[build-dependencies]` edge does not reach the target instance. +**Which row reads a package's conditional tables *(mcpp 2026.11.19.1+)*.** A +package's `[target..dependencies]` are read by the row the package is +built for (SPEC-004 §4.8). A package of the target is read by the build's +target. A package reached only through `[build-dependencies]` is a module of a +build program and is read by the row of the machine that runs the build: under +`--target x86_64-macos-musl` on Linux, a library of `build.mcpp` takes its +`[target.linux.dependencies]`. A package both sides ask for is read once for +each. `mcpp.lock` records the packages of both sides. + **A reading kept for compatibility.** A `[dependencies]` entry that asks for `host-module = true`, or for a feature that declares `rule_module`, is read as a `[build-dependencies]` entry: the package serves the build program and not the diff --git a/docs/35-platforms.md b/docs/35-platforms.md index 3fec2671b..05c401fce 100644 --- a/docs/35-platforms.md +++ b/docs/35-platforms.md @@ -191,6 +191,15 @@ directory or a repository are not bound by the group. An mcpp before 2026.11.7.1 checks the group after resolution instead; a family released as one index change resolves the same either way. +The group's version is a constraint of the request it binds and takes part in +every merge *(mcpp 2026.11.19.1+)*. A package that a build program and the target +both ask for has one version: a build program's `>=1.0, <2.0` and the group's +`=1.1.0` give 1.1.0, in whichever order the two requests arrive, and a request +that had already taken a newer version is replaced (`Merged 1.2.0 ⨯ +1.1.0 → v1.1.0`). Where no version satisfies every request the refusal lists +each request and its requester, the group's under the platform's name. A +package only a build program uses is still not bound. + ## The built-in platform is one release The target table selects one openkal-platform release for the two ok-image diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index ede13d92d..7d96266d1 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -5,8 +5,8 @@ | **规范编号** | SPEC-004 | | **标题** | `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约 | | **状态** | **草案(Draft)** | -| **版本** | 1.19 | -| **最后修改** | 2026-10-28 | +| **版本** | 1.21 | +| **最后修改** | 2026-11-19 | | **最低实现版本** | 条件化形状:mcpp **2026.8.29.1**(`[target..build-dependencies]` 起齐备);目标轴:mcpp **2026.9.6.4** | | **作者/维护** | mcpp-community | | **相关设计文档** | `.agents/docs/2026-09-07-mcpp-toml-unified-semantics-design.md`
`.agents/docs/2026-06-04-manifest-schema-ownership.md`
`.agents/docs/2026-09-03-xlings-workspace-as-the-one-table.md`
`.agents/docs/2026-09-25-issue-690-workspace-build-inheritance-consistency.md`
`.agents/docs/2026-09-27-eight-reports-by-home-and-one-optimisation-plan.md`
`.agents/docs/2026-10-05-std-module-pair-msvc-lto-and-export-discovery-design.md`
`.agents/docs/2026-10-06-windows-x86-arch-vocabulary-and-rc-follow-ups-design.md`
`.agents/docs/2026-10-10-one-source-one-meaning-design.md`
`.agents/docs/2026-10-11-build-dependencies-one-pipeline-design.md`
`.agents/docs/2026-10-23-target-base-and-execution-host-design.md`
`.agents/docs/2026-10-26-control-contract-runtime-and-ide-ecosystem-design.md`(§24) | @@ -363,6 +363,25 @@ feature-deps feature-xlings ← 限定词是门 **状态:已实现**(mcpp 2026.10.1.3,mcpp#755)。 +### 4.8 包的依赖表由构建它的那一行读取 + +一个包为哪一行构建,它的 `[target..dependencies]` 与 `[target..build-dependencies]` 就**必须**由 +哪一行选择。目标图中的包由构建的目标读取;只经 `[build-dependencies]` 到达的包是构建程序的模块,为本机构建 +(SPEC-013 §4 的宿主图),由本机的行读取,宿主图选用平台时由该平台在本机上服务的目标读取。 + +- 两侧都请求的包为每一侧各构建一次,每一侧按自己的行读取它的表,并各自发出请求。先到达的一侧发出它的请求; + 另一侧第一次请求该包时再发出另一侧的请求,因此结果与请求到达的先后无关。 +- 只出现在本机一行的读取中的条目,是该包在构建程序中的实例的请求,它的边按 `[build-dependencies]` 的边记录, + 目标图不经它到达该依赖。只出现在目标一行的读取中的条目,构建程序不经它到达该依赖。 +- `mcpp.lock` 记录两侧解析出的包:交叉构建时,构建程序链接的包在锁中,目标的行为一个只属于构建程序的包点名的 + 包不在锁中。 +- 点名层的 selector(`cfg(c-abi = ...)` 等)在主解析中仍按目标的层读取:本机一行的层在宿主图自身准备时选定, + 由宿主图读取。 + +**状态:已实现**(mcpp 2026.11.19.1,mcpp#956)。此前只属于构建程序的包的表也由目标读取:交叉到 macOS 时,构建程序的库 +请求了 macOS 的内核层实现,而它实际链接的 Linux 实现在主解析之外解析,不在锁中。e2e +`openkal::platforms/956 a build program's library reads its tables by this machine's row`。 + ## 5. 命名规约 ### 5.1 两种 case,按面划分 @@ -816,3 +835,4 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m | 1.18 | 2026-10-27 | mcpp 2026.10.26.2(mcpp#845):新增 §14 程序起点与镜像入口(`kind = "entry"`、`main = { from }`、`start`、`[test] main` 与 `start`、`mcpp:link-start`);§3.1.1 的按行形式可陈述起点。 | | 1.19 | 2026-10-28 | mcpp 2026.10.27.1(mcpp#847):§2 增加平台平面(`[platform]`、`[host]`),平台键可出现在工程与机器默认中;§4.2 增加 `[target.]` 的平台键与 `[host] platform`(SPEC-013)。 | | 1.20 | 2026-11-17 | mcpp 2026.11.17.1(设计 2026-11-16 §4):新增 §15 运行时文件与发布文件:`deploy` 的条目语法(字符串、通配符、`name`)、固定目录规则与过期副本的移除;`license_files` 与其缺省值。 | +| 1.21 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):新增 §4.8 包的依赖表由构建它的那一行读取;两侧各自发出请求,锁记录两侧。 | diff --git a/docs/specs/platforms.md b/docs/specs/platforms.md index 97d965087..a3bbb72ab 100644 --- a/docs/specs/platforms.md +++ b/docs/specs/platforms.md @@ -4,9 +4,9 @@ |---|---| | 规范编号 | SPEC-013 | | 标题 | 平台:一类目标的可复用配置,其声明、选择、合并、对依赖图的补全、链接与封装、运行器与载体 | -| 状态 | 评审中 v1.12 | -| 版本 | 1.12 | -| 最后修改 | 2026-11-17 | +| 状态 | 评审中 v1.13 | +| 版本 | 1.13 | +| 最后修改 | 2026-11-19 | | 对应实现 | mcpp >= 2026.10.27.1;§10 的执行宿主与 §11 的载体架构 mcpp >= 2026.10.29.1;`runner.hosts`、载体列表、`descriptor_keys` 与 `universal` 的弃用 mcpp >= 2026.10.30.1;§7.5 版本组参与解析、§14 平台系列与 §15.1 系列的锁 mcpp >= 2026.11.7.1;§14 内置平台为确切版本与提示 mcpp >= 2026.11.7.2;v1.7 条款 mcpp >= 2026.11.10.1;v1.10 条款 mcpp >= 2026.11.16.1;v1.11 条款(§14 索引的平台列) mcpp >= 2026.11.17.1;逐条标注 | | 相关设计文档 | `.agents/docs/2026-11-16-ecosystem-gap-supplement-part2-design.md`、`.agents/docs/2026-11-15-ecosystem-gap-supplement-design.md`、`.agents/docs/2026-10-28-platforms-design.md`、`.agents/docs/2026-10-28-platforms-part2-packaging-and-distribution-design.md`、`.agents/docs/2026-10-29-openkal-run-test-pack-design.md` | | 相关 issue | mcpp#846、mcpp#847、mcpp#848 | @@ -269,6 +269,13 @@ mcpp **必须**给出警告(`manifest/platform-selector`),并指出按三元组 解析之后的检查保留为断言:在每一轮补全之前与最后一轮之后,图中 `versions` 所列的包都是组中版本。早于 2026.11.7.1 的引擎只做这一检查;对一个以兼容范围互相依赖、且整组一次发布到索引的家族,两者的结果相同。**已实现** +版本组对一个成员的约束是该请求的约束,**必须**进入解析表的记录与此后的每一次合并(mcpp >= 2026.11.19.1,mcpp#956)。 +同一个包的全部请求由一个它们都容许的版本满足:构建程序的请求 `>=1.0, <2.0` 与版本组的 `=1.1.0` 给出 1.1.0, +与两者到达的先后无关;先到达的请求已取得更新的版本时,它被替换并输出一行 `Merged`。没有这样的版本时拒绝, +消息列出每一个请求及其请求者,版本组的约束以平台包的名字列出。始终没有进入目标图的包仍不受约束。 +此前改写后的约束不进入合并:构建程序的请求先取得区间内最新的版本,平台的层随后到达,合并认为已有版本满足 +双方书写的范围而保留它,解析之后的检查再拒绝一个解析器自己选出的版本。**已实现** + ### 7.6 多版本改名的边界 两个版本的同一个包以改名并存只改 C++ 模块名。导出头文件目录的包与以 `provides` 提供 `mcpp:` 层的包 @@ -628,3 +635,4 @@ openkal-platform 的 `targets` 因此不含 WebAssembly。Web 经 `wasm32-emscri | 1.11 | 2026-11-16 | mcpp 2026.11.17.1(设计 2026-11-16 §6.2):§3.1 第 5 步与 §14 索引的平台列 `platform_rows`;判据 16(e2e 1106)。 | | 1.12 | 2026-11-17 | mcpp 2026.11.17.1:§3.1 尚无行定义的三元组按字面成分求值选择器(e2e 1040);§10 运行器的陈述与许可分属两层,覆盖层的 `runner = true`/`false`(e2e 1092)。 | | 1.12 | 2026-11-17 | mcpp 2026.11.17.2(设计 2026-11-17 §5):§3.1 平台选定之前可用的谓词,以层选择平台时的警告;§10 `hosts` 参与运行器的采用;§14 目标表的平台列为 0.29.1 | +| 1.13 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):§7.5 版本组的约束进入合并,结果与请求到达的先后无关;拒绝列出每一个请求者。 | diff --git a/docs/zh/05-dependencies.md b/docs/zh/05-dependencies.md index d411f8f68..5c5a9a7da 100644 --- a/docs/zh/05-dependencies.md +++ b/docs/zh/05-dependencies.md @@ -463,6 +463,12 @@ protobuf = { version = "35.1", tools = ["protoc"] } # 构建期间运行 锁文件记录的版本。它们的特性分别激活:只由 `[build-dependencies]` 边请求的特性 不进入目标实例。 +**包的条件表由哪一行读取 *(mcpp 2026.11.19.1+)*。** 一个包的 `[target..dependencies]` +由构建它的那一行读取(SPEC-004 §4.8)。目标的包由构建的目标读取。只经 `[build-dependencies]` +到达的包是构建程序的模块,由执行构建的机器的行读取:在 Linux 上以 `--target x86_64-macos-musl` +构建时,`build.mcpp` 的库取它的 `[target.linux.dependencies]`。两侧都请求的包为每一侧各读取一次。 +`mcpp.lock` 记录两侧的包。 + **为兼容保留的读法。** `[dependencies]` 中请求 `host-module = true`、或请求声明了 `rule_module` 的特性的条目,按 `[build-dependencies]` 的条目读取:这个包服务构建程序 而不进入目标,与 mcpp 2026.10.11.1 之前相同。该读法自 mcpp 2026.10.11.1 弃用, diff --git a/docs/zh/35-platforms.md b/docs/zh/35-platforms.md index 40e5b9133..9591c3e84 100644 --- a/docs/zh/35-platforms.md +++ b/docs/zh/35-platforms.md @@ -160,6 +160,11 @@ error: openkal-musl is requested as '^0.27.0' by app, and the platform openkal-p 只为本机构建的包(工具的)以及从目录或仓库取得的包不受版本组约束。早于 2026.11.7.1 的 mcpp 改为在解析之后检查 版本组;一个家族以一次索引改动整组发布时,两者结果相同。 +版本组的版本是它所约束的请求的约束,参与每一次合并*(mcpp 2026.11.19.1+)*。构建程序与目标都请求的包只有一个版本: +构建程序的 `>=1.0, <2.0` 与版本组的 `=1.1.0` 给出 1.1.0,与两个请求到达的先后无关;已经取得更新版本的请求被替换 +(`Merged <包> 1.2.0 ⨯ 1.1.0 → v1.1.0`)。没有版本满足全部请求时,拒绝列出每一个请求及其请求者,版本组的约束以 +平台的名字列出。只有构建程序使用的包仍不受约束。 + ## 内置平台是一个版本 目标表为两个 ok 映像目标选择一个 openkal-platform 版本,即当前 mcpp 验证过的版本:mcpp 2026.11.10.1 为 0.25.0,2026.11.8.1 与 2026.11.9.1 为 0.24.4,2026.11.7.2 为 0.24.3 diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index 67f139e38..d851be14d 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -257,7 +257,9 @@ PackageInstances package_instances(const PrepareState& state) { const auto p = stack.back(); stack.pop_back(); for (auto const& e : state.dependencyEdges) { - if (e.consumerPackageIndex != p) continue; + // An edge of the target's reading of the consumer's table is not + // the build program's (SPEC-004 §4.8). + if (e.consumerPackageIndex != p || e.targetRowOnly) continue; const auto d = e.dependencyPackageIndex; if (d >= n || host[d]) continue; host[d] = true; diff --git a/src/build/prepare/graph.cpp b/src/build/prepare/graph.cpp index be542b595..ab79cccee 100644 --- a/src/build/prepare/graph.cpp +++ b/src/build/prepare/graph.cpp @@ -714,7 +714,7 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, auto merged = mcpp::pm::try_merge_semver( key.ns, key.shortName, effective_constraint(state, it->second), - item.originalConstraint, + request_constraint(item), state.index_route(*cfg), *state.targetPlatform); if (!merged) { // Level 1 fallback: multi-version mangling. Two @@ -747,14 +747,14 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, if (item.fromBuildTable) { return std::unexpected(std::format( "build dependency '{}{}{}' has irreconcilable versions:\n" - " '{}' (constraint '{}') requested by '{}'\n" + " '{}', resolved for these requests:{}\n" " '{}' (constraint '{}') requested by '{}'\n" "SemVer merge: {}\n" "A build dependency is a module of a build program, and a build " "loads one version of it. Align the requests on one version{}.", key.ns, key.ns.empty() ? "" : ".", key.shortName, - it->second.version, effective_constraint(state, it->second), it->second.requestedBy, - spec.version, item.originalConstraint, item.requestedBy, + it->second.version, requests_of(state, it->second), + spec.version, request_constraint(item), item.requestedBy, merged.error(), state.platform.active ? std::format(", or select the release of {} that carries the " @@ -765,15 +765,15 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, if (item.consumerDepIndex == kMainConsumer) { return std::unexpected(std::format( "dependency '{}{}{}' has irreconcilable versions:\n" - " '{}' (constraint '{}') requested by '{}'\n" + " '{}', resolved for these requests:{}\n" " '{}' (constraint '{}') requested by '{}'\n" "SemVer merge: {}\n" "Multi-version mangling can't help here — the conflict " "involves the main package directly. Pin one version " "explicitly in your mcpp.toml.", key.ns, key.ns.empty() ? "" : ".", key.shortName, - it->second.version, it->second.constraint, it->second.requestedBy, - spec.version, item.originalConstraint, item.requestedBy, + it->second.version, requests_of(state, it->second), + spec.version, request_constraint(item), item.requestedBy, merged.error()) + override_hint(key, it->second.version, spec.version)); } @@ -930,7 +930,7 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, if (!secPackage) return std::unexpected(secPackage.error()); state.packages.push_back(std::move(*secPackage)); recordDependencyEdge(state, item.consumerDepIndex, depPackageIndex, - spec, item.fromBuildTable, name); + spec, item.fromBuildTable, name, item.targetRowOnly); auto linkFlagsAdded = propagateLinkFlags(state, secStage, *state.dep_manifests.back()); ResolvedKey mangledKey{key.ns, mangledPackage}; @@ -958,23 +958,16 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, return {}; } - // Combine the constraint strings so future merges AND with - // both. Empty originalConstraint means "any" — use "*". - const std::string& addCstr = - item.originalConstraint.empty() ? std::string("*") - : item.originalConstraint; - if (it->second.constraint.empty()) - it->second.constraint = addCstr; - else - it->second.constraint += "," + addCstr; - it->second.constraintsBy.emplace_back(item.requestedBy, addCstr); + // The request joins the record, so future merges AND with it + // as well; the group's constraint goes with it. + note_request(state, it->second, item); if (*merged == it->second.version) { // The existing pin already satisfies the new constraint — // no re-fetch needed; just record this consumer edge. recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name); + spec, item.fromBuildTable, name, item.targetRowOnly); return {}; } @@ -1036,7 +1029,7 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, state.packages[it->second.depIndex + 1] = std::move(*mergedPackage); recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name); + spec, item.fromBuildTable, name, item.targetRowOnly); it->second.version = *merged; it->second.linkFlagsAdded = std::move(linkFlagsAdded); @@ -1059,6 +1052,52 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, return {}; } +// The sides of a package's dependency table and the requests each makes are +// in graph_requests.cpp (SPEC-004 §4.8). +// A side's requests of a package it has not yet asked. The first walk of a +// package made the requests of the side that reached it; a request from the +// other side makes that side's here, so a package first reached by a build +// program and then by the target has its dependencies asked for the target as +// well, where the platform's version group binds them. +static std::expected +step4b_walk_other_side(PrepareState& state, std::size_t depIndex, const WorkItem& item) { + if (depIndex >= state.dep_manifests.size() || !state.dep_manifests[depIndex]) return {}; + if (state.depSidesWalked.size() <= depIndex) state.depSidesWalked.resize(depIndex + 1, 0); + const unsigned char side = item.buildOnly ? kHostSide : kTargetSide; + // A slot no walk has marked is one of the paths that walk no children + // (a package of programs, a mangled secondary): it stays unwalked. + if (state.depSidesWalked[depIndex] == 0 || (state.depSidesWalked[depIndex] & side)) return {}; + state.depSidesWalked[depIndex] |= side; + auto const& dm = *state.dep_manifests[depIndex]; + auto requests = side_requests(state, dm, item.buildOnly); + if (!requests) return std::unexpected(requests.error()); + std::vector active; + if (auto requested = state.depFeatureRequests.find(depIndex); requested != state.depFeatureRequests.end()) + active = feature_closure(dm, requested->second.first, requested->second.second); + std::string label = dm.package.name; + for (auto const& [k, r] : state.resolved) + if (r.depIndex == depIndex) { + label = std::format("{}{}{}@{}", k.ns, k.ns.empty() ? "" : ".", k.shortName, + r.source == "version" ? r.version : r.source); + break; + } + const auto root = depIndex + 1 < state.packages.size() ? state.packages[depIndex + 1].root + : std::filesystem::path{}; + for (auto const& [childName, childSpec] : requests->dependencies) { + auto childReq = childSpec; + injectForwards(dm, active, childName, childReq); + state.worklist.push_back(side_item(*requests, childName, childReq, label, depIndex, root, item)); + } + for (auto const& [childName, childSpec] : requests->buildDependencies) { + auto childReq = childSpec; + injectForwards(dm, active, childName, childReq); + state.worklist.push_back({childName, childReq, label + " (build-dep)", childReq.version, + depIndex, root, item.devOnly, /*buildOnly=*/true, + /*fromBuildTable=*/true}); + } + return {}; +} + // A LATER EDGE THAT ASKS MORE FEATURES OF A RESOLVED PACKAGE (mcpp#897). The // package was walked with the features its first edge asked; the union of its // edges is what it is built with, so the feature-deps and the forwards of each @@ -1126,6 +1165,7 @@ step4b_handle_already_resolved(PrepareState& state, WorklistItemCtx& ctx, if (sourceKind == "version" && it->second.depIndex < state.dep_manifests.size() && state.dep_manifests[it->second.depIndex]) step4b_note_exact_pin(state, item, *state.dep_manifests[it->second.depIndex]); + if (auto r = step4b_walk_other_side(state, it->second.depIndex, item); !r) return r; // A package is dev-only until some non-dev consumer wants it. Order // of arrival must not decide, so this is an AND over every request. it->second.devOnly = it->second.devOnly && item.devOnly; @@ -1232,7 +1272,7 @@ step4b_handle_already_resolved(PrepareState& state, WorklistItemCtx& ctx, if (it->second.depIndex + 1 < state.packages.size()) { recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name); + spec, item.fromBuildTable, name, item.targetRowOnly); } return step4b_widen_features(state, it->second.depIndex, item, spec); } @@ -1308,7 +1348,7 @@ step4b_handle_already_resolved(PrepareState& state, WorklistItemCtx& ctx, if (it->second.depIndex + 1 < state.packages.size()) { recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name); + spec, item.fromBuildTable, name, item.targetRowOnly); } return step4b_widen_features(state, it->second.depIndex, item, spec); return {}; @@ -1900,7 +1940,7 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { } state.packages.push_back(std::move(*depPackage)); recordDependencyEdge(state, item.consumerDepIndex, depPackageIndex, spec, - item.fromBuildTable, name); + item.fromBuildTable, name, item.targetRowOnly); // Record this dep as resolved so future encounters of the same // (ns, name) hit the fast path (skip / merge / conflict). @@ -1926,11 +1966,11 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { .depIndex = state.dep_manifests.size() - 1, .linkFlagsAdded = std::move(linkFlagsAdded), .patches = spec.patches, - .constraintsBy = sourceKind == "version" - ? std::vector>{{item.requestedBy, - item.originalConstraint.empty() ? std::string("*") : item.originalConstraint}} - : std::vector>{}, }; + if (sourceKind == "version") { + state.resolved[key].constraint.clear(); + note_request(state, state.resolved[key], item); + } if (sourceKind == "version" && !state.dep_manifests.empty() && state.dep_manifests.back()) step4b_note_exact_pin(state, item, *state.dep_manifests.back()); @@ -1971,13 +2011,19 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { && std::ranges::find(childSpec.features, depFeat) == childSpec.features.end()) childSpec.features.push_back(depFeat); }; - for (auto& [child_name, child_spec] : state.dep_manifests.back()->dependencies) { + // The side that reached the package reads its tables by its own row + // (SPEC-004 §4.8); the other side's requests are made when that side + // first asks for the package (`step4b_walk_other_side`). + if (state.depSidesWalked.size() <= selfIdx) state.depSidesWalked.resize(selfIdx + 1, 0); + state.depSidesWalked[selfIdx] |= item.buildOnly ? kHostSide : kTargetSide; + auto side = side_requests(state, *state.dep_manifests.back(), item.buildOnly); + if (!side) return std::unexpected(side.error()); + for (auto& [child_name, child_spec] : side->dependencies) { auto childReq = child_spec; injectForwards(*state.dep_manifests.back(), depActive, child_name, childReq); injectMemberCliForwards(child_name, childReq); - state.worklist.push_back({child_name, childReq, thisDepLabel, - childReq.version, selfIdx, ctx.dep_root, - item.devOnly, item.buildOnly}); + state.worklist.push_back(side_item(*side, child_name, childReq, thisDepLabel, selfIdx, + ctx.dep_root, item)); } // A selected member's `[dev-dependencies]` under `mcpp test`, as a // root's: its own, and never walked into its dependencies' tests. @@ -2001,8 +2047,7 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { // These are build-only regardless of how this package was reached: a // library's build dependency has no business in its consumer's binary // either. - for (auto& [child_name, child_spec] : - state.dep_manifests.back()->buildDependencies) { + for (auto& [child_name, child_spec] : side->buildDependencies) { auto childReq = child_spec; injectForwards(*state.dep_manifests.back(), depActive, child_name, childReq); injectMemberCliForwards(child_name, childReq); diff --git a/src/build/prepare/graph_edges.cpp b/src/build/prepare/graph_edges.cpp index ef61781c7..f8df58c45 100644 --- a/src/build/prepare/graph_edges.cpp +++ b/src/build/prepare/graph_edges.cpp @@ -33,7 +33,8 @@ void recordDependencyEdge( std::size_t dependencyPackageIndex, const mcpp::manifest::DependencySpec& spec, bool buildTable, - const std::string& writtenKey) + const std::string& writtenKey, + bool targetRowOnly) { const auto consumerPackageIndex = packageIndexForConsumer(consumerDepIndex); if (consumerPackageIndex >= state.packages.size() @@ -114,6 +115,8 @@ void recordDependencyEdge( it->requestedFeatures.push_back(f); it->defaultFeatures = it->defaultFeatures || spec.defaultFeatures; it->wantsLibrary = it->wantsLibrary || wantsLibrary; + // Of the target's reading alone only while every declaration is. + it->targetRowOnly = it->targetRowOnly && targetRowOnly; it->reexport = it->reexport || spec.reexport; it->embed = it->embed || spec.embed; for (auto const& x : spec.exclude) @@ -135,6 +138,7 @@ void recordDependencyEdge( .embed = spec.embed, .exclude = spec.exclude, }); + state.dependencyEdges.back().targetRowOnly = targetRowOnly; } // THE PACKAGE BEING BUILT STANDS FOR ITSELF IN ITS TESTS' GRAPH. A dependency diff --git a/src/build/prepare/graph_requests.cpp b/src/build/prepare/graph_requests.cpp new file mode 100644 index 000000000..360c43268 --- /dev/null +++ b/src/build/prepare/graph_requests.cpp @@ -0,0 +1,143 @@ +// graph_requests.cpp -- P4b: what a request of the dependency worklist is bound +// by, and which side of the build makes it. Split from graph.cpp at the +// boundary of a request's own facts, when a request learned to carry the +// platform's version group into its merges and a package's tables came to be +// read by the row each of its instances is built for (mcpp#956; SPEC-013 §7.5, +// SPEC-004 §4.8). + +module mcpp.build.prepare; +import :state; + +import std; +import mcpp.build.prepare_inputs; +import mcpp.manifest; +import mcpp.pm.dep_spec; +import mcpp.toolchain.triple; + +namespace mcpp::build { + +// What a request is bound by: the range as written and, where the platform's +// version group binds it, the group's version (SPEC-013 §7.5). +std::string request_constraint(const WorkItem& item) { + std::string out = item.originalConstraint.empty() ? std::string("*") : item.originalConstraint; + if (!item.groupConstraint.empty()) out += "," + item.groupConstraint; + return out; +} + +// The requests a record answers, each with its requester: the written range +// under who wrote it, the group's version under the platform that states it. +void note_request(const PrepareState& state, ResolvedRecord& r, const WorkItem& item) { + const std::string written = item.originalConstraint.empty() ? std::string("*") : item.originalConstraint; + r.constraint += (r.constraint.empty() ? "" : ",") + written; + r.constraintsBy.emplace_back(item.requestedBy, written); + if (item.groupConstraint.empty()) return; + const auto by = std::format("the version group of {}", state.platformGroupFrom); + for (auto const& [who, c] : r.constraintsBy) + if (who == by && c == item.groupConstraint) return; + r.constraint += "," + item.groupConstraint; + r.constraintsBy.emplace_back(by, item.groupConstraint); +} + +// Every request of a record still in the graph, for a refusal: who asked and +// for what, one per line. +std::string requests_of(const PrepareState& state, const ResolvedRecord& r) { + std::string out; + for (auto const& [by, c] : r.constraintsBy) { + if (state.supersededRequesters.contains(by)) continue; + out += std::format("\n '{}' by {}", c, by); + } + return out; +} + +// A PACKAGE'S DEPENDENCY TABLE IS READ BY THE ROW IT IS BUILT FOR (SPEC-004 +// §4.8; mcpp#956). A package reached only through build dependencies is built +// for this machine --- it is a module of a build program --- and its +// `[target..dependencies]` were read by the build's target: crossing +// to macOS, a library of a build program asked for the macOS implementation of +// its kernel layer, which no program of the build links, while the Linux one +// its build program does link was resolved outside the walk and absent from +// the lock. A package both sides ask for is built for each, and each side +// reads the table by its own row. + +// The row the build programs are built for: this machine's, or the target the +// host graph's platform serves on it (SPEC-013 §6). +static std::expected host_row(PrepareState& state) { + if (!state.hostPlatformSel.none) { + if (auto r = load_host_platform(state); !r) return std::unexpected(r.error()); + if (!state.hostPlatformTarget.empty()) + return cfgpred::context_for(state.hostPlatformTarget, + state.hostPlatformSel.ref.spec.shortName); + } + return cfgpred::context_for(mcpp::toolchain::triple::host_triple().str()); +} + +// One of a package's dependency tables as the host row reads it. The stored +// table is the target's reading: the tables as written, then each conditional +// table whose selector the target matches. The host's reading differs by the +// conditional tables only one of the two rows matches. A selector that names a +// layer is left as the target read it: the host row's layers are chosen when +// the build programs' own graph is prepared, which reads those itself. +static DependencyTable +host_row_table(const mcpp::manifest::Manifest& dm, const cfgpred::Ctx& target, const cfgpred::Ctx& host, + DependencyTable mcpp::manifest::Manifest::* table, + DependencyTable mcpp::manifest::ConditionalConfig::* conditional) { + auto out = dm.*table; + if (!dm.beforeConditionalMerge) return out; + auto const& written = (*dm.beforeConditionalMerge).*table; + for (auto const& cc : dm.conditionalConfigs) { + if (cfgpred::uses_layer(cc.predicate)) continue; + if (!cfgpred::matches(cc.predicate, target) || cfgpred::matches(cc.predicate, host)) continue; + for (auto const& [key, spec] : cc.*conditional) { + std::erase_if(out, [&](auto const& entry) { + return entry.first == key || same_dependency_identity(entry.second, spec); + }); + if (auto was = written.find(key); was != written.end()) out[key] = was->second; + } + } + for (auto const& cc : dm.conditionalConfigs) { + if (cfgpred::uses_layer(cc.predicate)) continue; + if (cfgpred::matches(cc.predicate, target) || !cfgpred::matches(cc.predicate, host)) continue; + replace_dependencies(out, cc.*conditional); + } + return out; +} + +// The requests one side makes of a package's dependencies: the tables as that +// side's row reads them, and for each entry whether the other side's row has +// it. An entry of this machine's reading alone is a request of the package's +// build-program instance, so its edge is a build-table edge; an entry of the +// target's reading alone is marked so that the build program does not take it. +std::expected +side_requests(PrepareState& state, const mcpp::manifest::Manifest& dm, bool hostSide) { + SideRequests out{ dm.dependencies, dm.buildDependencies, {}, {} }; + if (dm.conditionalConfigs.empty() || !dm.beforeConditionalMerge) return out; + auto host = host_row(state); + if (!host) return std::unexpected(host.error()); + const auto target = state.cfgCtx(); + if (host->triple == target.triple && host->platformPackage == target.platformPackage) return out; + const auto hostDependencies = host_row_table(dm, target, *host, &mcpp::manifest::Manifest::dependencies, + &mcpp::manifest::ConditionalConfig::dependencies); + for (auto const& [key, _] : hostDependencies) + if (!dm.dependencies.contains(key)) out.ofThisMachineAlone.insert(key); + for (auto const& [key, _] : dm.dependencies) + if (!hostDependencies.contains(key)) out.ofTheTargetAlone.insert(key); + if (hostSide) { + out.dependencies = hostDependencies; + out.buildDependencies = host_row_table(dm, target, *host, &mcpp::manifest::Manifest::buildDependencies, + &mcpp::manifest::ConditionalConfig::buildDependencies); + } + return out; +} + +// One request of a side, as a work item. +WorkItem side_item(const SideRequests& side, const std::string& childName, + mcpp::manifest::DependencySpec childReq, const std::string& label, + std::size_t consumer, const std::filesystem::path& root, const WorkItem& item) { + const bool ofThisMachine = side.ofThisMachineAlone.contains(childName); + WorkItem out{childName, childReq, label, childReq.version, consumer, root, item.devOnly, + item.buildOnly || ofThisMachine, /*fromBuildTable=*/ofThisMachine}; + out.targetRowOnly = !item.buildOnly && side.ofTheTargetAlone.contains(childName); + return out; +} + +} // namespace mcpp::build diff --git a/src/build/prepare/host_graph.cpp b/src/build/prepare/host_graph.cpp index 5053e843b..d5a6f8a1f 100644 --- a/src/build/prepare/host_graph.cpp +++ b/src/build/prepare/host_graph.cpp @@ -43,6 +43,7 @@ import mcpp.build.source_digest; import mcpp.build.tool_store; import nlohmann.json; import mcpp.manifest; +import mcpp.config; // resolve_project_index_path import mcpp.modgraph.scanner; import mcpplibs.os.platform; import mcpp.home.files; @@ -579,7 +580,13 @@ void write_group(PrepareState& state, const fs::path& dir, const std::vectorindices) { std::vector fields; - if (idx.is_local()) fields.push_back("path = " + toml_string(path_text(idx.path))); + // Where the project's manifest places it: a relative path is + // relative to that manifest, and this one is written elsewhere. As + // written, the build programs' graph looked for the index beside + // itself and found no package of it (measured with mcpp#956). + if (idx.is_local()) + fields.push_back("path = " + toml_string(path_text( + mcpp::config::resolve_project_index_path(*state.root, idx)))); if (!idx.url.empty()) fields.push_back("url = " + toml_string(idx.url)); if (!idx.rev.empty()) fields.push_back("rev = " + toml_string(idx.rev)); if (!idx.tag.empty()) fields.push_back("tag = " + toml_string(idx.tag)); diff --git a/src/build/prepare/platform.cpp b/src/build/prepare/platform.cpp index 251e3dd5b..ed97a77a1 100644 --- a/src/build/prepare/platform.cpp +++ b/src/build/prepare/platform.cpp @@ -1020,7 +1020,17 @@ std::expected step4b_capability_defaults(PrepareState& state) // request admits it, and is refused naming both when it does not. A package // built for this machine only, and a development tree (a path or a // repository), are not bound by it, as the check after the walk leaves them. -std::expected step4b_pin_to_platform_group(PrepareState& state, const WorkItem& item, +// +// THE CONSTRAINT ENTERS THE MERGE (mcpp#956). The request was rewritten to the +// group's version and the rewriting stopped there: a second request of the +// same package was merged with the constraints as written, which the group's +// version was not among. A build program's request, which the group does not +// bind, took the newest version its range admits; the platform's layer then +// arrived, the merge found the kept version within both written ranges, and +// the check after the walk refused a version the resolver itself had chosen. +// One version of a package serves every request of it, and the group's +// constraint is one of those requests. +std::expected step4b_pin_to_platform_group(PrepareState& state, WorkItem& item, const std::string& name, mcpp::manifest::DependencySpec& spec) { if (!state.platformGroup.empty() && !spec.isPath() && !spec.isGit() && !item.buildOnly) { @@ -1045,7 +1055,13 @@ std::expected step4b_pin_to_platform_group(PrepareState& stat shortName, asked, item.requestedBy, state.platformGroupFrom, shortName, g->second, state.platformPackageShortName, g->second, shortName, g->second)); } - if (group) spec.version = "=" + g->second; + if (group) { + spec.version = "=" + g->second; + // The group's choice is this request's constraint from here on: + // the record of the package keeps it, and a later request the + // group does not bind is merged with it. + item.groupConstraint = spec.version; + } } } return {}; diff --git a/src/build/prepare/state.cppm b/src/build/prepare/state.cppm index 92b2782bd..231c69700 100644 --- a/src/build/prepare/state.cppm +++ b/src/build/prepare/state.cppm @@ -158,6 +158,11 @@ constexpr std::size_t kMainConsumer = static_cast(-1); // not the consumer's own sources. bool devTable = false; bool wantsLibrary = false; + // The edge is of the target's reading of the consumer's table alone + // (SPEC-004 §4.8): the consumer's instance in a build program does not + // have it. An edge of this machine's reading alone is a `buildTable` + // edge, which the target's instance does not have. + bool targetRowOnly = false; bool reexport = false; // `embed = true` and its `exclude` (design 2026-11-02 §14.1): the // dependency's sources are compiled into the consumer. @@ -260,6 +265,18 @@ constexpr std::size_t kMainConsumer = static_cast(-1); // The root's override answered this request (T2): its source is the // root's, and the platform's version group does not bind it. bool overridden = false; + // Last, since items are written positionally. `=` where the platform's version group binds this request + // (SPEC-013 §7.5). It is a constraint of the request as the written one + // is, so it enters the resolution record and every later merge: a + // request of the same package that the group does not bind (a build + // program's) is then resolved with it, in whichever order the two + // arrive (mcpp#956). + std::string groupConstraint; + // The request is of the target's reading of its consumer's table + // alone: a `[target..dependencies]` entry the row of this + // machine does not match (SPEC-004 §4.8). Its edge does not make the + // dependency a module of a build program. + bool targetRowOnly = false; }; struct DeclaringManifest { @@ -835,6 +852,11 @@ struct PrepareState { // version that had already left (os 0.4.4 and openkal-linux 0.26.0 after // both were merged down, measured). std::set supersededRequesters; + // The sides a package's dependency table has been walked for, by dep slot + // (SPEC-004 §4.8; mcpp#956): 1 the target's, 2 this machine's. A package is + // built once for each side that asks for it, and each side reads the table + // by its own row and makes its own requests. + std::vector depSidesWalked; // THE ROOT'S OVERRIDES (design 2026-11-10 supplement, T2): each // `override = true` declaration of the root's tables and of // `[workspace.dependencies]`, with the identity it answers for and the @@ -1053,7 +1075,7 @@ std::size_t packageIndexForConsumer(std::size_t consumerDepIndex); void recordDependencyEdge(PrepareState& state, std::size_t consumerDepIndex, std::size_t dependencyPackageIndex, const mcpp::manifest::DependencySpec& spec, bool buildTable, - const std::string& writtenKey); + const std::string& writtenKey, bool targetRowOnly = false); // graph_edges.cpp: the package being built stands for itself in its tests' // graph -- an edge reached through its `[dev-dependencies]` naming it is its // own, recorded here; true when the request is so answered. @@ -1360,7 +1382,25 @@ std::expected step4b_apply_platform(PrepareState& state); // added to the walk. std::expected step4b_capability_defaults(PrepareState& state); std::expected step4b_check_platform_versions(PrepareState& state); -std::expected step4b_pin_to_platform_group(PrepareState& state, const WorkItem& item, +// graph_requests.cpp: what a request is bound by (the range as written and the +// platform's version group), the requests a resolution record answers, and +// the requests each side of the build makes of a package's dependency tables +// (SPEC-013 §7.5, SPEC-004 §4.8). +enum : unsigned char { kTargetSide = 1, kHostSide = 2 }; +using DependencyTable = std::map; +struct SideRequests { + DependencyTable dependencies, buildDependencies; + std::set ofThisMachineAlone, ofTheTargetAlone; +}; +std::string request_constraint(const WorkItem& item); +void note_request(const PrepareState& state, ResolvedRecord& r, const WorkItem& item); +std::string requests_of(const PrepareState& state, const ResolvedRecord& r); +std::expected side_requests(PrepareState& state, + const mcpp::manifest::Manifest& dm, bool hostSide); +WorkItem side_item(const SideRequests& side, const std::string& childName, + mcpp::manifest::DependencySpec childReq, const std::string& label, + std::size_t consumer, const std::filesystem::path& root, const WorkItem& item); +std::expected step4b_pin_to_platform_group(PrepareState& state, WorkItem& item, const std::string& name, mcpp::manifest::DependencySpec& spec); void step4b_read_platform_group(PrepareState& state, const std::string& resolvedShortName); diff --git a/tests/e2e/openkal/platforms.cpp b/tests/e2e/openkal/platforms.cpp index c9c8ad556..9ca17be57 100644 --- a/tests/e2e/openkal/platforms.cpp +++ b/tests/e2e/openkal/platforms.cpp @@ -192,6 +192,96 @@ int main() { expect(fatal(e2e::succeeded(r))) << "C: a development tree was bound by the group: " << s::text(r); }; + "956 the version group binds one version for every request of a package"_test[s::unix_shell][s::gcc][s::fresh_sandbox] = [] { + // mcpp#956, SPEC-013 §7.5. The group's version is a constraint of the request it binds, + // and it enters the merge: a build program's request of the same package, which the + // group does not bind, is resolved with it in whichever order the two arrive. A the + // target's request first; B the build program's first (the kept version is replaced and + // a `Merged` line says so); C a package only a build program uses is not bound. + const auto tmp = s::case_dir(); + s::project(tmp / "plat", {{"mcpp.toml", "[package]\nname = \"group-platform\"\nversion = \"0.1.0\"\n\n[platform]\ntargets = [\"x86_64-linux-gnu\", \"aarch64-linux-gnu\", \"aarch64-macos\", \"x86_64-macos\"]\n\n[platform.versions]\nfam = \"1.1.0\"\n"}}); + const auto library = [&](std::string_view name, std::string_view dependencies) { + s::project(tmp / name, {{"mcpp.toml", std::format("[package]\nname = \"{0}\"\nversion = \"0.1.0\"\n\n[build]\nsources = [\"src/{0}.cppm\"]\n\n[targets.{0}]\nkind = \"lib\"\n\n{1}", name, dependencies)}, + {std::format("src/{}.cppm", name), std::format("export module {};\nexport int {}_value() {{ return 1; }}\n", name, name)}}); + }; + const std::string fam = "[indices]\nacme = { path = \"../proj/local-index\" }\n\n[dependencies.acme]\nfam = \"^1.0.0\"\n"; + library("tool", fam); // a build program's library + library("inner", fam); // reached by the target two edges down + library("mid", "[dependencies]\ninner = { path = \"../inner\" }\n"); + const std::string_view unit = "export module fam;\nexport int fam_value() { return 0; }\n"; + s::project proj(tmp / "proj", {{"local-index/pkgs/a/acme.fam.lua", R"(package = { + spec = "1", namespace = "acme", name = "acme.fam", + description = "version-group fixture", licenses = {"MIT"}, type = "package", + xpm = { + linux = { ["1.2.0"] = { url = "u", sha256 = "0" }, ["1.1.0"] = { url = "u", sha256 = "0" }, ["1.0.0"] = { url = "u", sha256 = "0" } }, + macosx = { ["1.2.0"] = { url = "u", sha256 = "0" }, ["1.1.0"] = { url = "u", sha256 = "0" }, ["1.0.0"] = { url = "u", sha256 = "0" } }, + }, + mcpp = { language = "c++23", import_std = false, sources = { "src/fam.cppm" }, + targets = { ["fam"] = { kind = "lib" } }, deps = {} }, +} +)"}, + {".mcpp/.xlings/data/xpkgs/acme.fam/1.1.0/src/fam.cppm", std::string(unit)}, + {".mcpp/.xlings/data/xpkgs/acme.fam/1.2.0/src/fam.cppm", std::string(unit)}, + {"build.mcpp", "import std;\nimport mcpp;\nint main() { return 0; }\n"}, + {"src/main.cpp", "int main() { return 0; }\n"}}); + const auto use = [&](std::string_view dependencies) { + proj.write("mcpp.toml", std::format("[package]\nname = \"proj\"\nversion = \"0.1.0\"\n\n[indices]\nacme = {{ path = \"local-index\" }}\n\n[target.'cfg(any(linux, macos))']\nplatform = {{ path = \"../plat\" }}\n\n{}\n[build-dependencies]\ntool = {{ path = \"../tool\" }}\n\n[targets.proj]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n", dependencies)); + fs::remove(proj.at("mcpp.lock")); + return proj.mcpp({"build", "--configure-only"}, inherited); + }; + // A: the target's request is walked first, the build program's finds the group's version kept. + auto r = use("[dependencies.acme]\nfam = \"^1.0.0\"\n"); + expect(fatal(e2e::succeeded(r))) << "A: " << s::text(r); + expect(fatal(e2e::contains(s::text(r), "fam =1.1.0 → v1.1.0"))) << "A: the group's version was not taken: " << s::text(r); + expect(fatal(e2e::contains(proj.read("mcpp.lock"), "version = \"1.1.0\""))) << "A: the lock does not record the group's version: " << proj.read("mcpp.lock"); + // B: the build program's request is walked first and takes the newest; the target's replaces it. + r = use("[dependencies]\nmid = { path = \"../mid\" }\n"); + expect(fatal(e2e::succeeded(r))) << "B: a version the resolver chose was refused: " << s::text(r); + expect(fatal(s::has_line(s::text(r), R"(Merged acme\.fam 1\.2\.0 ⨯ 1\.1\.0 → v1\.1\.0)"))) << "B: the two requests were not merged: " << s::text(r); + expect(fatal(!e2e::contains(proj.read("mcpp.lock"), "version = \"1.2.0\""))) << "B: the lock records the replaced version: " << proj.read("mcpp.lock"); + // C: only the build program uses the package, and the group does not bind it. + r = use(""); + expect(fatal(e2e::succeeded(r))) << "C: " << s::text(r); + expect(fatal(e2e::contains(s::text(r), "fam ^1.0.0 → v1.2.0"))) << "C: a package of a build program alone was bound by the group: " << s::text(r); + }; + + "956 a build program's library reads its tables by this machine's row"_test[s::musl][s::elf][s::fresh_sandbox] = [] { + // mcpp#956, SPEC-004 §4.8. A package reached only through build dependencies is a + // module of a build program, built for this machine: under `--target x86_64-linux-musl` + // its `[target..dependencies]` are read by the row of this machine, the + // package the target's row would have named is not resolved, and the lock records the + // one the build program links. + const auto musl_version = s::musl_gcc_version(); + if (!fs::exists(s::payload_root("xim-x-musl-gcc", musl_version) / "bin/x86_64-linux-musl-g++")) + skip("no musl-gcc payload in this home: not crossing to *-linux-musl"); + const auto tmp = s::case_dir(); + const auto descriptor = [](std::string_view name) { + return std::format(R"(package = {{ + spec = "1", namespace = "acme", name = "acme.{0}", + description = "row fixture", licenses = {{"MIT"}}, type = "package", + xpm = {{ linux = {{ ["1.0.0"] = {{ url = "u", sha256 = "0" }} }} }}, + mcpp = {{ language = "c++23", import_std = false, sources = {{ "src/{0}.cppm" }}, + targets = {{ ["{0}"] = {{ kind = "lib" }} }}, deps = {{}} }}, +}} +)", name); + }; + s::project(tmp / "tool", {{"mcpp.toml", "[package]\nname = \"tool\"\nversion = \"0.1.0\"\n\n[build]\nsources = [\"src/tool.cppm\"]\n\n[targets.tool]\nkind = \"lib\"\n\n[indices]\nacme = { path = \"../proj/local-index\" }\n\n[target.'cfg(env = \"gnu\")'.dependencies.acme]\nongnu = \"1.0.0\"\n\n[target.'cfg(env = \"musl\")'.dependencies.acme]\nonmusl = \"1.0.0\"\n"}, + {"src/tool.cppm", "export module tool;\nexport int tool_value() { return 1; }\n"}}); + s::project proj(tmp / "proj", { + {"local-index/pkgs/a/acme.ongnu.lua", descriptor("ongnu")}, + {"local-index/pkgs/a/acme.onmusl.lua", descriptor("onmusl")}, + {".mcpp/.xlings/data/xpkgs/acme.ongnu/1.0.0/src/ongnu.cppm", "export module ongnu;\nexport int ongnu_value() { return 0; }\n"}, + {".mcpp/.xlings/data/xpkgs/acme.onmusl/1.0.0/src/onmusl.cppm", "export module onmusl;\nexport int onmusl_value() { return 0; }\n"}, + {"build.mcpp", "import std;\nimport mcpp;\nint main() { return 0; }\n"}, + {"src/main.cpp", "int main() { return 0; }\n"}, + {"mcpp.toml", std::format("[package]\nname = \"proj\"\nversion = \"0.1.0\"\n\n[indices]\nacme = {{ path = \"local-index\" }}\n\n[target.x86_64-linux-musl]\ntoolchain = \"gcc@{}-musl\"\nlinkage = \"static\"\n\n[build-dependencies]\ntool = {{ path = \"../tool\" }}\n\n[targets.proj]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n", musl_version)}}); + auto r = proj.mcpp({"build", "--target", "x86_64-linux-musl", "--configure-only"}, inherited); + expect(fatal(e2e::succeeded(r))) << s::text(r); + const auto lock = proj.read("mcpp.lock"); + expect(fatal(e2e::contains(lock, "acme.ongnu"))) << "the row of this machine was not read, or the lock does not record what the build program links: " << lock; + expect(fatal(!e2e::contains(lock, "acme.onmusl"))) << "the target's row was read for a build program's library: " << lock; + }; + "1021 a platform series is locked and skips newer engines"_test[s::unix_shell][s::gcc][s::fresh_sandbox] = [] { // A platform selected as a series (`acme.plat@^0`): A it takes its newest release whose // `[package] mcpp` floor this mcpp meets, naming the one it skips, and the lock records From cfc595c5a4b107249c2066974a60e9574e5dd5c5 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 12:58:46 +0900 Subject: [PATCH 04/30] Whether an artifact runs here is one predicate with no branch before it (mcpp#957) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the branch that refused macOS and iOS rows off macOS, whatever runner was declared, is removed: those rows are decided by the row's execution system and the runner or execution host stated for it, as every row is - the system-names check counts a triple's system predicates, so a branch of that kind raises a count above its baseline - SPEC-016 §6, docs/04 in both languages, unit tests of the predicate --- .github/tools/check_system_names.py | 9 +++++-- .github/tools/system-names.baseline | 8 +++++- CHANGELOG.md | 7 +++++ docs/04-mcpp-toml.md | 12 ++++++--- docs/specs/targets-and-providers.md | 11 +++++--- docs/zh/04-mcpp-toml.md | 8 ++++-- src/build/execute.cppm | 19 ++++++-------- tests/unit/test_execution_reach.cpp | 40 +++++++++++++++++++++++++++++ 8 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 tests/unit/test_execution_reach.cpp diff --git a/.github/tools/check_system_names.py b/.github/tools/check_system_names.py index e59c48848..f9ca43555 100644 --- a/.github/tools/check_system_names.py +++ b/.github/tools/check_system_names.py @@ -16,7 +16,11 @@ `modules/*/src/`, without `modules/platform/` and without tests and data -- in the two forms a branch on a system takes: the word as a string literal (`"freebsd"`), and the host predicates of mcpplibs.os (`is_linux`, -`is_windows`, `is_macos`). Comments are not counted. +`is_windows`, `is_macos`) and the system predicates of a triple (`is_apple`, +`is_ios`, `is_android` and their like). Comments are not counted. The triple's +predicates were not in the vocabulary until a branch on `is_apple()` was found +standing before the one predicate that decides whether an artifact runs here +(mcpp#957). THE RATCHET @@ -39,7 +43,8 @@ LITERALS = ["linux", "macos", "windows", "freebsd", "android", "emscripten", "ios", "darwin", "netbsd", "openbsd", "haiku", "dos", "uefi"] -PREDICATES = ["is_linux", "is_windows", "is_macos"] +PREDICATES = ["is_linux", "is_windows", "is_macos", + "is_apple", "is_ios", "is_ios_simulator", "is_android", "is_openkal", "is_windows_gnu"] SOURCE_SUFFIXES = {".cppm", ".cpp", ".mcpp"} diff --git a/.github/tools/system-names.baseline b/.github/tools/system-names.baseline index bf2ab5414..436f8eea5 100644 --- a/.github/tools/system-names.baseline +++ b/.github/tools/system-names.baseline @@ -15,4 +15,10 @@ dos 0 uefi 0 is_linux 26 is_windows 126 -is_macos 32 +is_macos 31 +is_apple 5 +is_ios 7 +is_ios_simulator 6 +is_android 9 +is_openkal 3 +is_windows_gnu 2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ee34219..b74d7f5be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,13 @@ `[indices] x = { path = "" }` was copied as written into a manifest in another directory, and a build dependency from that index was `no package found for exact selector`. The path is written as the project's manifest resolves it. +- A macOS or iOS row on a host that is not macOS was refused by `mcpp run` and `mcpp test` + before the predicate that decides whether an artifact runs here, whatever runner was + declared for it: `cannot run ... artifacts on this host: macOS runs only on macOS` + (mcpp#957). The branch named a system in the engine and is removed; these rows are + decided as every row is, by the row's execution system and the runner or execution host + stated for it (SPEC-016 §6). The system-names check counts a triple's system predicates + (`is_apple`, `is_ios`, `is_android` and their like), so such a branch does not return. - `mcpp new --template ` refuses a package for the directory it lacks, `templates/` beside its manifest, and names the directory it looked for. It refused an inline descriptor for its form, and found the package root by a rule of its own that diff --git a/docs/04-mcpp-toml.md b/docs/04-mcpp-toml.md index e49f5d95b..2469bb579 100644 --- a/docs/04-mcpp-toml.md +++ b/docs/04-mcpp-toml.md @@ -1555,9 +1555,15 @@ A runner reached this way receives `MCPP_RUN_TRIPLE`, `MCPP_RUN_KIND` (`exe` / `bundle`), `MCPP_RUN_MODE` (`run` / `test`), `MCPP_RUN_ID`, `MCPP_RUN_CLOSURE` (a versioned JSON naming the artifact, the files it carries, the run's files (`run_files`, schema 2, 2026.11.17.1+) and the base) and, -when a base is stated, `MCPP_SYSROOT`. Its exit status is the program's. An -Apple artifact on a host that is not macOS, with no `run_under`, is refused -before anything is spawned, naming this key. +when a base is stated, `MCPP_SYSROOT`. Its exit status is the program's. + +Whether an artifact runs on this machine is one question for every row +(SPEC-016 §6): its execution system is this machine's, or a runner or an +execution host is stated for it. An artifact of another system with neither is +refused before anything is spawned, naming `runner` and `run_under`. A macOS or +iOS artifact on a host that is not macOS is such an artifact and no more: with a +runner whose `hosts` include this machine it runs through that runner +*(mcpp 2026.11.19.1+; before, such a row was refused whatever runner was declared)*. #### 2.7.3.3 The platform keys (2026.10.27.1+) diff --git a/docs/specs/targets-and-providers.md b/docs/specs/targets-and-providers.md index 0e105a6c8..11c562bed 100644 --- a/docs/specs/targets-and-providers.md +++ b/docs/specs/targets-and-providers.md @@ -4,9 +4,9 @@ |---|---| | 规范编号 | SPEC-016 | | 标题 | 目标与提供者:行、提供者、选择、平台定义的行、报告与锁 | -| 状态 | 评审中 v1.5 | -| 版本 | 1.5 | -| 最后修改 | 2026-11-18 | +| 状态 | 评审中 v1.6 | +| 版本 | 1.6 | +| 最后修改 | 2026-11-19 | | 对应实现 | mcpp >= 2026.11.9.1;v1.1 条款 mcpp >= 2026.11.10.1;v1.2 条款 mcpp >= 2026.11.12.1;v1.3 条款 mcpp >= 2026.11.16.1;v1.4 条款 mcpp >= 2026.11.17.1;v1.5 条款 mcpp >= 2026.11.18.1 | | 相关设计文档 | `.agents/docs/2026-11-15-ecosystem-gap-supplement-design.md`、`.agents/docs/2026-11-09-target-model-design.md`、`.agents/docs/2026-11-10-cross-platform-ecosystem-design.md`、`.agents/docs/2026-11-12-ecosystem-architecture-optimization-design.md` | | 相关 issue | mcpp#911、mcpp#913 | @@ -203,6 +203,10 @@ 文件读取其闭包,不运行本机加载器(SPEC-014,v1.1)。 - **一个谓词(v1.2)。** 「产物是否由本机运行」只有两个问法,各在一处回答,内置行与平台定义的行相同:执行系统是否为本机 系统(启动之前问);执行系统与架构是否都是本机的(检查加载器与运行时闭包之前问)。 +- **谓词之前没有按系统的分支(v1.6)。** 实现**禁止**在该谓词之前以某个系统的名字拒绝一行。此前 Apple 的行在非 macOS + 宿主上先于谓词被拒绝,不看是否声明了运行器,平台为这样的行陈述的运行器因此从未被读取(mcpp#957);该分支已删除, + 这些行与其余的行由同一个谓词判定。`check_system_names.py` 的词表含三元组的系统谓词(`is_apple`、`is_ios` 等), + 新的这类分支使计数高于基线而被拒绝。 - **不启动另一系统的程序(v1.2)。** 执行系统不是本机系统、又没有运行器与执行主机时,`mcpp run` 与 `mcpp test` 在启动任何 进程之前回答「已构建,未运行」,写出两个系统与可写的声明;退出码为 2。本机内核不读取程序所属的系统,另一系统的系统调用 号在本机另有含义,所以启动不是观察。`--no-runner` 陈述该身份在本机可直接执行,照此执行。同一系统、另一架构的产物照旧 @@ -260,3 +264,4 @@ | 1.5 | 2026-11-18 | mcpp 2026.11.18.1(设计 2026-11-17 §12):`object-format`、`exe-suffix`、`shared-suffix` 的三级读取(行、系统、格式的约定);系统表的这三个键;内置行的命名不取宿主的 | | 1.4 | 2026-11-17 | mcpp 2026.11.17.1(设计 2026-11-16 N48):行的 `exe-suffix-fixed` | | 1.3 | 2026-11-16 | mcpp 2026.11.16.1(设计 2026-11-15 G7、N16、N23、N26、N33、R1、R2、X1):执行关系表(§2.4);系统表的 `resources` 与 `command-scripts`(§2.3);提供者运行器的 `host`、`enabled` 与 `exclusive`(§6);Android 行的运行器(G7);行的 `exceptions`(工程未陈述时的异常模型);行的 `companions`;平台以工具链列出的 planned 行可作载体的行 | +| 1.6 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#957):§6 执行可达性的谓词之前没有按系统的分支;Apple 的行由同一个谓词判定;系统名检查的词表含三元组的系统谓词 | diff --git a/docs/zh/04-mcpp-toml.md b/docs/zh/04-mcpp-toml.md index 7c713478d..fa06de56f 100644 --- a/docs/zh/04-mcpp-toml.md +++ b/docs/zh/04-mcpp-toml.md @@ -1424,8 +1424,12 @@ runner 与 `run_under`。 经此到达的运行器收到 `MCPP_RUN_TRIPLE`、`MCPP_RUN_KIND`(`exe` / `bundle`)、`MCPP_RUN_MODE` (`run` / `test`)、`MCPP_RUN_ID`、`MCPP_RUN_CLOSURE`(一份带版本的 JSON,列出产物、它随带的文件、本次运行的文件 (`run_files`,schema 2,2026.11.17.1+)与基座), -以及陈述了基座时的 `MCPP_SYSROOT`;其退出码即程序的退出码。非 macOS 宿主上的 Apple 产物在没有 -`run_under` 时于执行前被拒绝,拒绝信息点名本键。 +以及陈述了基座时的 `MCPP_SYSROOT`;其退出码即程序的退出码。 + +产物是否由本机运行,对每一行是同一个问题(SPEC-016 §6):它的执行系统是本机的系统,或者为它陈述了运行器或执行主机。 +两者都没有的另一系统的产物在启动任何进程之前被拒绝,拒绝信息点名 `runner` 与 `run_under`。非 macOS 宿主上的 macOS 或 +iOS 产物就是这样的产物,没有别的规则:运行器的 `hosts` 含本机时,它经该运行器运行*(mcpp 2026.11.19.1+;此前这样的行不论 +声明了什么运行器都被拒绝)*。 #### 2.7.3.3 平台键(2026.10.27.1+) diff --git a/src/build/execute.cppm b/src/build/execute.cppm index b02b9b076..184699023 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -1522,19 +1522,16 @@ ExecutionHost locate_run_under(const mcpp::build::BuildContext& ctx, std::string // a way. One predicate for every row, built-in or defined: the row's execution // system against this machine's. `--no-runner` states that the triple is // native here, and is believed. -std::optional unreachable_execution(const mcpp::build::BuildContext& ctx, - std::string_view tripleKey, bool noRunner, - bool runnerDeclared = false) { +export std::optional unreachable_execution(const mcpp::build::BuildContext& ctx, + std::string_view tripleKey, bool noRunner, + bool runnerDeclared = false) { if (noRunner || !ctx.runUnder.empty()) return std::nullopt; auto t = mcpp::toolchain::triple::parse(tripleKey); - // An Apple artifact off macOS: no runner reaches it, only an execution host. - if (!mcpplibs::os::is_macos && t && t->is_apple()) - return std::format( - "cannot run {} artifacts on this host: {} runs only on macOS.\n" - " Name an execution host: [target.{}] run_under = [\"remote-run\", \"\", \"--\"]\n" - " (in mcpp.toml for the team, or ~/.mcpp/config.toml for this machine).", - tripleKey, t->is_ios_simulator() ? "the iOS Simulator" : t->is_ios() ? "iOS" : "macOS", - tripleKey); + // No system is named here (mcpp#957). A branch for Apple's rows stood + // before the predicate below and refused them off macOS whether or not a + // runner was declared, so a runner a platform states for such a row was + // never consulted. The predicate gives those rows the same refusal where + // nothing runs them and admits them where a runner does. // AN ARTIFACT OF ANOTHER SYSTEM IS NOT STARTED TO SEE WHAT HAPPENS (SPEC-016 // §6). A row's artifacts run on its execution system -- the `execution` its // row states, built-in or defined, else its own -- and that is this diff --git a/tests/unit/test_execution_reach.cpp b/tests/unit/test_execution_reach.cpp new file mode 100644 index 000000000..7d266d4ce --- /dev/null +++ b/tests/unit/test_execution_reach.cpp @@ -0,0 +1,40 @@ +// Whether an artifact runs here is one predicate for every row (SPEC-016 §6; +// mcpp#957). A branch for Apple's rows stood before it and refused them off +// macOS whether or not a runner was declared; these cases hold the predicate +// to the rows that branch covered. +import std; +import mcpplibs.mtest; +import mcpp.build.execute; +import mcpp.build.prepare; +import mcpp.toolchain.triple; + +using namespace mcpplibs::mtest; + +UTest a_declared_runner_reaches_a_row_of_another_system{"ExecutionReach/ADeclaredRunnerReachesARowOfAnotherSystem", [] { + const mcpp::build::BuildContext ctx; + for (std::string_view row : {"aarch64-macos", "x86_64-macos", "aarch64-ios", "aarch64-ios-sim"}) { + // A runner is declared for the row: it is what runs the artifact, on any host. + expect(!mcpp::build::unreachable_execution(ctx, row, false, true).has_value()) + << row << ": a declared runner was not consulted"; + // `--no-runner` states that the row is native here, and is believed. + expect(!mcpp::build::unreachable_execution(ctx, row, true, false).has_value()) << row; + } +}}; + +UTest without_a_runner_a_row_of_another_system_is_built_not_run{"ExecutionReach/WithoutARunnerARowOfAnotherSystemIsBuiltNotRun", [] { + const mcpp::build::BuildContext ctx; + const auto host = mcpp::toolchain::triple::host_triple().os; + for (std::string_view row : {"aarch64-macos", "aarch64-ios", "x86_64-linux-gnu", "x86_64-windows-msvc"}) { + const auto t = mcpp::toolchain::triple::parse(row); + expect(fatal(t.has_value())) << row; + const auto why = mcpp::build::unreachable_execution(ctx, row, false, false); + if (t->system_is_host()) { + expect(!why.has_value()) << row << " is of this system: " << why.value_or(""); + continue; + } + expect(fatal(why.has_value())) << row << " is of another system and was not refused"; + // The refusal names the two systems from the row's data and the keys that add a way. + expect(why->contains(std::format("{} artifacts run on {}, not on this {} host", row, t->execution_system(), host))) << *why; + expect(why->contains("runner =") && why->contains("run_under =")) << *why; + } +}}; From 70a386b2df8a2f9530dfcc29b86dcf12c4ecb11a Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 13:40:57 +0900 Subject: [PATCH 05/30] tests: the published member's descriptor points at its manifest, which carries the sibling's version (mcpp#960) --- tests/e2e/workspace/inheritance.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/e2e/workspace/inheritance.cpp b/tests/e2e/workspace/inheritance.cpp index 9a34a1cf3..2297fcf48 100644 --- a/tests/e2e/workspace/inheritance.cpp +++ b/tests/e2e/workspace/inheritance.cpp @@ -280,7 +280,7 @@ int lib_c(void) { return LEVEL; } // the original as mcpp.toml.orig, reproducibly; a consumer of the unpacked archive (and // a released client, when $MCPP_BOOT names one) builds it; a versioned sibling edge is // published as a version edge, a path-only one is refused naming the version to add; - // `emit xpkg` reads the effective manifest; a package that needs no normalisation is + // `emit xpkg` points at the archived manifest; a package that needs no normalisation is // archived as `git archive HEAD`. const auto tmp = s::case_dir(); s::project repo(tmp / "repo", { @@ -355,7 +355,10 @@ cxxflags = ["-DWS_FLAG=1"] expect(fatal(s::lacks(lib_toml, "path"))) << "a path survived"; r = repo.mcpp_in("lib", {"emit", "xpkg"}, inherited); expect(fatal(e2e::succeeded(r))) << "emit xpkg: " << s::text(r); - expect(fatal(e2e::contains(s::text(r), R"(["probe.util"] = "0.3.0")"))) << "the descriptor does not list the sibling: " << s::text(r); + // mcpp#960: the descriptor points at the archived manifest, which carries the sibling's + // version edge (checked above); it states no dependency of its own. + expect(fatal(e2e::contains(s::text(r), R"(mcpp = "*/mcpp.toml")"))) << "the descriptor does not point at the manifest: " << s::text(r); + expect(fatal(s::lacks(s::text(r), R"(["probe.util"])"))) << "the descriptor states a dependency beside its manifest: " << s::text(r); r = repo.mcpp_in("edgeless", {"publish", "--dry-run", "--allow-dirty"}, inherited); // mcpp#947 (1): a sibling edge that states no version takes the sibling's own. expect(fatal(e2e::succeeded(r))) << "a path-only sibling edge was not published under the sibling's version: " << s::text(r); From 21febba5bb97cf9348529f9792304d4299e3d4ef Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 13:40:58 +0900 Subject: [PATCH 06/30] A link's inputs are the files its words name and the files its linker reports reading (mcpp#931) --- CHANGELOG.md | 10 ++ docs/04-mcpp-toml.md | 14 ++- docs/30-build-mcpp.md | 35 +++++-- docs/specs/manifest-semantics.md | 24 ++++- docs/zh/04-mcpp-toml.md | 7 +- docs/zh/30-build-mcpp.md | 23 ++++- src/build/compile_commands.cppm | 13 +-- src/build/flags.cppm | 152 ++++++++++++++++++++++++++----- src/build/ninja_backend.cppm | 5 + src/toolchain/probe.cppm | 56 ++++++++++++ tests/e2e/build/link.cpp | 104 +++++++++++++++++++++ tests/e2e/web/emscripten.cpp | 25 +++++ tests/unit/test_build_flags.cpp | 45 +++++++++ 13 files changed, 462 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b74d7f5be..038c34f0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,16 @@ `templates/` beside its manifest, and names the directory it looked for. It refused an inline descriptor for its form, and found the package root by a rule of its own that ignored the descriptor's pointer. +- A file a link flag names in an option's value was no input of the link: + `--js-library=`, `-Wl,--version-script=` and `-Wl,-force_load,` could + change and `mcpp build` reported nothing to do (mcpp#931). A link word's candidates are the + whole word, the part after `=`, each `,`-separated segment and, under the MSVC dialect, the + value of a `/` option; an absolute path that names a regular file or a file an action + declares is an input (SPEC-004 §8.1). Where the linker writes what it read + (`--dependency-file`: GNU ld from 2.35, gold, ld.lld), the link edge records it as a + compile edge records the compiler's report, so a library found through `-L`/`-l` relinks + the program when it changes. Whether a linker accepts the option is measured once and kept + in the probe cache. ## [2026.11.18.1] - 2026-11-18 diff --git a/docs/04-mcpp-toml.md b/docs/04-mcpp-toml.md index 2469bb579..5d3883ca1 100644 --- a/docs/04-mcpp-toml.md +++ b/docs/04-mcpp-toml.md @@ -644,10 +644,16 @@ bmi_schedule = "auto" # Module-edge scheduling: auto (= off) | on | private_fragment = "unit" # A private module fragment: unit (built as a unit of its own, default) | as-written (chapter 26) ``` -A word of `[build] ldflags` that names an existing file by absolute path (a -prebuilt archive, an object) is an input of every link that carries it: a -changed file relinks (mcpp 2026.10.31.1+). A relative word is read by the -linker against the build directory and is not tracked. +A file a word of `[build] ldflags` names by absolute path is an input of every +link that carries it, whether it is the whole word (a prebuilt archive, an +object), an option's value (`--js-library=`, +`-Wl,--version-script=`), a `,`-separated segment +(`-Wl,-force_load,`) or a response file and what it lists; on an ELF +target whose linker reports what it read, a library found through `-L`/`-l` is +one too. A changed input relinks (whole words mcpp 2026.10.31.1+, the rest +2026.11.19.1+); see [Files a link reads](30-build-mcpp.md#files-a-link-reads). +A relative word is read by the linker against the build directory and is not +tracked. #### Compile-flag syntax *(mcpp 2026.9.17.1+)* diff --git a/docs/30-build-mcpp.md b/docs/30-build-mcpp.md index b20ac583e..90628a9a7 100644 --- a/docs/30-build-mcpp.md +++ b/docs/30-build-mcpp.md @@ -776,12 +776,35 @@ If nothing in the build can receive the outputs (an archive-only package), mcpp reports a degradation: the edge is reachable only through a link, so with no link the command would never run and the build would say nothing. -> Naming a pre-built object in `[build].ldflags` also reaches the linker, and -> should not be used for anything the build produces: ldflags is a flat string -> in the link command, not a file in the graph, so nothing tracks it and editing -> it reports `ninja: no work to do`. For Windows resources specifically, use -> [`[resources]`](04-mcpp-toml.md) — -> `object` is the escape hatch for everything else. +> A pre-built object named in `[build].ldflags` also reaches the linker, and a +> file named there is an input of the link (see *Files a link reads* below), but +> the action's outputs are the files the build produces. For Windows resources +> specifically, use [`[resources]`](04-mcpp-toml.md) — `object` is the escape +> hatch for everything else. + +#### Files a link reads + +A link is run again when a file it reads changes. Its inputs come from two +sources (SPEC-004 §8.1): + +1. **Files a link word names.** For each word of the link line — `[build] + ldflags`, `mcpp:link-flag=` and what dependencies propagate — the candidates + are the whole word, the part after `=`, each `,`-separated segment and the + part after `=` within it, and, under the MSVC link dialect, the part after + the first `:` of a word beginning with `/`. A candidate is an input when it + is an absolute path naming an existing regular file or a file an action of + the build declares. `@` adds the response file and the absolute paths + it lists. `--js-library=`, `-Wl,--version-script=`, + `-Wl,-force_load,` and `/DEF:` are covered. +2. **The linker's report.** On an ELF target whose linker accepts + `--dependency-file=` (GNU ld from 2.35, gold, ld.lld), the link edge + records the files the linker read, as a compile edge records the headers the + compiler read: libraries found through `-L`/`-l` and files a linker script + pulls in with `INPUT` are covered. Whether the linker accepts the option is + measured once per linker and kept in the probe cache. + +A relative path and a file that neither exists when the build is planned nor is +declared by an action are not inputs, unless the linker reports them. **You must name the output files.** mcpp fixes the source set, the fingerprint and the module graph during prepare, so an output whose *name* is unknown diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index 7d96266d1..ded597946 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -5,7 +5,7 @@ | **规范编号** | SPEC-004 | | **标题** | `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约 | | **状态** | **草案(Draft)** | -| **版本** | 1.21 | +| **版本** | 1.22 | | **最后修改** | 2026-11-19 | | **最低实现版本** | 条件化形状:mcpp **2026.8.29.1**(`[target..build-dependencies]` 起齐备);目标轴:mcpp **2026.9.6.4** | | **作者/维护** | mcpp-community | @@ -558,6 +558,27 @@ feature-deps feature-xlings ← 限定词是门 **状态:已实现(mcpp 2026.9.17.1;`defines` 的集合语义 mcpp 2026.9.25.1;`ldflags` 与链接指令 mcpp 2026.9.26.2,#703)。** +### 8.1 链接的输入 + +链接边的输入由两部分组成:规划时由链接词点名的文件,与链接器报告它读取的文件。 + +1. **点名的文件。** 对链接的每个词取候选:整个词;第一个 `=` 之后的部分;以 `,` 分隔的每一段, + 以及段内第一个 `=` 之后的部分;MSVC 链接方言下以 `/` 开头的词中第一个 `:` 之后的部分。 + 候选以 `@` 开头时取其后的部分,并且该响应文件列出的绝对路径也是候选。一个候选是绝对路径, + 并且指向一个现有的普通文件或本次规划中某个动作声明的输出时,它是携带该词的每个链接的隐式 + 输入,静态库的归档除外。目录不是输入;相对路径不是输入。 +2. **链接器的报告。** 目标的对象格式为 ELF,且链接所运行的链接器接受 `--dependency-file=<文件>` + (GNU ld 2.35 起、gold、ld.lld)时,链接边**必须**以该选项取得链接器读取的文件,作为边的依赖 + 文件(ninja `depfile`、`deps = gcc`)。链接器是否接受由一次实测回答:经编译器驱动时链接一个 + 空的共享对象,直接运行链接器时询问 `--version`;所用的词是链接行中决定链接器的词(`--target`、 + `-fuse-ld`、`--ld-path`、`-B`、驱动配置、`-flavor`),答案按这些词与它们点名的文件保存。 + 报告覆盖按 `-l` 搜索找到的库与链接脚本中 `INPUT` 引入的文件。其余链接器(ld64、wasm-ld、 + lld-link、link.exe)只有第 1 条。 + +改写一个输入使携带它的链接重新运行;没有输入变化的构建不运行链接。 + +**状态:已实现(整词 mcpp 2026.10.31.1,#869;响应文件 mcpp 2026.11.2.2,#883;其余 mcpp 2026.11.19.1,#931)。** + ## 9. 工作空间继承与构建需求的作用域 1. 工作空间成员**必须**恰好接收一次 `[workspace.package]`、`[workspace.build]` 与 @@ -836,3 +857,4 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m | 1.19 | 2026-10-28 | mcpp 2026.10.27.1(mcpp#847):§2 增加平台平面(`[platform]`、`[host]`),平台键可出现在工程与机器默认中;§4.2 增加 `[target.]` 的平台键与 `[host] platform`(SPEC-013)。 | | 1.20 | 2026-11-17 | mcpp 2026.11.17.1(设计 2026-11-16 §4):新增 §15 运行时文件与发布文件:`deploy` 的条目语法(字符串、通配符、`name`)、固定目录规则与过期副本的移除;`license_files` 与其缺省值。 | | 1.21 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):新增 §4.8 包的依赖表由构建它的那一行读取;两侧各自发出请求,锁记录两侧。 | +| 1.22 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#931):新增 §8.1 链接的输入:链接词点名的文件(选项的值、`,` 分隔的段、MSVC `/` 选项的值、动作声明的输出)与链接器自己的报告。 | diff --git a/docs/zh/04-mcpp-toml.md b/docs/zh/04-mcpp-toml.md index fa06de56f..43263d206 100644 --- a/docs/zh/04-mcpp-toml.md +++ b/docs/zh/04-mcpp-toml.md @@ -604,8 +604,11 @@ bmi_schedule = "auto" # Module-edge scheduling: auto (= off) | on | private_fragment = "unit" # A private module fragment: unit (built as a unit of its own, default) | as-written (chapter 26) ``` -`[build] ldflags` 中以绝对路径指向一个已存在文件的词(预构建的归档、目标文件)是带有它的每次链接的输入:文件改变即重新 -链接(mcpp 2026.10.31.1+)。相对路径的词由链接器相对构建目录读取,不跟踪。 +`[build] ldflags` 的词以绝对路径点名的文件是带有它的每次链接的输入:整个词(预构建的归档、目标文件)、选项的值 +(`--js-library=<文件>`、`-Wl,--version-script=<文件>`)、以 `,` 分隔的段(`-Wl,-force_load,<文件>`),以及响应文件 +和它列出的文件;目标为 ELF 且链接器报告它读取的文件时,经 `-L`、`-l` 找到的库也是输入。输入改变即重新链接(整词 +mcpp 2026.10.31.1+,其余 2026.11.19.1+),见[链接读取的文件](30-build-mcpp.md#链接读取的文件)。相对路径的词由链接器 +相对构建目录读取,不跟踪。 #### 编译 flag 的写法 *(mcpp 2026.9.17.1+)* diff --git a/docs/zh/30-build-mcpp.md b/docs/zh/30-build-mcpp.md index c553e11c5..38a9ab2ad 100644 --- a/docs/zh/30-build-mcpp.md +++ b/docs/zh/30-build-mcpp.md @@ -655,10 +655,25 @@ stamp 不会喂给任何编译或链接边,只喂给下文说的那条 order-o 如果本次构建里没有任何东西能接收这些产物(纯静态库包),mcpp 会报一条 degradation: 这条边只能经由链接被达成,没有链接就意味着命令一次都不会跑,而构建什么都不说。 -> 把预编译对象写进 `[build].ldflags` 同样能到达链接器,但**不要**用它承载构建产物: -> ldflags 是链接命令里的一串字符、不是图里的文件,没有任何东西跟踪它,改了它得到的是 -> `ninja: no work to do`。Windows 资源请用 [`[resources]`](04-mcpp-toml.md); -> `object` 是其余一切的出口。 +> 写进 `[build].ldflags` 的预编译对象同样到达链接器,其中点名的文件是链接的输入(见下文 +> “链接读取的文件”),但构建产生的文件由动作的输出承载。Windows 资源请用 +> [`[resources]`](04-mcpp-toml.md);`object` 是其余一切的出口。 + +#### 链接读取的文件 + +链接读取的文件改变时,链接重新运行。链接的输入有两个来源(SPEC-004 §8.1): + +1. **链接词点名的文件。** 对链接行的每个词(`[build] ldflags`、`mcpp:link-flag=` 以及依赖传播的词) + 取候选:整个词;`=` 之后的部分;以 `,` 分隔的每一段及段内 `=` 之后的部分;MSVC 链接方言下以 `/` + 开头的词中第一个 `:` 之后的部分。候选是绝对路径,且指向一个现有的普通文件或构建中某个动作声明的 + 文件时,它是输入。`@<文件>` 加入该响应文件及其列出的绝对路径。`--js-library=<文件>`、 + `-Wl,--version-script=<文件>`、`-Wl,-force_load,<文件>` 与 `/DEF:<文件>` 都在其中。 +2. **链接器的报告。** 目标为 ELF、且链接器接受 `--dependency-file=<文件>`(GNU ld 2.35 起、gold、 + ld.lld)时,链接边记录链接器读取的文件,与编译边记录编译器读取的头文件相同:经 `-L`、`-l` + 找到的库与链接脚本以 `INPUT` 引入的文件都在其中。链接器是否接受该选项按链接器实测一次, + 答案保存在探测缓存中。 + +相对路径,以及规划时既不存在、也不由动作声明的文件,不是输入,除非链接器报告了它。 **必须写出输出文件名。** mcpp 在 prepare 期就定死源码集、fingerprint 与模块图, 所以名字未知的产物无法构建。内容可以晚到,名字不行。畸形 action 是**硬错误**, diff --git a/src/build/compile_commands.cppm b/src/build/compile_commands.cppm index cabee23db..0bc1edfb0 100644 --- a/src/build/compile_commands.cppm +++ b/src/build/compile_commands.cppm @@ -235,18 +235,7 @@ namespace mcpp::build { // `\"` stayed escaped and a quote inside a word stayed in the word, so the // database listed arguments the compiler never received. std::vector split_flags(std::string_view s) { - std::string unescaped; - unescaped.reserve(s.size()); - for (std::size_t i = 0; i < s.size(); ++i) { - if (s[i] == '$' && i + 1 < s.size() - && (s[i + 1] == ' ' || s[i + 1] == ':' || s[i + 1] == '$')) { - unescaped.push_back(s[++i]); - continue; - } - unescaped.push_back(s[i]); - } - return mcpp::manifest::host_command_words(unescaped, - mcpplibs::os::is_windows); + return rendered_words(s); } namespace { diff --git a/src/build/flags.cppm b/src/build/flags.cppm index ad2c42980..3c52a92d3 100644 --- a/src/build/flags.cppm +++ b/src/build/flags.cppm @@ -75,6 +75,11 @@ struct CompileFlags { // changed archive relinks (design 2026-10-30 part 2 §25.3 C1; it did not, // and the clangd probe deleted its image to take a new archive). std::vector ldFileInputs; + // The word that asks the linker for the files it read, written on every + // link edge with `depfile = $out.d` (SPEC-004 §8.1); empty where the + // linker writes no such report, and those links have only the inputs + // above. + std::string linkReport; // `trim_paths` (mcpp#844): the rendered prefix map of each package, by // `CompileUnit::packageName`, appended to that package's C and C++ units. // Empty on cl.exe, which has no such switch. @@ -334,11 +339,27 @@ std::string atomic_link_flag(const std::vector& linkDirs, // `mcpp::manifest::host_command_words`: reading the result back yields `arg`. std::string shell_quote_arg(std::string_view arg); +// The files one link word names, which are inputs of the link that carries +// it (SPEC-004 §8.1): absolute paths, in the word, after its `=`, in its +// `,`-separated segments, after the `:` of an MSVC `/` option, and those a +// named response file lists; a regular file, or one an action declares. +std::vector +link_word_inputs(std::string_view word, bool msvcDialect, + const std::set& declaredOutputs); + // One word of a compile-flag list as it is written on a ninja `command =` // line: quoted for the host's command-line reader, then `$` doubled so that // ninja hands the reader a literal `$`. A plain word comes back unchanged. std::string ninja_command_word(std::string_view word); +// The inverse of the two: the words a host receives from a rendered line. +// Ninja replaces `$ `, `$:` and `$$` while it builds the command line, and +// only then does the host read the line into words; a word is escaped for +// ninja before it is quoted for the host, so a quoted path's space arrives as +// `$ ` inside the quotes, and it is a space again before the reader sees the +// quotes. The reader is the host's own (#655). +std::vector rendered_words(std::string_view rendered); + // The ninja text of a compile-flag list: every element read into words // (`mcpp::manifest::flag_words`), every word written by `ninja_command_word`, // each preceded by a space. The compile databases list the same words. @@ -478,6 +499,30 @@ std::string normalize_ldflag(const std::filesystem::path& root, const std::strin } // namespace +// The words of a rendered link line that decide which linker its driver runs +// (`--target`, `-fuse-ld`, `--ld-path`, `-B`, a driver configuration) and +// `-flavor`, the one a linker reads to decide which linker it is. +std::vector link_selection_words(std::string_view rendered) { + constexpr std::array kSelecting { + "--target=", "-target", "-fuse-ld=", "--ld-path=", "-B", "--no-default-config", + "--config", "--gcc-toolchain=", "--gcc-install-dir=" }; + std::vector out; + const auto words = rendered_words(rendered); + for (std::size_t i = 0; i < words.size(); ++i) { + const auto& w = words[i]; + const bool pairs = (w == "-target" || w == "-flavor" || w == "-B" || w == "--config") + && i + 1 < words.size(); + if (pairs || w == "-flavor") { + out.push_back(w); + if (pairs) out.push_back(words[++i]); + continue; + } + if (std::ranges::any_of(kSelecting, [&](auto p) { return w.starts_with(p); })) + out.push_back(w); + } + return out; +} + // The absolute paths of existing files a response file names. std::vector response_file_paths(const std::filesystem::path& rsp) { std::vector out; @@ -493,6 +538,54 @@ std::vector response_file_paths(const std::filesystem::pa return out; } +// THE FILES A LINK WORD NAMES (SPEC-004 §8.1, mcpp#931). The rule had grown +// one spelling at a time: a whole word (mcpp#869), then a response file +// (mcpp#883), and an option's value was still no input -- a changed +// `--js-library=` or `-Wl,--version-script=` left the program as +// it was. Every place a word can carry a path is a candidate: the word, the +// part after `=`, each `,`-separated segment and the part after `=` within +// it, and under the MSVC dialect the part after the first `:` of a `/` +// option. A candidate is an input when it is an absolute path that names a +// regular file now or a file an action of this plan declares. A directory +// (`-L`, `-rpath`, `--sysroot`) is no regular file; taking a candidate too +// many costs one link when that file changes, one too few a stale artifact. +std::vector +link_word_inputs(std::string_view word, bool msvcDialect, + const std::set& declaredOutputs) { + std::vector candidates { word }; + auto after = [](std::string_view s, char c) { + auto pos = s.find(c); + return pos == std::string_view::npos ? std::string_view{} : s.substr(pos + 1); + }; + candidates.push_back(after(word, '=')); + if (word.find(',') != std::string_view::npos) + for (auto segment : std::views::split(word, ',')) { + const std::string_view s { segment.begin(), segment.end() }; + candidates.push_back(s); + candidates.push_back(after(s, '=')); + } + if (msvcDialect && word.starts_with('/')) candidates.push_back(after(word, ':')); + + std::vector out; + for (auto c : candidates) { + const bool responseFile = c.starts_with('@'); + if (responseFile) c.remove_prefix(1); + const std::filesystem::path p { std::string(c) }; + if (c.empty() || !p.is_absolute()) continue; + std::error_code ec; + const bool file = std::filesystem::is_regular_file(p, ec); + if (!file && !declaredOutputs.contains(p.lexically_normal())) continue; + if (std::ranges::find(out, p) == out.end()) out.push_back(p); + // A RESPONSE FILE IS AN INPUT, AND SO IS EVERY FILE IT NAMES: a build + // program that lists the archives a `prepare` action makes in one + // rewrites nothing when the action rebuilds them (clangd's ok image, + // 2026-11-02: a stale image after its libraries changed). + if (responseFile && file) + for (auto& named : response_file_paths(p)) out.push_back(std::move(named)); + } + return out; +} + std::string atomic_link_flag(const std::vector& linkDirs, bool staticLink) { for (auto& dir : linkDirs) { @@ -577,6 +670,20 @@ std::string ninja_command_word(std::string_view word) { return out; } +std::vector rendered_words(std::string_view s) { + std::string unescaped; + unescaped.reserve(s.size()); + for (std::size_t i = 0; i < s.size(); ++i) { + if (s[i] == '$' && i + 1 < s.size() + && (s[i + 1] == ' ' || s[i + 1] == ':' || s[i + 1] == '$')) { + unescaped.push_back(s[++i]); + continue; + } + unescaped.push_back(s[i]); + } + return mcpp::manifest::host_command_words(unescaped, mcpplibs::os::is_windows); +} + std::string ninja_flag_list(const std::vector& elements) { std::string out; for (auto const& word : mcpp::manifest::flag_words(elements)) { @@ -1326,6 +1433,12 @@ CompileFlags compute_flags_impl(const BuildPlan& plan, bool packageLinkFlags) { // `__cxa_throw` and the personality first, and libc++abi answered the rest // (mcpp#838, measured: `catch (const std::exception&)` missed). std::string user_ldflags; + // A file an action of this plan declares is an input though it does not + // exist yet: the first build plans the link before the action has run. + std::set declaredOutputs; + for (auto const& a : plan.actions) + for (auto const& o : a.outputs) + if (std::filesystem::path p { o }; p.is_absolute()) declaredOutputs.insert(p.lexically_normal()); const auto linkWords = packageLinkFlags ? mcpp::manifest::flag_words(plan.manifest.buildConfig.ldflags) : std::vector{}; @@ -1339,28 +1452,8 @@ CompileFlags compute_flags_impl(const BuildPlan& plan, bool packageLinkFlags) { } user_ldflags += ' '; user_ldflags += rendered; - // A word that names an existing file by absolute path is a link - // input. A relative word is read by the linker against the build - // directory, and an option is no file. - if (!word.starts_with('-') && !word.starts_with('@') - && std::filesystem::path(word).is_absolute()) { - std::error_code ec; - if (std::filesystem::is_regular_file(word, ec)) f.ldFileInputs.emplace_back(word); - } - // A RESPONSE FILE IS AN INPUT, AND SO IS EVERY FILE IT NAMES. A build - // program that lists the archives a `prepare` action makes in one - // (`link_flag("@")`) rewrites nothing when the action rebuilds - // them, so the link stayed up to date over new archives (clangd's ok - // image, 2026-11-02: a stale image after its libraries changed). - if (word.starts_with('@')) { - const std::filesystem::path rsp { word.substr(1) }; - std::error_code ec; - if (rsp.is_absolute() && std::filesystem::is_regular_file(rsp, ec)) { - f.ldFileInputs.push_back(rsp); - for (auto const& named : response_file_paths(rsp)) - f.ldFileInputs.push_back(named); - } - } + for (auto& input : link_word_inputs(word, isMsvcDialect, declaredOutputs)) + f.ldFileInputs.push_back(std::move(input)); } // C standard. The file-level `$cflags` carries the engine default, a @@ -2760,6 +2853,21 @@ CompileFlags compute_flags_impl(const BuildPlan& plan, bool packageLinkFlags) { if (f.ldC.find("--ld-path=") == std::string::npos) f.ldC += opt; } + // THE LINKER'S OWN REPORT (SPEC-004 §8.1, mcpp#931). What the flags name + // is known when the graph is written; what the linker finds by searching + // (`-l`, a script's `INPUT`) is known only to the linker, which writes it + // as a compiler writes its dependency file. Asked of the linker this + // line runs, on an ELF target: ld64, wasm-ld, lld-link and link.exe + // write no make rules (LLVM 23.1.3). + if (!isMsvcDialect && targetObjectFormat == mcpp::build::dist::Format::Elf) { + const bool direct = !f.ldDriver.empty(); + std::vector argv { direct ? f.ldDriver.string() : f.cxxBinary.string() }; + for (auto const& w : link_selection_words(f.ld)) argv.push_back(w); + if (mcpp::toolchain::linker_reports_inputs(argv, direct, + mcpp::toolchain::compiler_env(plan.toolchain))) + f.linkReport = direct ? " --dependency-file=$out.d" : " -Wl,--dependency-file=$out.d"; + } + return f; } diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 0223d7c02..5eb83a766 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -2550,6 +2550,10 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, : std::string{}; auto link_rule = [&](std::string_view name, std::string cmd, std::string_view desc) { + // The linker's report of what it read (SPEC-004 §8.1): a link edge + // depends on it as a compile edge depends on the compiler's. + const bool reports = desc != "AR" && !separateLinker && !flags.linkReport.empty(); + if (reports) cmd += flags.linkReport; // The permit wraps the publication: the attempt's output is the // artifact in place when the command it holds exits. if (desc != "AR") cmd = linkPrefix + publish_link(std::move(cmd)); @@ -2586,6 +2590,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, } else { append(std::format(" command = {}\n", cmd)); } + if (reports) append(" depfile = $out.d\n deps = gcc\n"); append(std::format(" description = {} $out\n\n", desc)); }; if (separateLinker) { diff --git a/src/toolchain/probe.cppm b/src/toolchain/probe.cppm index d16d59e21..e850b8b33 100644 --- a/src/toolchain/probe.cppm +++ b/src/toolchain/probe.cppm @@ -40,6 +40,17 @@ std::expected run_capture(const std::vector`: GNU ld from 2.35, gold, ld.lld). `argv` is the +// program and the words of the link that decide which linker runs: a compiler +// driver with its `--target`, `-fuse-ld`, `--ld-path` and `-B` words, or the +// linker itself (`direct`). A driver is asked by one real link of an empty +// shared object; a linker by `--version`, which GNU ld 2.42, gold 1.16 and +// ld.lld 23.1.3 answer only after refusing an option they do not know +// (measured). The answer is kept under the identity of the files the words +// name (design 2026-11-19 §6.2). +bool linker_reports_inputs(const std::vector& argv, bool direct, + const mcpp::run::Env& env = {}); std::string extract_version(std::string_view s); std::string first_line_of(std::string_view s); @@ -242,6 +253,51 @@ bool compresses_debug_sections(const std::filesystem::path& compiler, const mcpp && r->find("unsupported option '-gz") == std::string::npos; } +bool linker_reports_inputs(const std::vector& argv, bool direct, + const mcpp::run::Env& env) { + if (argv.empty()) return false; + // Asked once per process as well: every package's flags ask it again. + static std::mutex mutex; + static std::map answered; + const auto key = probe_cache_key(argv, env, direct); + std::lock_guard lock(mutex); + if (auto it = answered.find(key); it != answered.end()) return it->second; + const auto work = mcpp::home::root() / "cache" / "probe" / "link-report"; + const auto answer = work / (mcpp::toolchain::hash_string(key) + ".answer"); + if (std::ifstream in(answer); in) { + std::string v; + in >> v; + if (v == "yes" || v == "no") return answered[key] = (v == "yes"); + } + std::error_code ec; + std::filesystem::create_directories(work, ec); + const auto depfile = work / std::format("probe-{:x}.d", std::random_device{}()); + std::vector cmd = argv; + if (direct) { + cmd.push_back("--dependency-file=" + depfile.string()); + cmd.push_back("--version"); + } else { + const auto src = work / "empty.c"; + if (!std::filesystem::exists(src, ec)) std::ofstream(src, std::ios::binary).flush(); + for (auto const& a : {std::string("-shared"), std::string("-nostdlib"), std::string("-x"), + std::string("c"), src.string(), std::string("-o"), + (work / std::format("probe-{:x}.so", std::random_device{}())).string(), + "-Wl,--dependency-file=" + depfile.string()}) + cmd.push_back(a); + } + auto r = mcpp::run::execute(cmd, {.env = env, .host_tool = true, + .streams = mcpp::run::Streams::merged, + .report_spawn_failure = false}); + const bool yes = r.exit_code == 0 && (direct || std::filesystem::is_regular_file(depfile, ec)); + std::filesystem::remove(depfile, ec); + if (!direct) std::filesystem::remove(cmd[cmd.size() - 2], ec); + mcpp::log::verbose("toolchain", std::format("the linker of '{}' reports what it read: {}{}", + mcpp::build::command_line::display(argv), yes ? "yes" : "no", + yes ? std::string{} : " (" + first_line_of(r.output) + ")")); + if (mcpp::effects::writes_global_cache()) std::ofstream(answer) << (yes ? "yes\n" : "no\n"); + return answered[key] = yes; +} + std::expected run_capture(const std::vector& argv, const mcpp::run::Env& env, bool withErrors) { const auto key = probe_cache_key(argv, env, withErrors); diff --git a/tests/e2e/build/link.cpp b/tests/e2e/build/link.cpp index df80b1645..3e006375d 100644 --- a/tests/e2e/build/link.cpp +++ b/tests/e2e/build/link.cpp @@ -400,6 +400,110 @@ int main() { std::println("val {}", val()); } expect(fatal(e2e::succeeded(r))) << "the third build\n" << s::text(r); expect(s::no_line(s::text(r), R"(^\[[0-9]+/[0-9]+\] .*bin/app)")) << "a build with nothing changed linked the program again\n" << s::text(r); }; + "931 a file an option's value names is a link input"_test[s::gcc][s::unix_shell] = [] { + // Design 2026-11-19 §6.2 rule A: a path after `=` and in a `,`-separated segment is a link + // input as a whole word is. An archive given inside `-Wl,--whole-archive,` that is + // replaced relinks the program, and so does a version script given as + // `-Wl,--version-script=`; a build with nothing changed links nothing. + const auto work = s::case_dir(); + s::project val{work / "val", {{"mcpp.toml", R"cpp([package] +name = "val" +version = "0.1.0" + +[targets.val] +kind = "lib" +)cpp"}}}; + fs::create_directories(work / "prebuilt"); + auto make_archive = [&](int value) { + val.write("src/val.cpp", std::format("extern \"C\" int val() {{ return {}; }}\n", value)); + auto b = val.mcpp({"build"}); + expect(fatal(e2e::succeeded(b))) << "the library did not build\n" << s::text(b); + const auto a = s::find_files(val.at("target"), R"(libval.*\.a)"); + expect(fatal(!a.empty())) << "no archive of the library"; + fs::copy_file(a.front(), work / "prebuilt/libval.a", fs::copy_options::overwrite_existing); + }; + make_archive(41); + s::project{work / "prebuilt", {{"app.map", "{ global: *; };\n"}}}; + const auto work_s = work.string(); + s::project app{work / "app", {{"mcpp.toml", std::format(R"cpp([package] +name = "app" +version = "0.1.0" + +[build] +ldflags = ["-Wl,--whole-archive,{0}/prebuilt/libval.a,--no-whole-archive", + "-Wl,--version-script={0}/prebuilt/app.map"] + +[targets.app] +kind = "bin" +main = "src/main.cpp" +)cpp", work_s)}, {"src/main.cpp", R"cpp(import std; +extern "C" int val(); +int main() { std::println("val {}", val()); } +)cpp"}}}; + auto r = app.mcpp({"run"}); + expect(fatal(e2e::succeeded(r))) << "the first run\n" << s::text(r); + expect(fatal(e2e::contains(s::text(r), "val 41"))) << "the first run did not print val 41\n" << s::text(r); + make_archive(42); + r = app.mcpp({"run"}); + expect(fatal(e2e::succeeded(r))) << "the second run\n" << s::text(r); + expect(fatal(e2e::contains(s::text(r), "val 42"))) << "a replaced archive in a ','-separated segment did not relink the program\n" << s::text(r); + s::project{work / "prebuilt", {{"app.map", "{ global: *; };\n/* changed */\n"}}}; + r = app.mcpp({"build", "-v"}); + expect(fatal(e2e::succeeded(r))) << "the build after the script changed\n" << s::text(r); + expect(s::has_line(s::text(r), R"(^\[[0-9]+/[0-9]+\] .*bin/app)")) << "a changed version script named after '=' did not relink the program\n" << s::text(r); + r = app.mcpp({"build", "-v"}); + expect(fatal(e2e::succeeded(r))) << "the last build\n" << s::text(r); + expect(s::no_line(s::text(r), R"(^\[[0-9]+/[0-9]+\] .*bin/app)")) << "a build with nothing changed linked the program again\n" << s::text(r); + }; + "931 a library the linker found by searching is a link input"_test[s::gcc][s::elf] = [] { + // Design 2026-11-19 §6.2 rule B: where the linker writes what it read, the link edge + // carries that report, and a library found through `-L`/`-l` -- named by no word of the + // line -- is an input: replacing it relinks the program. + const auto work = s::case_dir(); + s::project val{work / "val", {{"mcpp.toml", R"cpp([package] +name = "val" +version = "0.1.0" + +[targets.val] +kind = "lib" +)cpp"}}}; + fs::create_directories(work / "prebuilt"); + auto make_archive = [&](int value) { + val.write("src/val.cpp", std::format("extern \"C\" int val() {{ return {}; }}\n", value)); + auto b = val.mcpp({"build"}); + expect(fatal(e2e::succeeded(b))) << "the library did not build\n" << s::text(b); + const auto a = s::find_files(val.at("target"), R"(libval.*\.a)"); + expect(fatal(!a.empty())) << "no archive of the library"; + fs::copy_file(a.front(), work / "prebuilt/libval.a", fs::copy_options::overwrite_existing); + }; + make_archive(41); + const auto work_s = work.string(); + s::project app{work / "app", {{"mcpp.toml", std::format(R"cpp([package] +name = "app" +version = "0.1.0" + +[build] +ldflags = ["-L{0}/prebuilt", "-lval"] + +[targets.app] +kind = "bin" +main = "src/main.cpp" +)cpp", work_s)}, {"src/main.cpp", R"cpp(import std; +extern "C" int val(); +int main() { std::println("val {}", val()); } +)cpp"}}}; + auto r = app.mcpp({"run"}); + expect(fatal(e2e::succeeded(r))) << "the first run\n" << s::text(r); + expect(fatal(e2e::contains(s::text(r), "val 41"))) << "the first run did not print val 41\n" << s::text(r); + const auto ninja = s::find_files(app.at("target"), R"(build\.ninja)"); + expect(fatal(!ninja.empty())) << "no build.ninja"; + expect(fatal(e2e::contains(s::read_file(ninja.front()), "--dependency-file=$out.d"))) + << "the link rule does not ask the linker what it read\n" << s::read_file(ninja.front()); + make_archive(42); + r = app.mcpp({"run"}); + expect(fatal(e2e::succeeded(r))) << "the second run\n" << s::text(r); + expect(fatal(e2e::contains(s::text(r), "val 42"))) << "a replaced library found through -L/-l did not relink the program\n" << s::text(r); + }; "1062 link flags on a compiler link are named not dropped"_test[s::elf][s::llvm] = [] { // [platform.link] flags a compiler-driven link does not use are named with the remedy // ([platform.build] ldflags); an ld.lld link takes them without a word. diff --git a/tests/e2e/web/emscripten.cpp b/tests/e2e/web/emscripten.cpp index 4dd6670c1..36f897c92 100644 --- a/tests/e2e/web/emscripten.cpp +++ b/tests/e2e/web/emscripten.cpp @@ -268,6 +268,31 @@ E2E emscripten{"emscripten", [] { } }; + "931 a JS library an option names is a link input"_test[s::elf] = [] { + // mcpp#931, design 2026-11-19 §6.2 rule A: the file after `=` in `--js-library=` is + // read by emcc, not by the linker, and is an input of the link; a changed library relinks + // the program, which then answers with the new text. + if (!have_emsdk() || !s::on_path("node")) skip("931 -- no emsdk payload or node on this machine"); + const auto work = s::case_dir(); + auto library = [&](int value) { + s::project{work / "js", {{"val.js", std::format( + "mergeInto(LibraryManager.library, {{ js_val: function() {{ return {}; }} }});\n", value)}}}; + }; + library(41); + s::project app(work / "app", {{"mcpp.toml", std::format( + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n\n" + "[target.wasm32-emscripten]\nrunner = [\"node\"]\n\n" + "[target.wasm32-emscripten.build]\nldflags = [\"--js-library={}\"]\n", + (work / "js/val.js").generic_string())}, + {"src/main.cpp", "import std;\nextern \"C\" int js_val();\nint main() { std::println(\"val {}\", js_val()); }\n"}}); + auto r = app.mcpp({"run", "--target", "wasm32-emscripten"}); + expect(fatal(e2e::succeeded(r) && s::has_line(s::text(r), "^val 41$"))) << "the first run\n" << s::text(r); + library(42); + r = app.mcpp({"run", "--target", "wasm32-emscripten"}); + expect(fatal(e2e::succeeded(r))) << "the second run\n" << s::text(r); + expect(s::has_line(s::text(r), "^val 42$")) << "a changed --js-library file did not relink the program\n" << s::text(r); + }; + "1079 an emscripten rows longjmp model round trips"_test[s::elf] = [] { // `longjmp = "wasm"` makes a non-local exit over two call levels round-trip; on a Linux // row it is refused naming that row's os. diff --git a/tests/unit/test_build_flags.cpp b/tests/unit/test_build_flags.cpp index a3eb7cc86..6d71c6f0b 100644 --- a/tests/unit/test_build_flags.cpp +++ b/tests/unit/test_build_flags.cpp @@ -345,3 +345,48 @@ UTest a_graph_that_declares_no_absence_leaves_no_sidecar{"CAbiAbsentAdvice/AGrap auto [name, back] = mcpp::build::read_c_abi_absent_sidecar(dir.path); expect(back.empty()); }}; + +UTest every_place_a_word_carries_a_path_is_a_link_input{"LinkWordInputs/EveryPlaceAWordCarriesAPath", [] { + // mcpp#931: the whole word, the part after `=`, each `,` segment and the + // part after `=` within it, and a response file with what it lists. + Tmp dir; + const auto a = dir.path / "a.a", m = dir.path / "v.map", js = dir.path / "lib.js", + rsp = dir.path / "libs.rsp"; + touch(a); touch(m); touch(js); + std::ofstream(rsp) << a.string() << "\n"; + const std::set none; + using mcpp::build::link_word_inputs; + expect(link_word_inputs(a.string(), false, none) == std::vector{a}); + expect(link_word_inputs("--js-library=" + js.string(), false, none) == std::vector{js}); + expect(link_word_inputs("-Wl,--version-script=" + m.string(), false, none) == std::vector{m}); + expect(link_word_inputs("-Wl,-force_load," + a.string(), false, none) == std::vector{a}); + expect(link_word_inputs("@" + rsp.string(), false, none) == std::vector{rsp, a}); +}}; + +UTest a_directory_or_a_relative_path_is_no_link_input{"LinkWordInputs/ADirectoryOrARelativePathIsNone", [] { + Tmp dir; + const std::set none; + using mcpp::build::link_word_inputs; + expect(link_word_inputs("-L" + dir.path.string(), false, none).empty()); + expect(link_word_inputs("-Wl,-rpath," + dir.path.string(), false, none).empty()); + expect(link_word_inputs("--sysroot=" + dir.path.string(), false, none).empty()); + expect(link_word_inputs("-Wl,--version-script=v.map", false, none).empty()); + expect(link_word_inputs(dir.path.string() + "/absent.a", false, none).empty()); +}}; + +UTest a_file_an_action_declares_is_an_input_before_it_exists{"LinkWordInputs/AnActionsOutputBeforeItExists", [] { + Tmp dir; + const auto gen = dir.path / "gen" / "exports.map"; + const std::set declared { gen }; + expect(mcpp::build::link_word_inputs("-Wl,--version-script=" + gen.string(), false, declared) + == std::vector{gen}); +}}; + +UTest an_msvc_option_value_is_a_link_input{"LinkWordInputs/AnMsvcOptionValue", [] { + // `/DEF:` under the MSVC dialect; the same word is no option elsewhere. + Tmp dir; + const auto def = dir.path / "x.def"; + touch(def); + const std::set none; + expect(mcpp::build::link_word_inputs("/DEF:" + def.string(), true, none) == std::vector{def}); +}}; From 087f45736a18fc555dad9ecf447d8be8e7352d7f Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 13:40:58 +0900 Subject: [PATCH 07/30] A row a member inherits from its workspace is the root's statement, and is not reported as the member's own (mcpp#954) --- CHANGELOG.md | 7 +++++++ docs/07-workspace.md | 7 +++++++ docs/22-target-side.md | 4 +++- docs/zh/07-workspace.md | 4 ++++ docs/zh/22-target-side.md | 4 +++- modules/manifest/src/types.cppm | 13 ++++++++++++ src/build/prepare/config.cpp | 4 ++++ src/build/prepare/features.cpp | 30 +++++++++++++++++++++------ src/project.cppm | 1 + tests/e2e/workspace/inheritance.cpp | 32 +++++++++++++++++++++++++++++ 10 files changed, 98 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 038c34f0f..43009b479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,13 @@ compile edge records the compiler's report, so a library found through `-L`/`-l` relinks the program when it changes. Whether a linker accepts the option is measured once and kept in the probe cache. +- Every member of a workspace whose root states `[workspace.target..abi]` was + warned that it declares `[target..abi]`, which only the root decides: a build of + about 60 members for wasm32-emscripten printed the warning 262 times (mcpp#954). A + conditional row records whether it is the package's own or inherited from a workspace; a + row inherited from this build's workspace is the root's statement and is not reported. A + member's own table is reported once when it is reached as a dependency, and the warning + names the manifest that states it. ## [2026.11.18.1] - 2026-11-18 diff --git a/docs/07-workspace.md b/docs/07-workspace.md index b5f5f841e..0414de1bc 100644 --- a/docs/07-workspace.md +++ b/docs/07-workspace.md @@ -244,6 +244,13 @@ A conditional row counts as it evaluates for the target being built: a from the others (§5.4), and one that holds is applied when the member is built under the workspace's root. +A workspace whose root is only `[workspace]` states the build's abi in +`[workspace.target..abi]`. Each member inherits the row, and the row +is the root's statement: no member is reported for it. A member whose own +manifest has `[target..abi]` is reported when it is reached as a +dependency, as any dependency's table is (mcpp 2026.11.19.1+; before that +release every member that inherited the workspace's row was reported, mcpp#954). + A build without `--target` targets the host, and `[target.]` applies to it as `--target ` would. diff --git a/docs/22-target-side.md b/docs/22-target-side.md index f4d8eb8a3..45077113b 100644 --- a/docs/22-target-side.md +++ b/docs/22-target-side.md @@ -1437,7 +1437,9 @@ keeps the key it had. **Only the root manifest decides.** The switch belongs to the artefact, and the root is the only package that builds one. A dependency that writes `[target..abi]` is reported (`abi/dependency-table`) and changes -nothing. A dependency states what it needs instead: +nothing; a row a workspace member inherits from its build's +`[workspace.target..abi]` is the root's and is not reported +([workspace](07-workspace.md)). A dependency states what it needs instead: ```toml [package] diff --git a/docs/zh/07-workspace.md b/docs/zh/07-workspace.md index aab611ac1..e91732844 100644 --- a/docs/zh/07-workspace.md +++ b/docs/zh/07-workspace.md @@ -229,6 +229,10 @@ default = "llvm@23.1.3" 条件行按所构建的目标求值后计入:其他目标上的 `dialect_cxxflags` 或 `.abi` 行不会把成员与 其他成员分开(§5.4);对当前目标成立的行,在成员于工作空间的根下构建时生效。 +根只有 `[workspace]` 的工作空间在 `[workspace.target..abi]` 陈述构建的 abi。每个成员继承该行, +该行是根的陈述,不为任何成员报告。成员自己的清单含 `[target..abi]` 时,它作为依赖到达时 +被报告,与任何依赖的该表相同(mcpp 2026.11.19.1+;此前每个继承工作空间该行的成员都被报告,mcpp#954)。 + 不带 `--target` 的构建以宿主为目标,`[target.<宿主三元组>]` 对它生效,与 `--target <宿主三元组>` 相同。 diff --git a/docs/zh/22-target-side.md b/docs/zh/22-target-side.md index 84f81ffa4..f5663c768 100644 --- a/docs/zh/22-target-side.md +++ b/docs/zh/22-target-side.md @@ -1195,7 +1195,9 @@ enabled. Compile with -sNO_DISABLE_EXCEPTION_CATCHING or **只有根 manifest 能做决定。** 这个开关属于产物,而根包是唯一构建产物的 包。依赖写下的 `[target..abi]` 会被报告 -(`abi/dependency-table`),且不改变任何东西。依赖改为声明自己的需求: +(`abi/dependency-table`),且不改变任何东西;工作空间成员从所在构建的 +`[workspace.target..abi]` 继承的行属于根,不被报告([工作空间](07-workspace.md))。 +依赖改为声明自己的需求: ```toml [package] diff --git a/modules/manifest/src/types.cppm b/modules/manifest/src/types.cppm index 329e52c94..560a80bf7 100644 --- a/modules/manifest/src/types.cppm +++ b/modules/manifest/src/types.cppm @@ -1208,6 +1208,14 @@ struct BuildConfig : BuildInputs { // statement only, rendered into every translation unit and the link, like // `threads`; the model reaches every cache key through those flags. bool abiLongjmpDeclared = false; + // Whether one of the abi members above was stated by a row of the + // package's own manifest, not by a `[workspace.target..abi]` row the + // package inherits (mcpp#954): a dependency's own table is reported, the + // workspace's is the root's decision. + bool abiDeclaredByPackage = false; + // The workspace manifest whose inherited row stated one of them, when + // one did: the root's statement only when it is this build's workspace. + std::filesystem::path abiInheritedFrom; std::string abiLongjmpModel; std::string cStandard; // Escape hatch for the hermetic link check: a sandbox toolchain whose @@ -1758,6 +1766,11 @@ struct TargetEntry { // buildConfig. See .agents/docs/2026-06-29-manifest-environment-and-platform-design.md. struct ConditionalConfig { std::string predicate; // the [target.] key + // Where the row was stated (mcpp#954): empty for the package's own + // manifest, the workspace manifest for a `[workspace.target.]` row + // a member inherits. A decision that depends on who stated a row reads + // this, not the package the row reached. + std::filesystem::path inheritedFrom; // Everything `[target..build]` may contribute, and nothing else. // // Previously four hand-listed vectors, which is why per-glob `flags` was diff --git a/src/build/prepare/config.cpp b/src/build/prepare/config.cpp index d4032b44b..1eafd9e3a 100644 --- a/src/build/prepare/config.cpp +++ b/src/build/prepare/config.cpp @@ -346,6 +346,10 @@ void merge_conditional_config(mcpp::manifest::Manifest& m, m.buildConfig.abiLongjmpModel = cc.abiLongjmpModel; m.buildConfig.abiLongjmpDeclared = true; } + if (cc.abiThreadsDeclared || cc.abiExceptionsDeclared || cc.abiLongjmpDeclared) { + if (cc.inheritedFrom.empty()) m.buildConfig.abiDeclaredByPackage = true; + else m.buildConfig.abiInheritedFrom = cc.inheritedFrom; + } // `[target..build] dialect_cxxflags` (#717): recorded for every // package, like the abi switches above, but APPENDED rather than // replaced -- there is no "last matching section wins" here, because diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index d851be14d..e662e9ff8 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -195,6 +195,14 @@ void merge_program_platform(PrepareState& state, mcpp::manifest::platform::State } namespace { +// Two spellings of one manifest are one file: a workspace root is reached +// through a member's relative path as well as through its own. +bool same_file(const std::filesystem::path& a, const std::filesystem::path& b) { + std::error_code ec; + if (std::filesystem::equivalent(a, b, ec)) return true; + return a.lexically_normal() == b.lexically_normal(); +} + bool names_layer(const std::vector& entries, std::string_view layer) { for (auto const& e : entries) { std::string_view v = e; @@ -512,14 +520,24 @@ static std::expected step6_activate_features(PrepareState& st // ignored, and points at the key a dependency does have. Covers // BOTH members: a dependency that declares only `exceptions` must // be reported exactly as one that declares only `threads`. - if (pcap != state.m->package.name - && (pkg.manifest.buildConfig.abiThreadsDeclared - || pkg.manifest.buildConfig.abiExceptionsDeclared)) + // A row the package inherits from this build's workspace + // (`[workspace.target..abi]`) is the root's statement, and is + // not reported for each member that receives it (mcpp#954: 262 + // warnings for one table). Another workspace's row is not this + // build's root, and is reported as the package's own is. + const auto& abi = pkg.manifest.buildConfig; + const auto statedIn = abi.abiDeclaredByPackage ? pkg.manifest.sourcePath : abi.abiInheritedFrom; + const bool rootStated = !abi.abiDeclaredByPackage && !abi.abiInheritedFrom.empty() + && state.wsManifest && same_file(abi.abiInheritedFrom, state.wsManifest->sourcePath); + if (pcap != state.m->package.name && !rootStated + && (abi.abiDeclaredByPackage || !abi.abiInheritedFrom.empty())) mcpp::diag::warning("abi/dependency-table", std::format( - "`{}` declares [target..abi], which only the root " - "manifest decides; a dependency states what it needs with " + "`{}` declares [target..abi], which only the root manifest " + "decides ({}); a dependency states what it needs with " "`requires_abi = {{ threads = true }}` or " - "`requires_abi = {{ exceptions = true }}`", pcap)); + "`requires_abi = {{ exceptions = true }}`, and a workspace states the " + "build's abi in [workspace.target..abi]", pcap, + statedIn.empty() ? std::string("its manifest") : statedIn.generic_string())); // `[targets.*] required_features` on a DEPENDENCY. // // THIS GATE EXISTED ONLY FOR THE ROOT. The root's targets are diff --git a/src/project.cppm b/src/project.cppm index e909ac1ce..0518388f4 100644 --- a/src/project.cppm +++ b/src/project.cppm @@ -495,6 +495,7 @@ export void inherit_workspace_layer_rows(mcpp::manifest::Manifest& member, std::vector rows; for (auto cc : layer->conditionalConfigs) { cc.xlings = {}; + cc.inheritedFrom = workspace.sourcePath.empty() ? wsRoot / "mcpp.toml" : workspace.sourcePath; anchor(cc.inputs.includeDirs); anchor(cc.inputs.includeDirsAfter); anchor(cc.inputs.privateIncludeDirs); diff --git a/tests/e2e/workspace/inheritance.cpp b/tests/e2e/workspace/inheritance.cpp index 2297fcf48..3b3b8ceaf 100644 --- a/tests/e2e/workspace/inheritance.cpp +++ b/tests/e2e/workspace/inheritance.cpp @@ -600,6 +600,38 @@ main = "src/main.cpp" expect(fatal(e2e::succeeded(r))) << s::text(r); expect(s::lacks(s::text(r), "shared.cppm") && s::lacks(s::text(r), "mid.cppm")) << "B: back at the first member a package was compiled again: " << s::text(r); }; + "954 the workspace's abi table is the root's statement"_test[s::gcc][s::elf] = [] { + // mcpp#954. A workspace whose root is only `[workspace]` states the build's abi once, in + // `[workspace.target..abi]`; its members inherit the row and no member is told + // that it declares a table only the root decides (A). A member whose own manifest has the + // table is reported once, by name, when another member depends on it (B). + const auto tmp = s::case_dir(); + const auto lib = [](std::string_view own) { + return std::format("[package]\nname = \"core\"\nversion = \"0.1.0\"\n\n{}[targets.core]\nkind = \"lib\"\n", own); + }; + const std::string app = "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncore = { path = \"../core\" }\n\n" + "[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"; + s::project ws(tmp / "ws", { + {"mcpp.toml", "[workspace]\nmembers = [\"core\", \"app\"]\n\n[workspace.target.'cfg(os = \"linux\")'.abi]\nthreads = true\n"}, + {"core/mcpp.toml", lib("")}, + {"core/src/core.cppm", "export module ws954.core;\nexport int core_value() { return 3; }\n"}, + {"app/mcpp.toml", app}, + {"app/src/main.cpp", "import ws954.core;\nint main() { return core_value() == 3 ? 0 : 1; }\n"}, + }); + auto r = ws.mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << "A: the workspace did not build: " << s::text(r); + expect(s::lacks(s::text(r), "declares [target..abi]")) << "A: an inherited abi row was reported as a member's own table: " << s::text(r); + ws.write("core/mcpp.toml", lib("[target.'cfg(os = \"linux\")'.abi]\nthreads = true\n\n")); + r = ws.mcpp({"build", "-p", "app"}); + expect(fatal(e2e::succeeded(r))) << "B: the workspace did not build: " << s::text(r); + const auto text = s::text(r); + std::size_t n = 0; + for (auto pos = text.find("`core` declares [target..abi]"); pos != std::string::npos; + pos = text.find("`core` declares [target..abi]", pos + 1)) + ++n; + expect(n == 1u) << "B: the member's own abi table was reported " << n << " times: " << text; + expect(s::lacks(text, "`app` declares")) << "B: the member without a table was reported: " << text; + }; }}; } // namespace From c650a370d03b4cdae09790440af4fd8743d72fca Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 14:02:07 +0900 Subject: [PATCH 08/30] An action reads a link unit's objects from the engine, and a plugin's feature carries its own program to every build program that links it (mcpp#959, mcpp#955) --- CHANGELOG.md | 19 +++++ docs/30-build-mcpp.md | 22 +++++- docs/31-authoring-a-rule-package.md | 2 +- docs/specs/build-plugins.md | 34 +++++++-- docs/specs/manifest-semantics.md | 6 +- docs/zh/30-build-mcpp.md | 14 +++- docs/zh/31-authoring-a-rule-package.md | 2 +- modules/buildmcpp/src/directives.cppm | 7 +- modules/buildmcpp/src/program_protocol.cppm | 9 ++- modules/manifest/src/types.cppm | 6 ++ src/build/build_program.cppm | 24 +++++++ src/build/hostprogram.cppm | 44 ++++++++++-- src/build/ninja_backend.cppm | 78 +++++++++++++++++++-- src/build/plan.cppm | 50 +++++++++++++ src/build/prepare/features.cpp | 31 ++++++++ src/build/prepare/plan.cpp | 28 ++++++++ src/cli.cppm | 14 +++- src/cli/cmd_build.cppm | 52 ++++++++++++++ tests/e2e/actions/commands.cpp | 69 ++++++++++++++++++ tests/e2e/build-program/hosttools.cpp | 43 ++++++++++++ tests/unit/test_build_directives.cpp | 20 ++++-- 21 files changed, 546 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43009b479..7c4e601a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ ## [2026.11.19.1] - 2026-11-19 +### Added + +- `${mcpp.objects:}` and `${mcpp.objects_rsp:}`: a link unit's object list, one + absolute path per line in link order or in response-file quoting, for an `artifact` or + `check` action; the list and every object in it are inputs of the action (mcpp#959, + SPEC-007 R3.16). A rule that walked `obj/` and excluded paths guessed what the engine knows. + `mcpp::objects_of` and `mcpp::objects_rsp_of` return the tokens; build-program protocol 20. +- `${mcpp.self} write --output [--append ]... -- ...`: writes known text and + appended files to one output, only when the bytes differ (mcpp#959). The engine's own + commands are bounded by one criterion, stated in SPEC-007 R3.17: one output, decided by the + declared inputs and the argv, the same on every run. + ### Fixed - `mcpp publish` and `mcpp emit xpkg` wrote a descriptor that nothing read: a table of @@ -64,6 +76,13 @@ row inherited from this build's workspace is the root's statement and is not reported. A member's own table is reported once when it is reached as a dependency, and the warning names the manifest that states it. +- A plugin's library could not reach its own program: `mcpp::dep_bin` answered the path in + the build program of the package whose edge named the plugin, and `""` one package away, + through a framework, with nothing said (mcpp#955). A tool a feature declares + (`[features.] tools`) reaches every build program that links the package's library with + the feature active (SPEC-007 R6.3), so a multi-step action is a program the plugin carries. + A tool no edge and no feature declares is reported (`build-program/undeclared-tool`) with + the line that declares it. ## [2026.11.18.1] - 2026-11-18 diff --git a/docs/30-build-mcpp.md b/docs/30-build-mcpp.md index 90628a9a7..ac0a46817 100644 --- a/docs/30-build-mcpp.md +++ b/docs/30-build-mcpp.md @@ -1172,8 +1172,17 @@ none to rely on), and the only interpolations are a closed set: | `${mcpp.compile_db}` | path to `compile_commands.json` (what clang-tidy's `-p` wants) | | `${mcpp.contract}` *(2026.10.30.1+)* | path to the configuration's contract, `mcpp-contract.json`, beside its compile database: the compiler of each language with the words every unit is compiled with (target, sysroot, target-side headers, platform and profile flags), and the link driver, its words and the archiver. A foreign build system reads it to compile for the target as the engine does; a package's own include directories are not in it, nor the engine's C++ dialect and module words (`-std=`, `-fmodules`), which are its choice for its own units. Since 2026.11.7.1, link arguments describe the target configuration; the consuming package's link flags and libraries are excluded, since they can name outputs the foreign build has yet to produce | | `${mcpp.target_file:}` | the built file of target `` | +| `${mcpp.objects:}` *(2026.11.19.1+)* | the absolute path of a file listing the files the link of target `` takes as explicit inputs — its objects, the objects of statically linked dependencies and of the standard module, not the libraries named by `-l` — one absolute path per line, in link order. `artifact` and `check` roles only; the list and every file in it are inputs of the action, so a recompiled object runs it again. `mcpp::objects_of("")` returns the token (protocol 20) | +| `${mcpp.objects_rsp:}` *(2026.11.19.1+)* | the same list in response-file quoting, for a tool that reads `@file` (`ar`, `llvm-ar`, `lib.exe`); a path with a space is quoted. `mcpp::objects_rsp_of("")` returns the token | | `${mcpp.stage_dir}` *(2026.9.11.1+)* | the tree `mcpp pack` staged, absolute. `artifact` role only, and only under `mcpp pack --pkg-format ` | -| `${mcpp.self}` *(2026.9.13.1+)* | the engine's own executable, absolute. An action's command is an argv with no shell, so a build program has no portable way to copy a file; the engine is present wherever a build runs, and `${mcpp.self} stage --verify content --output ` copies one file, creates the destination's parent, and writes only when the bytes differ. That argument shape is a contract from 2026.9.13.1 on; a build program that names it declares that release as its floor | +| `${mcpp.self}` *(2026.9.13.1+)* | the engine's own executable, absolute. An action's command is an argv with no shell, so a build program has no portable way to copy a file; the engine is present wherever a build runs, and `${mcpp.self} stage --verify content --output ` copies one file, creates the destination's parent, and writes only when the bytes differ. That argument shape is a contract from 2026.9.13.1 on; a build program that names it declares that release as its floor. `${mcpp.self} write --output [--append ]... -- ...` *(2026.11.19.1+)* writes the lines after `--`, each followed by a newline, then the bytes of each `--append` file (declare them as the action's inputs), and writes only when the bytes differ | + +The engine's own commands are bounded by one criterion (SPEC-007 R3.17): one +output, whose content the declared inputs and the argv decide, the same on every +run. `stage` and `write` meet it. Deleting, creating directories, globbing or +running several programs in order do not: a multi-step action is a program a +plugin carries, declared as a tool of one of its features (see *Host tools from a dependency* +below), not a vocabulary the engine interprets. Every path a `${mcpp.*}` token expands to is spelled with **forward slashes**. A token is spliced into the text the build program wrote, and that text spells @@ -2029,6 +2038,17 @@ int main() { return grpcgen::generate_all() ? 0 : 1; } - **Visibility, not execution.** `dep_bin()` returns a path; whether anything runs is still the consumer's `build.mcpp`'s decision. Nothing changes about who builds the tool or how the tool store is keyed. +- **A feature's tool reaches every build program that links the package's + library** (2026.11.19.1+). A plugin's library runs inside the build program + of whichever package links it, a framework's user included; with + `[features.] tools = [""]` active, each of those programs sees the + tool, so a plugin carries its own program. Before that release the path + reached only the package whose edge named the plugin, and `dep_bin` answered + `""` one package away. +- **A tool nobody declared is reported** (2026.11.19.1+, protocol 20). + `dep_bin` answers `""` for a tool no edge and no active feature declares, and + the engine warns (`build-program/undeclared-tool`) naming the package, the + tool and the line that declares it. - **Unqualified names are resolved by a ladder, not by luck.** Once two libraries can re-export, both may offer the tail `protobuf`. The fully-qualified `MCPP_DEP___BIN_` is always published; the diff --git a/docs/31-authoring-a-rule-package.md b/docs/31-authoring-a-rule-package.md index 3db8d3b43..1cb9a5d7e 100644 --- a/docs/31-authoring-a-rule-package.md +++ b/docs/31-authoring-a-rule-package.md @@ -27,7 +27,7 @@ them. | `mcpp::action` | one edge in the build graph: a command with declared inputs and outputs | a build program, or a rule module it imports | | `device_extensions` | an extension the engine classifies as a **device source** instead of refusing it | a feature of a package | | `rule_module` | the module a consumer's build program imports to reach the rule | the same feature | -| `tools = [...]` | a generator or compiler **built from source for the build machine**, reached with `mcpp::dep_bin` | a dependency edge | +| `tools = [...]` | a generator or compiler **built from source for the build machine**, reached with `mcpp::dep_bin` | a dependency edge, or a feature of the package (`[features.] tools`), which reaches every build program that links the package's library (2026.11.19.1+) | | `[xlings]`, `[feature-xlings]` | a prebuilt tool the rule runs, installed on demand | the package, or one of its features | What the ecosystem has built out of them: diff --git a/docs/specs/build-plugins.md b/docs/specs/build-plugins.md index 4f0d5bbc0..743011079 100644 --- a/docs/specs/build-plugins.md +++ b/docs/specs/build-plugins.md @@ -4,10 +4,10 @@ |---|---| | 规范编号 | SPEC-007 | | 标题 | 构建插件:配置、施工与校验的分工,运行时与规划期的义务 | -| 状态 | 草案 v0.13 | -| 版本 | 0.13 | -| 最后修改 | 2026-11-16 | -| 对应实现 | 逐条标注。未注明版本的「已实现」条款对应 mcpp >= 2026.9.26.1;注明 mcpp#702 的条款对应 mcpp >= 2026.9.26.2;注明 mcpp#707、#708、#709、#711 的条款对应 mcpp >= 2026.9.27.1;注明 mcpp#723 的条款对应 mcpp >= 2026.9.28.1;注明 mcpp 2026.9.28.2 的条款对应该版本;§9 与注明 mcpp#734 的条款对应 mcpp >= 2026.9.28.3;注明 mcpp#778 的条款对应 mcpp >= 2026.10.10.1;§10 对应 mcpp >= 2026.10.11.1;R9.10 与协议 17 对应 mcpp >= 2026.10.27.1;§3.1 与协议 18 对应 mcpp >= 2026.11.5.1;协议 19 对应 mcpp >= 2026.11.16.1 | +| 状态 | 草案 v0.14 | +| 版本 | 0.14 | +| 最后修改 | 2026-11-19 | +| 对应实现 | 逐条标注。未注明版本的「已实现」条款对应 mcpp >= 2026.9.26.1;注明 mcpp#702 的条款对应 mcpp >= 2026.9.26.2;注明 mcpp#707、#708、#709、#711 的条款对应 mcpp >= 2026.9.27.1;注明 mcpp#723 的条款对应 mcpp >= 2026.9.28.1;注明 mcpp 2026.9.28.2 的条款对应该版本;§9 与注明 mcpp#734 的条款对应 mcpp >= 2026.9.28.3;注明 mcpp#778 的条款对应 mcpp >= 2026.10.10.1;§10 对应 mcpp >= 2026.10.11.1;R9.10 与协议 17 对应 mcpp >= 2026.10.27.1;§3.1 与协议 18 对应 mcpp >= 2026.11.5.1;协议 19 对应 mcpp >= 2026.11.16.1;协议 20 与 R3.16、R3.17、R6.7 对应 mcpp >= 2026.11.19.1 | | 相关设计文档 | `.agents/docs/2026-09-26-compile-database-and-issue-699-design.md`(§5)、`.agents/docs/2026-09-28-ecosystem-design-and-optimisation-plan.md`(WS1、WS3)、`.agents/docs/2026-09-28-build-cost-foreign-toolsets-and-library-surface-design.md`(§3、§4)、`.agents/docs/2026-10-10-one-source-one-meaning-design.md`(§4)、`.agents/docs/2026-10-11-build-dependencies-one-pipeline-design.md` | | 相关 issue | mcpp#699、mcpp#701、mcpp#702、mcpp#703、mcpp#707、mcpp#708、mcpp#709、mcpp#711、mcpp#734、mcpp#755、mcpp#778、mcpp#797 | | 使用文档 | [docs/30 - build.mcpp](../30-build-mcpp.md)、[docs/31 - 编写规则包](../31-authoring-a-rule-package.md) | @@ -145,6 +145,22 @@ `graph.packages[].outDir` 取得它,**禁止**拼写其路径。构建程序的二进制与缓存不按配置分开。 `${mcpp.out_dir}`(构建目录)不变。(**已实现**,mcpp#778) +- **R3.16** 一个链接单元由哪些对象组成是引擎的事实。`artifact` 与 `check` action **可以**以 + `${mcpp.objects:<名字>}` 引用它:名字按 `${mcpp.target_file:<名字>}` 的规则解析,未知与有歧义的 + 名字同样被拒绝;记号展开为构建目录中一个清单文件的绝对路径(正斜杠),清单按链接顺序列出该单元 + 的链接边作为显式输入的文件(它的对象、静态并入的依赖与标准库模块的对象;以 `-l` 指名的库与 + 框架不在内),每行一个绝对路径,UTF-8,`\n` 结尾。`${mcpp.objects_rsp:<名字>}` 是同一清单按 + 响应文件的引号规则写出的形式,供以 `@file` 读取的工具使用。引擎在写出规划时写清单,字节不变时 + 不写;清单与其中每个文件是该 action 的隐式输入。`source`、`prepare`、`object` action 引用它们时 + 规划被拒绝,消息写出 action 与允许的角色。构建程序以 `mcpp::objects_of(name)` 与 + `mcpp::objects_rsp_of(name)` 取得记号(协议 20)。(**已实现**,mcpp 2026.11.19.1,mcpp#959) +- **R3.17** 引擎自身经 `${mcpp.self}` 提供的命令以一条判据为界:**一个输出,内容完全由声明的输入 + 与 argv 决定,重复执行结果相同**。`stage`(放置一个文件)与 `write`(写出规划已知的文本)满足; + 删除、创建目录、通配、按顺序运行多个程序不满足,它们属于插件携带的步骤程序(R6.3)。 + `${mcpp.self} write --output <文件> [--append <文件>]... -- <行>...` 写出 `--` 之后的各行(每行 + 以 `\n` 结尾),再依次写出每个 `--append` 文件的字节;输出写在旁边后改名,字节不变时不写。 + `--append` 的文件**必须**声明为 action 的输入。(**已实现**,mcpp 2026.11.19.1,mcpp#959) + ### 3.1 可选 action 进度(协议 18) 以下条款对应 mcpp 2026.11.5.1+。 @@ -226,6 +242,14 @@ - **R6.3** 插件的某个特性需要本包的程序在构建机器上运行时,**应当**在该特性上声明 `[features.] tools = [""]`,而不是要求每个消费方在依赖边上重复写 `tools`。启用该 特性的消费方得到该工具,与边上写了 `tools` 相同(SPEC-004 §10.2)。(**已实现**,mcpp#709) + 图中链接了该包的宿主模块、且该包的构建程序实例上该特性激活的**每个**构建程序都得到该工具 + (`MCPP_DEP_<包>_BIN_<工具>`),包括经一个框架到达该包的消费方:插件的库在哪个构建程序中运行, + 哪个构建程序就能取得插件自己的程序。多步动作的运行器因此是插件携带的程序,不是引擎解释的 + 一组操作。(**已实现**,mcpp 2026.11.19.1,mcpp#955) +- **R6.7** `mcpp::dep_bin(包, 工具)` 被问到一个没有任何依赖边、也没有任何激活的特性声明的工具时 + 回答空串,并陈述 `mcpp:undeclared-tool=<包>\t<工具>`(协议 20);引擎以 `warning` + (`build-program/undeclared-tool`)报告,写出包、工具与应当写的那一行。缓存命中的运行再次报告。 + (**已实现**,mcpp 2026.11.19.1,mcpp#955) - **R6.4** 一个需要随消费方发布、在消费方的目标上运行的程序(更新器、辅助进程)**必须**以依赖 边的 `artifacts` 取得(SPEC-004 §10.3),**禁止**以 `tools` 取得:`tools` 为构建机器构建, 交叉构建中得到错误架构的程序。action 以 `${mcpp.artifact:<依赖>/<目标>}` 引用它的路径。 @@ -271,6 +295,7 @@ | 17 | 2026.10.27.1 | 平台的链接与运行器字段(SPEC-013 §12):`link_driver`、`link_input`、`link_input_from`、`artifact_name`、`seal`、`run_host`;平台阶段(`mcpp::phase()` 为 `"platform"`) | | 18 | 2026.11.5.1 | `action.progress` 与可选运行期快照通道(§3.1);旧载荷默认关闭 | | 19 | 2026.11.16.1 | 包格式提供者陈述它组装产物所取的程序:`pack_program`(`mcpp:pack-program=`,SPEC-014 §6);一条陈述的运行器:`runner_spec`(`mcpp:runner-spec=`,一个 JSON 载荷,字段与 SPEC-013 §10 的表一一对应;同一名字不能既由它又由逐词调用陈述) | + | 20 | 2026.11.19.1 | 链接单元的对象清单:`objects_of`、`objects_rsp_of`(`${mcpp.objects:<名字>}`、`${mcpp.objects_rsp:<名字>}`,R3.16);`dep_bin` 对未声明的工具陈述 `mcpp:undeclared-tool=`(R6.7) | - **R9.3** 构建信息以事实陈述解析出的工具链,不针对任何外部构建系统:`tool(role)`(这一行的 工具)、`abi_tool(role)`(目标 ABI 的原生工具,MSVC ABI 上为工具集的 `cl`、`link`、`lib`、 @@ -353,3 +378,4 @@ | 0.11 | 2026-10-28 | 随 mcpp 2026.10.27.1(mcpp#847):R9.2 的协议表增加协议 17(平台的链接与运行器字段);新增 R9.10 平台阶段(SPEC-013),平台阶段可声明包格式与 artifact action(SPEC-014 §6)。 | | 0.12 | 2026-11-05 | 随 mcpp 2026.11.5.1:新增 §3.1 可选 action 进度(协议 18),规定通道、限额、序号、关闭与失败/取消行为。 | | 0.13 | 2026-11-16 | 随 mcpp 2026.11.16.1(设计 2026-11-15 N30):R9.2 的协议表增加协议 19(`pack_program`)。;协议 19 增加 `runner_spec`(设计 2026-11-15 R2)。 | +| 0.14 | 2026-11-19 | 随 mcpp 2026.11.19.1(设计 2026-11-19 §8):新增 R3.16(链接单元的对象清单,协议 20,mcpp#959)、R3.17(引擎自身命令的判据与 `${mcpp.self} write`)、R6.7(未声明的工具);R6.3 补充特性的工具到达链接该包库的每个构建程序(mcpp#955);R9.2 的协议表增加协议 20。 | diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index ded597946..b4d67853b 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -702,8 +702,10 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m `[features.] tools = [""]` 陈述启用特性 `f` 需要本包的程序 `` 在构建机器上 运行。启用该特性的依赖边,等同于在边上写了 `tools = [""]`;不启用时不构建。条目 **必须**指名本包的一个 `bin` 目标,否则清单在加载时被拒绝,消息列出本包的 `bin` 目标。 +图中链接了本包宿主模块、且本包构建程序实例上该特性激活的每个构建程序都可见该程序,包括经另一个包 +到达本包的消费方(SPEC-007 R6.3)。 -**状态:已实现**(mcpp 2026.9.27.1,mcpp#709)。 +**状态:已实现**(mcpp 2026.9.27.1,mcpp#709;到达链接本包库的每个构建程序:mcpp 2026.11.19.1,mcpp#955)。 ### 10.3 `artifacts`:随消费方发布的程序 @@ -857,4 +859,4 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m | 1.19 | 2026-10-28 | mcpp 2026.10.27.1(mcpp#847):§2 增加平台平面(`[platform]`、`[host]`),平台键可出现在工程与机器默认中;§4.2 增加 `[target.]` 的平台键与 `[host] platform`(SPEC-013)。 | | 1.20 | 2026-11-17 | mcpp 2026.11.17.1(设计 2026-11-16 §4):新增 §15 运行时文件与发布文件:`deploy` 的条目语法(字符串、通配符、`name`)、固定目录规则与过期副本的移除;`license_files` 与其缺省值。 | | 1.21 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):新增 §4.8 包的依赖表由构建它的那一行读取;两侧各自发出请求,锁记录两侧。 | -| 1.22 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#931):新增 §8.1 链接的输入:链接词点名的文件(选项的值、`,` 分隔的段、MSVC `/` 选项的值、动作声明的输出)与链接器自己的报告。 | +| 1.22 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#931):新增 §8.1 链接的输入:链接词点名的文件(选项的值、`,` 分隔的段、MSVC `/` 选项的值、动作声明的输出)与链接器自己的报告;§10.2 特性的 `tools` 到达链接本包库的每个构建程序(mcpp#955)。 | diff --git a/docs/zh/30-build-mcpp.md b/docs/zh/30-build-mcpp.md index 38a9ab2ad..1b1cc3d35 100644 --- a/docs/zh/30-build-mcpp.md +++ b/docs/zh/30-build-mcpp.md @@ -988,8 +988,14 @@ mcpp 会写出 `<暂存树>.stage-manifest` —— 一个兄弟文件,永不 | `${mcpp.compile_db}` | `compile_commands.json` 的路径(clang-tidy 的 `-p` 要的就是它) | | `${mcpp.contract}` *(2026.10.30.1+)* | 配置的契约 `mcpp-contract.json` 的路径,与该配置的编译数据库同目录:每种语言的编译器与每个单元都带的词(目标、sysroot、目标侧的头文件、平台与 profile 的标志),以及链接驱动、它的词与归档工具。外部构建系统读它,以引擎的方式为目标编译;包自己的包含目录不在其中,引擎为自己的单元选择的 C++ 方言与模块词(`-std=`、`-fmodules`)也不在其中。自 2026.11.7.1 起,链接参数描述目标配置,不含消费包自己的链接标志和库;这些库可能正是外部构建尚待生成的产物 | | `${mcpp.target_file:}` | target `` 构建出的文件 | +| `${mcpp.objects:}` *(2026.11.19.1+)* | 一个清单文件的绝对路径,清单按链接顺序、每行一个绝对路径列出 target `` 的链接作为显式输入的文件:它的对象、静态并入的依赖与标准库模块的对象,不含以 `-l` 指名的库。只用于 `artifact` 与 `check` 角色;清单与其中每个文件都是该 action 的输入,对象重新编译后 action 重新运行。`mcpp::objects_of("")` 返回该记号(协议 20) | +| `${mcpp.objects_rsp:}` *(2026.11.19.1+)* | 同一清单按响应文件的引号规则写出,供以 `@file` 读取的工具(`ar`、`llvm-ar`、`lib.exe`)使用;含空格的路径加引号。`mcpp::objects_rsp_of("")` 返回该记号 | | `${mcpp.stage_dir}` *(2026.9.11.1+)* | `mcpp pack` 暂存出的那棵树,绝对路径。仅 `artifact` role 可用,且仅在 `mcpp pack --pkg-format ` 下可用 | -| `${mcpp.self}` *(2026.9.13.1+)* | 引擎自己的可执行文件,绝对路径。action 的命令是没有 shell 的 argv,构建程序因此没有可移植的拷贝手段;引擎在构建运行的每台机器上都在,`${mcpp.self} stage --verify content --output ` 拷贝一个文件、创建目标的父目录、只在字节不同时写入。这个参数形状自 2026.9.13.1 起是契约;写下它的构建程序即以该版本为下限 | +| `${mcpp.self}` *(2026.9.13.1+)* | 引擎自己的可执行文件,绝对路径。action 的命令是没有 shell 的 argv,构建程序因此没有可移植的拷贝手段;引擎在构建运行的每台机器上都在,`${mcpp.self} stage --verify content --output ` 拷贝一个文件、创建目标的父目录、只在字节不同时写入。这个参数形状自 2026.9.13.1 起是契约;写下它的构建程序即以该版本为下限。`${mcpp.self} write --output <文件> [--append <文件>]... -- <行>...` *(2026.11.19.1+)* 写出 `--` 之后的各行(每行以换行结尾),再写出每个 `--append` 文件的字节(这些文件须声明为 action 的输入),只在字节不同时写入 | + +引擎自身的命令以一条判据为界(SPEC-007 R3.17):一个输出,内容完全由声明的输入与 argv 决定, +重复执行结果相同。`stage` 与 `write` 满足;删除、建目录、通配、按顺序运行多个程序不满足:多步动作 +是插件携带的程序,作为其某个特性的工具声明(见下文“依赖产出的 host 工具”),不是引擎解释的一组词汇。 `${mcpp.*}` 展开出的每个路径都用**正斜杠**拼写。token 会被拼进构建程序写下的 文本,而那段文本用自己的分隔符 `/`;在 Windows 上按原生拼写就会把 `C:\…` 前缀 @@ -1687,6 +1693,12 @@ int main() { return grpcgen::generate_all() ? 0 : 1; } `path` / `git` 不合并 —— feature 仍然无法静默覆盖无条件条目的身份。 - **传播的是可见性,不是执行。** `dep_bin()` 只返回路径,跑不跑仍由消费者的 `build.mcpp` 决定;谁构建了这个工具、tool store 怎么做键,都不改变。 +- **特性的工具到达链接该包库的每个构建程序**(2026.11.19.1+)。插件的库在链接它的那个包的构建 + 程序中运行,经框架到达的使用者也在内;`[features.] tools = [""]` 激活时,这些构建程序都 + 能取得该工具,插件因此携带自己的程序。此前路径只到达写出插件那条边的包,隔一个包 `dep_bin` + 回答 `""`。 +- **无人声明的工具被报告**(2026.11.19.1+,协议 20)。没有任何边、也没有任何激活的特性声明的工具, + `dep_bin` 回答 `""`,引擎给出警告(`build-program/undeclared-tool`),写出包、工具与声明它的那一行。 - **裸名由阶梯决定,而不是靠运气。** 一旦两个库都能再导出,它们可能同时提供 尾名 `protobuf`。全限定的 `MCPP_DEP___BIN_` 总是发布;裸名 依次绑定到 `mcpplibs.`、`compat.`、无命名空间的 ``,最后才是「剩下 diff --git a/docs/zh/31-authoring-a-rule-package.md b/docs/zh/31-authoring-a-rule-package.md index 641a05079..4809a5403 100644 --- a/docs/zh/31-authoring-a-rule-package.md +++ b/docs/zh/31-authoring-a-rule-package.md @@ -24,7 +24,7 @@ mcpp 的构建表面由包扩展,而不是由发布扩展。做扩展的有五 | `mcpp::action` | 构建图里的一条边:一条命令,带声明的输入与输出 | 构建程序,或它 import 的规则模块 | | `device_extensions` | 一个扩展名被引擎归类为**设备源**,而不是被拒绝 | 包的某个 feature | | `rule_module` | 消费方的构建程序 import 哪个模块来够到这条规则 | 同一个 feature | -| `tools = [...]` | 一个**从源码为构建机构建出来**的生成器或编译器,用 `mcpp::dep_bin` 取到 | 一条依赖边 | +| `tools = [...]` | 一个**从源码为构建机构建出来**的生成器或编译器,用 `mcpp::dep_bin` 取到 | 一条依赖边,或包的一个特性(`[features.] tools`),后者到达链接该包库的每个构建程序(2026.11.19.1+) | | `[xlings]`、`[feature-xlings]` | 规则要运行的预建工具,按需安装 | 包本身,或它的某个 feature | 生态用它们建出来的东西: diff --git a/modules/buildmcpp/src/directives.cppm b/modules/buildmcpp/src/directives.cppm index 9c28a220c..56413787e 100644 --- a/modules/buildmcpp/src/directives.cppm +++ b/modules/buildmcpp/src/directives.cppm @@ -217,6 +217,10 @@ enum class Slot : std::size_t { // table it is in data -- `cmd`, `host`, `hosts`, `requires`, `enabled`, // `longlived`, `exclusive` -- in one JSON payload (`mcpp::runner_spec`). RunnerSpecs, + // Protocol 20 (design 2026-11-19 §8.4): `\t` for each tool + // `mcpp::dep_bin` was asked for that no edge and no feature declares. The + // program received "" and goes on; the engine says which line declares it. + UndeclaredTools, Count }; inline constexpr std::size_t kSlotCount = static_cast(Slot::Count); @@ -315,7 +319,7 @@ struct Def { int sinceProtocol; }; -inline constexpr std::array kTable{{ +inline constexpr std::array kTable{{ // wire tag slot scope transform must missingPrefix missingSuffix since {"cxxflag", "cxxflag", Slot::CxxFlags, Scope::PackagePrivate, Transform::Verbatim, false, "", "", 1}, {"cflag", "cflag", Slot::CFlags, Scope::PackagePrivate, Transform::Verbatim, false, "", "", 1}, @@ -430,6 +434,7 @@ inline constexpr std::array kTable{{ // the unknown-tag path and discards the whole record. {"warning", "warning", Slot::Warnings, Scope::Advisory, Transform::Verbatim, false, "", "", 5}, {"diagnostic", "diagnostic", Slot::Diagnostics, Scope::Advisory, Transform::Verbatim, false, "", "", 14}, + {"undeclared-tool", "undeclared-tool", Slot::UndeclaredTools, Scope::Advisory, Transform::Verbatim, false, "", "", 20}, {"action", "action", Slot::Actions, Scope::GraphNode, Transform::Verbatim, false, "", "", 1}, // The probe channel: a rule package measures, the engine compares. See // Slot::Facts for the shape of each value. diff --git a/modules/buildmcpp/src/program_protocol.cppm b/modules/buildmcpp/src/program_protocol.cppm index 8332a3050..510ac0d9b 100644 --- a/modules/buildmcpp/src/program_protocol.cppm +++ b/modules/buildmcpp/src/program_protocol.cppm @@ -133,7 +133,14 @@ export namespace mcpp::build::program_protocol { // when no program is named. Same cost as v5's: a package calling // `mcpp::pack_program()` fails on an older engine at the build.mcpp COMPILE. // No directive of v18 changes spelling. -inline constexpr int kProtocolVersion = 19; +// v20 (design 2026-11-19 §8): `mcpp::objects_of(name)` and +// `mcpp::objects_rsp_of(name)` -- the tokens `${mcpp.objects:}` and +// `${mcpp.objects_rsp:}`, a link unit's object list -- and +// `mcpp:undeclared-tool=`, which `dep_bin` states for a tool no edge and no +// feature declares. Same cost as v5's: a package calling `objects_of` fails on +// an older engine at the build.mcpp COMPILE. No directive of v19 changes +// spelling. +inline constexpr int kProtocolVersion = 20; // ── Cache-format epoch ───────────────────────────────────────────────────── // diff --git a/modules/manifest/src/types.cppm b/modules/manifest/src/types.cppm index 560a80bf7..3d9e6e1b8 100644 --- a/modules/manifest/src/types.cppm +++ b/modules/manifest/src/types.cppm @@ -716,6 +716,12 @@ struct BuildAction { // member that check refused. The dispatch asks instead which artifact // actions the REQUEST INTRODUCED; see mcpp.pack.pipeline. bool consumesStageDir = false; + // Set by the engine (design 2026-11-19 §8.2): the build-dir-relative + // outputs of the link units whose object lists this action's command or + // inputs name through `${mcpp.objects:}` or + // `${mcpp.objects_rsp:}`. The backend writes each list and makes it, + // and every object in it, an implicit input of the action. + std::vector objectsOf; std::vector inputs; // absolute or package-relative std::vector outputs; // ditto; declared, see INV-D // Object only: which link units receive the outputs. Empty = every LINKED diff --git a/src/build/build_program.cppm b/src/build/build_program.cppm index 00a521dba..387916ad3 100644 --- a/src/build/build_program.cppm +++ b/src/build/build_program.cppm @@ -58,6 +58,28 @@ void report_stated_diagnostics(std::string_view packageName, sd.impact, sd.hint); } } +// Design 2026-11-19 §8.4: a tool `mcpp::dep_bin` was asked for that no edge +// and no feature declares. The program received "" -- which also reads as a +// tool the build machine lacks -- so the engine names the declaration. +void report_undeclared_tools(std::string_view packageName, + const mcpp::build::directives::Directives& d) { + namespace dirs = mcpp::build::directives; + std::set said; + for (auto const& raw : d.at(dirs::Slot::UndeclaredTools)) { + if (!said.insert(raw).second) continue; + const auto tab = raw.find('\t'); + const auto pkg = raw.substr(0, tab); + const auto tool = tab == std::string::npos ? std::string{} : raw.substr(tab + 1); + mcpp::diag::report(mcpp::diag::Severity::Warning, "build-program/undeclared-tool", + std::format("{}: the build program asked for the tool '{}' of '{}', which no " + "dependency edge and no active feature declares", + packageName, tool, pkg), + "mcpp::dep_bin answered \"\", and the program ran without the tool", + std::format("declare it on the edge, `\"{}\" = {{ ..., tools = [\"{}\"] }}` under " + "[build-dependencies], or in the providing package's feature, " + "`[features.] tools = [\"{}\"]`", pkg, tool, tool)); + } +} } // namespace export namespace mcpp::build { @@ -1240,6 +1262,7 @@ std::expected run_build_program_impl( for (auto const& a : dirs::advisories(m.package.name, cache.directives)) mcpp::ui::warning(a); report_stated_diagnostics(m.package.name, cache.directives); + report_undeclared_tools(m.package.name, cache.directives); mcpp::build::progress::program_finished( who, env.requested, mcpp::build::progress::ProgramOutcome::Cached, {}, {}); return {}; @@ -1524,6 +1547,7 @@ std::expected run_build_program_impl( for (auto const& a : dirs::advisories(m.package.name, d)) mcpp::ui::warning(a); report_stated_diagnostics(m.package.name, d); + report_undeclared_tools(m.package.name, d); // A run kept although it asked for a payload (`PrepareIntent::Describe`) is never // cached: replaying it would keep the build from ever asking again. if (cacheThisRun) { diff --git a/src/build/hostprogram.cppm b/src/build/hostprogram.cppm index 5d7059def..b86f6298c 100644 --- a/src/build/hostprogram.cppm +++ b/src/build/hostprogram.cppm @@ -1089,9 +1089,13 @@ inline void toolchain(const char* key, const char* value) { } // mcpp#355: absolute path to a HOST tool built by a dependency — the binary -// behind one of its `kind = "bin"` targets. Returns "" unless the consumer -// declared it: = { version = "…", tools = ["protoc"] } -// The path already carries the platform's executable suffix. +// behind one of its `kind = "bin"` targets. Declared on an edge, +// = { version = "…", tools = ["protoc"] } +// or by a feature of the package (`[features.] tools`), which reaches every +// build program that links the package's library with the feature active. +// The path already carries the platform's executable suffix. A tool no edge +// and no feature declares answers "" and is reported by the engine +// (`build-program/undeclared-tool`, protocol 20) with the line to write. inline const char* dep_bin(const char* pkg, const char* tool) { char buf[256] = "MCPP_DEP_"; unsigned long o = 9; @@ -1106,8 +1110,40 @@ inline const char* dep_bin(const char* pkg, const char* tool) { buf[o++] = '_'; buf[o++] = 'B'; buf[o++] = 'I'; buf[o++] = 'N'; buf[o++] = '_'; put(tool); buf[o] = 0; - return env_or(buf); + const char* path = env_or(buf); + if (!*path) std::printf("mcpp:undeclared-tool=%s\t%s\n", pkg, tool); + return path; +} + +// ── The objects of a link unit (protocol 20, design 2026-11-19 §8.2) ──────── +// +// Which objects a program or library is linked from is the engine's fact, and +// a rule that walks `obj/` afterwards guesses it. `objects_of("app")` is the +// token `${mcpp.objects:app}`, which the engine replaces in an `artifact` or +// `check` action with the absolute path of a file listing, one absolute path +// per line in link order, the objects the unit's link takes (the static +// dependencies' and the standard module's included, libraries not); the list +// and every object in it are inputs of the action. `objects_rsp_of` is the +// same list in response-file quoting, for a tool that reads `@file` (`ar`, +// `llvm-ar`, `lib.exe`). A build program that calls them fails to compile on +// an engine that does not replace the tokens, rather than handing a tool the +// token unreplaced. +struct token_text { + char text[320] = ""; + const char* c_str() const { return text; } + operator const char*() const { return text; } +}; +inline token_text token_of_(const char* head, const char* name) { + token_text t; + unsigned long o = 0; + for (const char* p = head; *p && o + 2 < sizeof t.text; ++p) t.text[o++] = *p; + for (const char* p = name; *p && o + 2 < sizeof t.text; ++p) t.text[o++] = *p; + t.text[o++] = '}'; + t.text[o] = 0; + return t; } +inline token_text objects_of(const char* name) { return token_of_("${mcpp.objects:", name); } +inline token_text objects_rsp_of(const char* name) { return token_of_("${mcpp.objects_rsp:", name); } } // ── Protocol announcement ─────────────────────────────────────────────── // Emitted before main() runs, so a program that uses `import mcpp;` never has diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 5eb83a766..8c09b50e9 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -1116,6 +1116,38 @@ read_graph_c_library_sidecar(const std::filesystem::path& outputDir) { namespace { constexpr std::string_view kConsumerIncludeSidecar = ".mcpp-consumer-include-dirs"; +// One path as a response file states it for GNU (`ar`, `llvm-ar`, the +// drivers) and for `lib.exe`: plain when it holds no separator or quote, +// otherwise in double quotes with `"` and `\` escaped. The paths are generic +// (`/`), so a Windows path carries no backslash to be read two ways. +std::string response_file_word(std::string_view path) { + if (path.find_first_of(" \t\"'\\") == std::string_view::npos) return std::string(path); + std::string out = "\""; + for (char c : path) { + if (c == '"' || c == '\\') out.push_back('\\'); + out.push_back(c); + } + out.push_back('"'); + return out; +} + +// A file whose content the plan decides, written beside itself and renamed, +// and only when its bytes differ: a consumer that depends on it does not run +// again for an equal plan. +void write_text_if_changed(const std::filesystem::path& file, const std::string& text) { + if (std::ifstream in(file, std::ios::binary); in) { + const std::string was{std::istreambuf_iterator(in), std::istreambuf_iterator()}; + if (was == text) return; + } + std::error_code ec; + std::filesystem::create_directories(file.parent_path(), ec); + auto tmp = file; + tmp += std::format(".{:x}.tmp", std::random_device{}()); + { std::ofstream(tmp, std::ios::binary | std::ios::trunc) << text; } + std::filesystem::rename(tmp, file, ec); + if (ec) std::filesystem::remove(tmp, ec); +} + // The header named by one compiler diagnostic line, or empty. Three spellings, // one per driver family; anything else is not a missing-header report. std::string missing_header_in(std::string_view line) { @@ -3599,6 +3631,9 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // artifact naming — see triple.cppm) AND, unless `-sSINGLE_FILE=1`, // `.wasm` beside it in one run: a companion of the row (N33). + // Each link unit's explicit inputs as its edge spells them, by its + // build-dir-relative output: the object lists of design 2026-11-19 §8.2. + std::map> linkInsByOutput; // Link units for (auto& lu : plan.linkUnits) { attribute(lu.package); @@ -3607,6 +3642,13 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, const auto& uflags = lu.linkGroup >= 0 ? groupFlags[static_cast(lu.linkGroup)] : flags; std::string ins; + // The explicit inputs as the edge spells them, in link order: what + // `${mcpp.objects:}` lists (design 2026-11-19 §8.2). + std::vector linkIns; + auto add_in = [&](const std::filesystem::path& p) { + ins += " " + escape_ninja_path(p); + linkIns.push_back(p); + }; // FIRST, before every other object. Mach-O runs __init_offsets in // LINK order and has no priority-ordered init section, so "runs // before the user's global constructors" is spelled "is the first @@ -3621,9 +3663,9 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // fixture). The predicate is the one the link line itself uses. if (need_ios_init_shim && lu.kind != LinkUnit::StaticLibrary && unit_needs_cxx_runtime(lu)) - ins += " " + escape_ninja_path(ios_init_obj); + add_in(ios_init_obj); for (auto& o : lu.objects) { - ins += " " + escape_ninja_path(o); + add_in(o); } // The platform's link inputs, in every executable and shared library // this plan links; with seal steps, the artifact is written beneath @@ -3631,7 +3673,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // what bin/ holds is what was published, and an unsealed file is not. const bool platformUnit = pf.active && lu.kind != LinkUnit::StaticLibrary; if (platformUnit) - for (auto const& in : platformInputs) ins += " " + escape_ninja_path(in); + for (auto const& in : platformInputs) add_in(in); const bool sealUnit = platformSeal && lu.kind != LinkUnit::StaticLibrary; const auto unsealed = link_edge_output(plan, lu); @@ -3652,19 +3694,20 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, switch (lu.kind) { case LinkUnit::Binary: case LinkUnit::TestBinary: - if (takesStd) ins += " " + escape_ninja_path(std_o_dst); - if (takesCompat) ins += " " + escape_ninja_path(compat_o_dst); + if (takesStd) add_in(std_o_dst); + if (takesCompat) add_in(compat_o_dst); rule = cxxUnit ? "cxx_link" : "c_link"; break; case LinkUnit::StaticLibrary: rule = "cxx_archive"; // `ar`: no driver, no runtime break; case LinkUnit::SharedLibrary: - if (takesStd) ins += " " + escape_ninja_path(std_o_dst); - if (takesCompat) ins += " " + escape_ninja_path(compat_o_dst); + if (takesStd) add_in(std_o_dst); + if (takesCompat) add_in(compat_o_dst); rule = cxxUnit ? "cxx_shared" : "c_shared"; break; } + linkInsByOutput[lu.output.generic_string()] = linkIns; std::string implicit; for (auto& input : lu.implicitInputs) { implicit += " " + escape_ninja_path(input); @@ -4186,6 +4229,27 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, std::string outs, ins; for (auto const& o : a.outputs) outs += " " + escape_ninja_path(o); for (auto const& in : a.inputs) ins += " " + escape_ninja_path(in); + // THE OBJECT LISTS THE ACTION READS (design 2026-11-19 §8.2): written + // now, when their content is decided, and only when it differs; each + // list and every object in it are inputs of the action, spelled as the + // link edge spells them, so a recompiled object runs the action again + // and an unchanged build does not. + for (auto const& unit : a.objectsOf) { + auto it = linkInsByOutput.find(unit); + if (it == linkInsByOutput.end()) continue; + std::string lines, rsp; + for (auto const& o : it->second) { + const auto abs = (o.is_absolute() ? o : plan.outputDir / o).lexically_normal(); + lines += abs.generic_string() + "\n"; + rsp += response_file_word(abs.generic_string()) + "\n"; + ins += " " + escape_ninja_path(o); + } + const auto list = mcpp::build::objects_list_path(plan, unit, ".list"); + const auto quoted = mcpp::build::objects_list_path(plan, unit, ".rsp"); + write_text_if_changed(list, lines); + write_text_if_changed(quoted, rsp); + ins += " " + escape_ninja_path(list) + " " + escape_ninja_path(quoted); + } append(std::format("build{} : mcpp_action_{}{}\n", outs, i, ins)); // A check or prepare action reports its start through the wrapper, // by its first output (design §6.4); the record gives it its label. diff --git a/src/build/plan.cppm b/src/build/plan.cppm index ed8cf9f2b..7c25c0a31 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -893,6 +893,22 @@ struct TargetFileAnswer { TargetFileAnswer resolve_target_file(const BuildPlan& plan, std::string_view name, std::string_view actingMember); +// Where the backend writes the object list of the link unit whose +// build-dir-relative output is `output` (design 2026-11-19 §8.2): +// `/objects/`, `.list` one absolute path per line and +// `.rsp` in response-file quoting. Absolute and generic, as every path token is. +std::filesystem::path objects_list_path(const BuildPlan& plan, std::string_view output, + std::string_view suffix); + +// Replaces `${mcpp.objects:}` and `${mcpp.objects_rsp:}` in `s` +// with `objects_list_path` of the unit `resolve_target_file` names, records the +// unit in `units` once, and a name no unit has, or several members define, in +// the two sets the refusal reads. True when `s` held either token. +bool substitute_objects_tokens(const BuildPlan& plan, std::string& s, std::string_view actingMember, + std::vector& units, + std::set& unresolved, + std::map>& ambiguous); + // The references an action's arguments made that the plan cannot answer, as // the refusal a user reads: names no link unit has (`unresolvedTargets`), names // several members define for a package that acts for none of them @@ -4803,6 +4819,40 @@ make_plan(const mcpp::manifest::Manifest& manifest, return plan; } +// The name is resolved as `${mcpp.target_file:}` resolves it; the token becomes +// the absolute path of the list the backend writes, one object per line +// (`objects`) or in response-file quoting (`objects_rsp`). +bool substitute_objects_tokens(const BuildPlan& plan, std::string& s, std::string_view actingMember, + std::vector& units, + std::set& unresolved, + std::map>& ambiguous) { + bool used = false; + constexpr std::array, 2> kTokens{{ + {"${mcpp.objects:", ".list"}, {"${mcpp.objects_rsp:", ".rsp"}}}; + for (auto [head, suffix] : kTokens) { + for (std::size_t p; (p = s.find(head)) != std::string::npos; ) { + auto close = s.find('}', p); + if (close == std::string::npos) break; + used = true; + auto name = s.substr(p + head.size(), close - p - head.size()); + auto answer = resolve_target_file(plan, name, actingMember); + if (answer.output.empty()) unresolved.insert(name); + else if (!answer.members.empty()) ambiguous[name] = answer.members; + if (!answer.output.empty() && std::ranges::find(units, answer.output) == units.end()) + units.push_back(answer.output); + s.replace(p, close - p + 1, objects_list_path(plan, answer.output, suffix).generic_string()); + } + } + return used; +} + +std::filesystem::path objects_list_path(const BuildPlan& plan, std::string_view output, + std::string_view suffix) { + auto p = plan.outputDir / "objects" / std::filesystem::path(std::string(output)); + p += std::string(suffix); + return p.lexically_normal(); +} + TargetFileAnswer resolve_target_file(const BuildPlan& plan, std::string_view name, std::string_view actingMember) { TargetFileAnswer answer; diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index e662e9ff8..e66bcfd2e 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -1195,6 +1195,36 @@ static std::expected step6_xlings_workspace_from_graph(Prepar return {}; } +// The tools the active features of `library`'s build-program instance declare. +static std::vector active_feature_tools(const PrepareState& state, std::size_t library) { + std::vector out; + auto const& declared = state.packages[library].manifest.featureTools; + for (auto const& f : state.buildProgramFeaturesByPackage[library]) { + auto it = declared.find(f); + if (it == declared.end()) continue; + out.insert(out.end(), it->second.begin(), it->second.end()); + } + return out; +} + +// A FEATURE'S TOOLS REACH EVERY BUILD PROGRAM THAT LINKS THE PACKAGE'S LIBRARY +// (design 2026-11-19 §8.4, mcpp#955). A plugin's library runs inside the +// build program of whichever package links it -- a framework's user, reached +// through the framework -- and asks for its own program with +// `mcpp::dep_bin`. The edge into the plugin is the framework's, so the +// visibility above stopped at the framework and the user's program received +// "". With the feature active for the build-program instance, the program +// that links the library sees the tool, as the library's author declared. +static void grant_library_feature_tools(PrepareState& state, std::size_t consumer) { + for (const auto library : state.buildProgramLibrariesByConsumer[consumer]) { + if (library >= state.packages.size() || library >= state.buildProgramFeaturesByPackage.size()) + continue; + for (auto const& tool : active_feature_tools(state, library)) + state.provisionGraph.visible[consumer].insert( + prov::Provision{prov::Kind::Tool, library, tool}); + } +} + static std::expected>, std::string> step6_tool_requests(PrepareState& state) { // ── #355: HOST tool provisioning ──────────────────────────────────── @@ -1269,6 +1299,7 @@ step6_tool_requests(PrepareState& state) { work.push_back(e.dependencyPackageIndex); } state.buildProgramLibrariesByConsumer[c].assign(seen.begin(), seen.end()); + grant_library_feature_tools(state, c); } if (state.overrides.tool_depth >= mcpp::build::tool_store::kMaxDepth diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index 1bf8e3b8a..75d64984c 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -1156,6 +1156,14 @@ static std::expected step13_build_graph_actions(PrepareState& // NAMES ids and a collision there costs a duplicate line, not a wrong // edge. bool thisActionUsesStageDir = false; + // `${mcpp.objects:}` / `${mcpp.objects_rsp:}` (design + // 2026-11-19 §8.2): the link units whose object lists THIS action + // reads, by their build-dir-relative output; the backend writes the + // lists and makes them and their objects inputs of the action. + std::vector thisActionObjects; + // The token used by an action whose role runs before or inside the + // compile and link the list is drawn from. + std::set objectsWrongRole; // Where the declaring package's binaries land: `bin/`, or in a // workspace plan the product directory of the member that declared // the action (§15 of the 2026-09-29 workspace design). Set per @@ -1235,6 +1243,13 @@ static std::expected step13_build_graph_actions(PrepareState& else if (!answer.members.empty()) ambiguousTargets[name] = answer.members; s.replace(p, close - p + 1, answer.output); } + // A link unit's objects (design 2026-11-19 §8.2), read only by a + // role that runs after the link it is drawn from. + if (mcpp::build::substitute_objects_tokens(ctx.plan, s, actingMember, thisActionObjects, + unresolvedTargets, ambiguousTargets) + && role != mcpp::manifest::BuildAction::Role::Artifact + && role != mcpp::manifest::BuildAction::Role::Check) + objectsWrongRole.insert(actionId); // `${mcpp.artifact:/}` (mcpp#711): a dependency's // program that an edge requested with `artifacts = [...]`, spelled // like `${mcpp.target_file:}` -- the link unit's build-dir-relative @@ -1280,6 +1295,7 @@ static std::expected step13_build_graph_actions(PrepareState& if (!g.linkOnly && g.member == owner) binDir = ctx.plan.outputDir / g.productDir; for (auto a : mm.buildConfig.actions) { thisActionUsesStageDir = false; + thisActionObjects.clear(); const auto sub = [&](std::string v) { return substitute(std::move(v), a.id.c_str(), a.role); }; @@ -1308,6 +1324,7 @@ static std::expected step13_build_graph_actions(PrepareState& a.inputs.push_back( mcpp::pack::stage_manifest_path(stage->dir).string()); } + a.objectsOf = thisActionObjects; a.packageName = owner; ctx.plan.actions.push_back(std::move(a)); } @@ -1384,6 +1401,17 @@ static std::expected step13_build_graph_actions(PrepareState& "there is anything to stage.\n" " use: role = \"artifact\"", ids)); } + if (!objectsWrongRole.empty()) { + std::string ids; + for (auto const& n : objectsWrongRole) ids += (ids.empty() ? "" : ", ") + n; + return std::unexpected(std::format( + "build.mcpp action(s) [{}] reference ${{mcpp.objects:}} or " + "${{mcpp.objects_rsp:}} with a role other than \"artifact\" or \"check\".\n" + " The list names the objects a link takes; a source, object or prepare action " + "runs before\n" + " that link or feeds it, so the list it would read is not yet decided.\n" + " use: role = \"artifact\" (or \"check\")", ids)); + } if (auto refused = mcpp::build::refuse_unresolved_references( ctx.plan, unresolvedTargets, ambiguousTargets, unresolvedArtifacts); !refused) diff --git a/src/cli.cppm b/src/cli.cppm index 21d384dfb..3b635c10c 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -1397,6 +1397,18 @@ int run(int argc, char** argv) { .option(cl::Option("list").takes_value().value_name("FILE") .help("Place every `\\t` pair of FILE in one process (#734 E4); several lines naming one destination are its sources")) .action(cmd_stage)) + // Named by a build program's actions through `${mcpp.self}` (SPEC-007 + // R3.8): one output, its content decided by the lines after `--` and + // the files named by `--append`, written only when the bytes differ. + .subcommand(cl::App("write") + .description("Write the lines after `--`, then each --append file, to --output; write only when the content differs (invoked by ${mcpp.self} actions)") + .option(cl::Option("output").short_name('o').takes_value().value_name("PATH") + .help("Destination path; its parent directory is created")) + .option(cl::Option("append").takes_value().multiple().value_name("FILE") + .help("A file whose bytes follow the lines, in the order given; declare it as the action's input")) + .action([&passthrough](const cl::ParsedArgs& p) { + return cmd_write(p, std::span(passthrough)); + })) .subcommand(cl::App("link-output") .description("(internal: invoked by ninja) Remove the row's companions, move a linked artifact aside, then run the linker command after `--` (a running program keeps its file)") .option(cl::Option("output").takes_value().value_name("PATH").help("the artifact the command links")) @@ -1903,7 +1915,7 @@ int run(int argc, char** argv) { "new", "build", "run", "test", "clean", "add", "remove", "update", "search", "publish", "pack", "emit", "xpkg", "toolchain", "cache", "index", "self", "explain", - "version", "dyndep", "why", "resolve", "stage", "coff-def", + "version", "dyndep", "why", "resolve", "stage", "write", "coff-def", "place-dlls", "link-output", "platform-step", "bmi-compile", "bmi-supervise", "bmi-await", }); diff --git a/src/cli/cmd_build.cppm b/src/cli/cmd_build.cppm index 8de9ce777..f08cae4df 100644 --- a/src/cli/cmd_build.cppm +++ b/src/cli/cmd_build.cppm @@ -2259,6 +2259,58 @@ export int cmd_install_configuration(std::string_view target) { return 0; } +// `${mcpp.self} write --output [--append ]... -- ...` +// (SPEC-007 R3.8, design 2026-11-19 §8.3). The family of the engine's own +// commands is bounded by one criterion: one output, whose content the +// declared inputs and the argv decide, the same on every run. A line is +// written with `\n` after it; an appended file's bytes follow unchanged. The +// output is written beside itself and renamed, and only when its bytes differ, +// so a consumer that depends on it is not run again for an equal result. +export int cmd_write(const mcpplibs::cmdline::ParsedArgs& parsed, + std::span lines) { + namespace fs = std::filesystem; + const fs::path out = parsed.option_or_empty("output").value(); + if (out.empty()) { + std::println(libc::stderr(), "error: write needs --output [--append ]... -- ..."); + return 2; + } + std::string text; + for (auto const& line : lines) { text += line; text += '\n'; } + for (auto const& name : parsed.option_or_empty("append").values) { + std::ifstream in(fs::path{name}, std::ios::binary); + if (!in) { + std::println(libc::stderr(), "error: write: cannot read {}, which --append names", name); + return 1; + } + text.append(std::istreambuf_iterator(in), std::istreambuf_iterator()); + } + std::error_code ec; + if (std::ifstream old(out, std::ios::binary); old) { + const std::string was{std::istreambuf_iterator(old), std::istreambuf_iterator()}; + if (was == text) return 0; + } + if (out.has_parent_path()) fs::create_directories(out.parent_path(), ec); + auto tmp = out; + tmp += std::format(".{:x}.tmp", std::random_device{}()); + { + std::ofstream f(tmp, std::ios::binary | std::ios::trunc); + f << text; + if (!f) { + std::println(libc::stderr(), "error: write: cannot write {}", tmp.string()); + fs::remove(tmp, ec); + return 1; + } + } + fs::rename(tmp, out, ec); + if (ec) { + std::println(libc::stderr(), "error: write: cannot rename {} over {}: {}", + tmp.string(), out.string(), ec.message()); + fs::remove(tmp, ec); + return 1; + } + return 0; +} + export int cmd_stage(const mcpplibs::cmdline::ParsedArgs& parsed) { // `--list FILE` (#734 E4): many placements, one process. Each destination // keeps the single-file semantics below -- content comparison, an diff --git a/tests/e2e/actions/commands.cpp b/tests/e2e/actions/commands.cpp index e507f64f3..88dde0c95 100644 --- a/tests/e2e/actions/commands.cpp +++ b/tests/e2e/actions/commands.cpp @@ -228,6 +228,75 @@ int main() { expect(eq(mtime(*copy), before)) << "the copy was rewritten on a no-op rebuild"; }; + "959 an action reads a link unit's objects and writes through mcpp self"_test[s::gcc][s::unix_shell] = [] { + // mcpp#959, design 2026-11-19 §8.2-§8.3. `${mcpp.objects:app}` is a file listing the + // objects the program's link takes, one absolute path per line; `${mcpp.objects_rsp:app}` + // is the same list for a tool that reads `@file`, here the toolchain's `ar`, in a build + // directory whose path holds a space. `${mcpp.self} write` writes the lines after `--` + // and the appended list; a no-op build runs neither action, and a changed source runs + // them again. A source action that names the token is refused with the role. + const auto root = s::case_dir() / "with space"; + const std::string program = R"cpp(import std; +import mcpp; +int main() { + mcpp::action w; + w.id = "list"; + w.role = "artifact"; + w.arg("${mcpp.self}").arg("write").arg("--output").arg("${mcpp.out_dir}/objs.txt") + .arg("--append").arg(mcpp::objects_of("app")).arg("--").arg("# the objects of app") + .output("${mcpp.out_dir}/objs.txt") + .submit(); + mcpp::action a; + a.id = "archive"; + a.role = "artifact"; + a.arg(mcpp::tool("ar")).arg("rcs").arg("${mcpp.out_dir}/app-objects.a") + .arg(std::format("@{}", mcpp::objects_rsp_of("app").c_str()).c_str()) + .output("${mcpp.out_dir}/app-objects.a") + .submit(); +} +)cpp"; + s::project p{root, s::files{{"mcpp.toml", "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"src/main.cpp", "#include \nint helper();\nint main() { std::printf(\"%d\\n\", helper()); }\n"}, + {"src/helper.cpp", "int helper() { return 7; }\n"}, + {"build.mcpp", program}}}; + auto r = p.mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << "build failed: " << s::text(r); + const auto listed = s::find_files(p.at("target"), R"(objs\.txt)"); + expect(fatal(!listed.empty())) << "the write action produced nothing"; + const auto text = s::read_file(listed.front()); + expect(fatal(text.starts_with("# the objects of app\n"))) << "the lines after -- are not first:\n" << text; + expect(fatal(s::count_lines(text, "/main.o") == 1u && s::count_lines(text, "/helper.o") == 1u)) + << "the list does not name each object once:\n" << text; + for (std::istringstream in{text}; !in.eof();) { + std::string line; + std::getline(in, line); + if (line.empty() || line.starts_with("#")) continue; + expect(fs::path(line).is_absolute() && fs::is_regular_file(line)) << "not an absolute path of an object: " << line; + } + const auto archive = s::find_files(p.at("target"), R"(app-objects\.a)"); + expect(fatal(!archive.empty())) << "the archive action produced nothing"; + const auto members = s::run(*s::on_path("ar"), {"t", archive.front().string()}).out; + expect(e2e::contains(members, "main.o") && e2e::contains(members, "helper.o")) + << "ar read the response file wrongly in a path with a space:\n" << members; + const auto before = mtime(listed.front()); + std::this_thread::sleep_for(std::chrono::seconds{1}); + r = p.mcpp({"build", "-v"}); + expect(fatal(e2e::succeeded(r))) << "no-op rebuild failed: " << s::text(r); + expect(s::lacks(s::text(r), "app-objects.a")) << "a no-op build ran the archive action: " << s::text(r); + expect(eq(mtime(listed.front()), before)) << "the written file was rewritten on a no-op build"; + p.write("src/helper.cpp", "int helper() { return 8; }\n"); + r = p.mcpp({"build", "-v"}); + expect(fatal(e2e::succeeded(r))) << "rebuild failed: " << s::text(r); + expect(e2e::contains(s::text(r), "app-objects.a")) << "a recompiled object did not run the action: " << s::text(r); + p.write("build.mcpp", "import mcpp;\nint main() {\n mcpp::action a;\n a.id = \"early\";\n a.role = \"source\";\n" + " a.arg(\"${mcpp.self}\").arg(\"write\").arg(\"--output\").arg(\"${mcpp.out_dir}/gen.h\")\n" + " .arg(\"--append\").arg(mcpp::objects_of(\"app\")).arg(\"--\").output(\"${mcpp.out_dir}/gen.h\").submit();\n}\n"); + r = p.mcpp({"build"}); + expect(fatal(!e2e::succeeded(r))) << "a source action reading the object list was accepted"; + expect(e2e::contains(s::text(r), "with a role other than \"artifact\" or \"check\"") && e2e::contains(s::text(r), "early")) + << "the refusal does not name the action and the roles: " << s::text(r); + }; + "799 an action runs with its env and cwd"_test = [] { // `mcpp::action::env` and `::cwd`: the command sees the variable and runs in the // directory, its declared output is not resolved against that directory, a changed diff --git a/tests/e2e/build-program/hosttools.cpp b/tests/e2e/build-program/hosttools.cpp index a6856544c..7f09ca5eb 100644 --- a/tests/e2e/build-program/hosttools.cpp +++ b/tests/e2e/build-program/hosttools.cpp @@ -795,6 +795,49 @@ int main() { expect(e2e::contains(s::text(b3), "nosuch") && e2e::contains(s::text(b3), "codegen")) << "3: the refusal does not name the entry and the bin targets"; }; + "955 a feature's tool reaches every build program that links the package's library"_test[s::gcc] = [] { + // mcpp#955, design 2026-11-19 §8.4. `plug` has a library and a program `steps`; its + // library asks for the program with `mcpp::dep_bin`. Declared by a feature of `plug`, the + // tool reaches the build program of a package that names `plug` (A) and of one that reaches + // it through a framework (B). Declared nowhere, the program receives "" and the engine + // names the declaration (C). + const s::options home{.where = s::home::linked}; + const auto root = s::case_dir(); + const std::string with_feature = "[package]\nname = \"plug\"\nversion = \"0.1.0\"\n\n[features]\ndefault = [\"steps\"]\n\n" + "[features.steps]\ntools = [\"steps\"]\n\n[targets.plug]\nkind = \"lib\"\n\n" + "[targets.steps]\nkind = \"bin\"\nmain = \"tools/steps_main.cpp\"\n"; + s::project plug{root / "plug", {{"mcpp.toml", with_feature}, + {"src/plug.cppm", "export module plug;\nimport std;\nimport mcpp;\nexport void plug_declare() {\n" + " mcpp::warning(std::format(\"steps=[{}]\", mcpp::dep_bin(\"plug\", \"steps\")).c_str());\n}\n"}, + {"tools/steps_main.cpp", "int main() { return 0; }\n"}}}; + s::project fw{root / "fw", {{"mcpp.toml", "[package]\nname = \"fw\"\nversion = \"0.1.0\"\n\n[build-dependencies]\n" + "plug = { path = \"../plug\", host-module = true }\n\n[targets.fw]\nkind = \"lib\"\n"}, + {"src/fw.cppm", "export module fw;\nimport mcpp;\nimport plug;\nexport void fw_declare() { plug_declare(); }\n"}}}; + s::project direct{root / "direct", {{"mcpp.toml", "[package]\nname = \"direct\"\nversion = \"0.1.0\"\n\n[build-dependencies]\n" + "plug = { path = \"../plug\", host-module = true }\n\n[targets.direct]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"build.mcpp", "import mcpp;\nimport plug;\nint main() { plug_declare(); return 0; }\n"}, + {"src/main.cpp", "int main() { return 0; }\n"}}}; + s::project through{root / "through", {{"mcpp.toml", "[package]\nname = \"through\"\nversion = \"0.1.0\"\n\n[build-dependencies]\n" + "fw = { path = \"../fw\" }\n\n[targets.through]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"build.mcpp", "import mcpp;\nimport fw;\nint main() { fw_declare(); return 0; }\n"}, + {"src/main.cpp", "int main() { return 0; }\n"}}}; + const std::regex answered{R"(steps=\[[^\]]*steps[^\]]*\])"}; + auto r = direct.mcpp({"build"}, home); + expect(fatal(e2e::succeeded(r))) << "A: build failed: " << s::text(r); + expect(std::regex_search(s::text(r), answered)) << "A: the package naming plug did not receive the tool: " << s::text(r); + r = through.mcpp({"build"}, home); + expect(fatal(e2e::succeeded(r))) << "B: build failed: " << s::text(r); + expect(std::regex_search(s::text(r), answered)) << "B: the package reaching plug through fw did not receive the tool: " << s::text(r); + plug.write("mcpp.toml", "[package]\nname = \"plug\"\nversion = \"0.1.0\"\n\n[targets.plug]\nkind = \"lib\"\n\n" + "[targets.steps]\nkind = \"bin\"\nmain = \"tools/steps_main.cpp\"\n"); + through.remove("target"); + r = through.mcpp({"build"}, home); + expect(fatal(e2e::succeeded(r))) << "C: build failed: " << s::text(r); + expect(e2e::contains(s::text(r), "steps=[]")) << "C: an undeclared tool was answered: " << s::text(r); + expect(e2e::contains(s::text(r), "the tool 'steps' of 'plug', which no dependency edge and no active feature declares")) + << "C: an undeclared tool was not reported: " << s::text(r); + }; + "802 a host build applies its host row"_test[s::gcc][s::elf] = [] { // mcpp#704: a build without --target targets the host, and [target.] // describes it: without the row a plain build is self-contained (no NEEDED diff --git a/tests/unit/test_build_directives.cpp b/tests/unit/test_build_directives.cpp index bc1aefed4..fe9ed1b19 100644 --- a/tests/unit/test_build_directives.cpp +++ b/tests/unit/test_build_directives.cpp @@ -1104,16 +1104,28 @@ UTest deploy_row_is_protocol_eleven_with_link_global_scope_and_a_tag{"BuildDirec expect(eq(def->scope, dirs::Scope::LinkGlobal)); expect(eq(def->sinceProtocol, 11)); expect(!(def->tag.empty())); - expect(eq(dirs::kProtocolVersion, 19)); + expect(eq(dirs::kProtocolVersion, 20)); }}; -UTest protocol_nineteen_is_accepted_and_twenty_is_not{"BuildDirectives/ProtocolNineteenIsAcceptedAndTwentyIsNot", [] { - auto ok = parse("mcpp:protocol=19\n"); +UTest protocol_twenty_is_accepted_and_twenty_one_is_not{"BuildDirectives/ProtocolTwentyIsAcceptedAndTwentyOneIsNot", [] { + auto ok = parse("mcpp:protocol=20\n"); expect(!(dirs::protocol_error(ok).has_value())); - auto no = parse("mcpp:protocol=20\n"); + auto no = parse("mcpp:protocol=21\n"); expect(dirs::protocol_error(no).has_value()); }}; +// Design 2026-11-19 §8.4: a tool `dep_bin` was asked for and no edge or feature +// declares is an advisory of protocol 20, kept with the program's record. +UTest undeclared_tool_is_a_protocol_twenty_advisory{"BuildDirectives/UndeclaredToolIsAProtocolTwentyAdvisory", [] { + auto def = dirs::find_by_wire("undeclared-tool"); + expect(fatal(ne(def, nullptr))); + expect(eq(def->scope, dirs::Scope::Advisory)); + expect(eq(def->sinceProtocol, 20)); + expect(!(def->tag.empty())); + auto d = parse("mcpp:protocol=20\nmcpp:undeclared-tool=plug\tsteps\n"); + expect(eq(d.at(dirs::Slot::UndeclaredTools).size(), std::size_t{1})); +}}; + // SPEC-013 §4.1: the platform's fields a program states (protocol 17) become a // platform statement at the build-program tier; malformed lines are refused. UTest platform_fields_are_protocol_seventeen_and_reach_the_statement{"BuildDirectives/PlatformFieldsAreProtocolSeventeenAndReachTheStatement", [] { From e0c4637bc33f472c5e5f93ec5b334c1f825ec961 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 15:32:21 +0900 Subject: [PATCH 09/30] A package instance has a directory of its own, and a build keeps every instance its commands ask for (mcpp#953) A package's objects lie in obj/// and its BMIs in ///. The digest covers what decides the package's command lines: its include directories and flags, its features' defines among them, and the digests of the packages whose modules it imports. It holds no version and no compiler build. One build directory held one instance, so a shared package that mcpp test or another selection compiled with other features was compiled again at every switch; both instances are now kept. instances.tsv records when a build last used each instance. mcpp clean --stale removes the instances no build used within the window and keeps those of the plan that wrote the graph; mcpp clean --report lists them. A GCC BMI records its imports at their instance paths, so the build cache keys a package's entry by its instance. A @ name was measured and not taken: ninja quotes an output holding '@'. --- CHANGELOG.md | 16 +++ docs/07-workspace.md | 14 ++- docs/09-commands-by-scenario.md | 9 +- docs/specs/manifest-semantics.md | 11 +- docs/zh/07-workspace.md | 10 +- docs/zh/09-commands-by-scenario.md | 6 +- src/build/execute.cppm | 79 +++++++++++- src/build/ninja_backend.cppm | 5 + src/build/plan.cppm | 178 +++++++++++++++++++++++++++- src/build/prepare/plan.cpp | 14 ++- tests/e2e/deps/path.cpp | 3 +- tests/e2e/test/workspace.cpp | 2 +- tests/e2e/workspace/graph.cpp | 21 +++- tests/e2e/workspace/inheritance.cpp | 44 +++++++ tests/e2e/workspace/members.cpp | 2 +- tests/e2e/workspace/statements.cpp | 7 +- tests/unit/test_object_address.cpp | 29 ++++- 17 files changed, 418 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c4e601a2..30255338d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,22 @@ commands are bounded by one criterion, stated in SPEC-007 R3.17: one output, decided by the declared inputs and the argv, the same on every run. +### Changed + +- A package instance has a directory of its own: `obj///` and + `///`, the digest covering the package's include directories and + flags (its features' defines among them) and the digests of the packages whose modules it + imports (mcpp#953). One directory held one instance, so when `mcpp test` or another + selection asked a shared package for other features, each switch between the commands + compiled it again (measured: 43 such edges between two members of a 31-member workspace). + Both instances are now kept, and alternating compiles neither again. `instances.tsv` + records when a build last used each instance; `mcpp clean --stale` removes the instances + no build used within `[build] keep_days`, keeping those of the plan that wrote the graph, + and `mcpp clean --report` lists them. The root package's objects keep their place. A + `@` name was measured and not taken: ninja quotes an output that holds + `@`, and a rule writing `"$out.d"` then names a file with quotes in it. The objects of the + previous layout are not read again and go with `mcpp clean`. + ### Fixed - `mcpp publish` and `mcpp emit xpkg` wrote a descriptor that nothing read: a table of diff --git a/docs/07-workspace.md b/docs/07-workspace.md index 0414de1bc..2564f4c42 100644 --- a/docs/07-workspace.md +++ b/docs/07-workspace.md @@ -623,7 +623,14 @@ member that several members use is compiled once. declares it, and is refused when no selected member declares it. A package that several members of one configuration use is compiled once, with the union of the features they ask for; a package that members of two - configurations use is compiled once in each. + configurations use is compiled once in each. Each package instance has a + directory of its own, `obj///`, the digest covering its + include directories and flags (its features' defines among them) and the + digests of the packages whose modules it imports: when `mcpp test` or + another selection asks a package for other features, that instance is + compiled beside the first, and alternating the commands compiles neither + again (2026.11.19.1+; before that release one directory held one instance, + and each switch recompiled it, mcpp#953). - **The root's declarations.** Each selected member declares as the root did when it was planned alone: its `path` or `git` override of a dependency wins over another package's declaration, `linkage` on its dependency edges is @@ -736,8 +743,9 @@ myproject/ ├── mcpp.lock # one lock for the workspace └── target/// # named by the configuration's lineage (§5.5) ├── build.ninja, compile_commands.json # one graph and one database per configuration - ├── obj// # intermediate objects of every package - ├── gcm.cache// # each package's BMIs (pcm.cache with clang) + ├── obj/// # intermediate objects of each package instance + ├── gcm.cache/// # each instance's BMIs (pcm.cache with clang) + ├── instances.tsv # when a build last used each instance ├── modmap/ # the module maps of the units that import them └── bin/ ├── server/ # a member's products: bin// diff --git a/docs/09-commands-by-scenario.md b/docs/09-commands-by-scenario.md index 90908d3d4..b58bb4420 100644 --- a/docs/09-commands-by-scenario.md +++ b/docs/09-commands-by-scenario.md @@ -42,8 +42,13 @@ it uses; `--stale` also removes those no kept configuration names (2026.10.31.3+). A configuration's new fingerprint takes over the directory of its lineage instead of opening another (docs/04 §2.9), and after a build or a test run, at most once a day, `--stale` runs by itself with the window `[build] -keep_days` (config.toml; default 3 days). `mcpp clean --report` gives the size of each configuration's -objects, BMIs and products and of each `.build-mcpp` area, and removes nothing: +keep_days` (config.toml; default 3 days). Within a configuration that is +kept, `--stale` removes each package instance (`obj///` and +its BMIs) that no build has used within that window; the instances of the +plan that wrote the configuration's graph are kept whatever the window +(2026.11.19.1+). `mcpp clean --report` gives the size of each configuration's +objects, BMIs and products, of each package instance with the days since a +build used it, and of each `.build-mcpp` area, and removes nothing: ``` $ mcpp clean --stale --dry-run diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index b4d67853b..fb334ca86 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -672,11 +672,18 @@ mcpp 2026.9.26.2,#703)。** 写在 `[workspace.dependencies]` 中时实现**必须**给出警告,`--strict` 下**必须**拒绝。 16. **虚拟根旁的 `build.mcpp`。** 不带 `[package]` 的工作空间根旁的 `build.mcpp` 不属于任何包, 不运行;实现**必须**给出警告,`--strict` 下**必须**拒绝。 +17. **包实例的目录。** 一个配置的构建目录中,根以外的每个包按实例存放:对象在 + `obj/<包>/<摘要>/`,BMI 在 BMI 目录的 `<包>/<摘要>/`。摘要**必须**覆盖决定该包编译命令的因素 + (它的 include 目录与编译参数,含其 feature 的宏)以及它所导入模块的各包的摘要,**禁止**含版本 + 与编译器的构建号。因此不同命令或选择对同一个包请求的不同 feature 各自占有一个实例,交替执行 + 时不重新编译。实现以 `instances.tsv` 记录每个实例最近被构建使用的时间;`mcpp clean --stale` + 删除窗口内未被使用的实例,写出该配置构建图的规划所用的实例**必须**保留。 **状态:已实现(第 1 至 7 条 mcpp 2026.9.25.1;第 8 至 10 条 mcpp 2026.9.27.1,mcpp#713、 #714、#710;第 10 条的 `dialect_cxxflags` 为 mcpp 2026.9.28.1,#717;第 1 条的根包、第 10 条的 `[profile.]` 与第 11 条为 mcpp 2026.10.5.2,#771;第 7、8、10 条的工作空间层与第 12、13 条为 -mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 mcpp 2026.10.11.1,#797)。** +mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 mcpp 2026.10.11.1,#797; +第 17 条为 mcpp 2026.11.19.1,#953)。** ## 10. 依赖的程序 @@ -859,4 +866,4 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m | 1.19 | 2026-10-28 | mcpp 2026.10.27.1(mcpp#847):§2 增加平台平面(`[platform]`、`[host]`),平台键可出现在工程与机器默认中;§4.2 增加 `[target.]` 的平台键与 `[host] platform`(SPEC-013)。 | | 1.20 | 2026-11-17 | mcpp 2026.11.17.1(设计 2026-11-16 §4):新增 §15 运行时文件与发布文件:`deploy` 的条目语法(字符串、通配符、`name`)、固定目录规则与过期副本的移除;`license_files` 与其缺省值。 | | 1.21 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):新增 §4.8 包的依赖表由构建它的那一行读取;两侧各自发出请求,锁记录两侧。 | -| 1.22 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#931):新增 §8.1 链接的输入:链接词点名的文件(选项的值、`,` 分隔的段、MSVC `/` 选项的值、动作声明的输出)与链接器自己的报告;§10.2 特性的 `tools` 到达链接本包库的每个构建程序(mcpp#955)。 | +| 1.22 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#931):新增 §8.1 链接的输入:链接词点名的文件(选项的值、`,` 分隔的段、MSVC `/` 选项的值、动作声明的输出)与链接器自己的报告;§10.2 特性的 `tools` 到达链接本包库的每个构建程序(mcpp#955);§9 新增第 17 条包实例的目录(mcpp#953)。 | diff --git a/docs/zh/07-workspace.md b/docs/zh/07-workspace.md index e91732844..f2a294455 100644 --- a/docs/zh/07-workspace.md +++ b/docs/zh/07-workspace.md @@ -542,7 +542,10 @@ mcpp test --workspace --workspace-timeout 1800 # whole fan-out (default 0 = no 修改它们会重新编译该成员以及导入它的单元,构建目录保持不变。 - **Feature。** `--features f` 在每个声明了 `f` 的被选成员中启用它;没有被选成员声明它时, 命令被拒绝。同一配置中被多个成员使用的包只编译一次,feature 取它们请求的并集;被两个 - 配置的成员使用的包在每个配置中各编译一次。 + 配置的成员使用的包在每个配置中各编译一次。每个包实例有自己的目录 `obj/<包>/<摘要>/`,摘要覆盖 + 它的 include 目录与编译参数(含 feature 的宏)以及它所导入模块的各包的摘要:`mcpp test` 或另一 + 种选择为某个包请求其他 feature 时,该实例在原实例旁编译,交替执行这些命令时两者都不再重新编译 + (2026.11.19.1+;此前一个目录只容纳一个实例,每次切换都重新编译,mcpp#953)。 - **根的声明。** 每个被选成员都像单独规划时的根那样声明依赖:它对某个依赖的 `path` 或 `git` 覆盖胜过其他包的声明,它依赖边上的 `linkage` 被采纳,它的注册表依赖参与索引刷新 的判断(2026.9.30.2+)。两个被选成员对同一依赖的 checkout(种类或引用)或链接形态意见不一 @@ -618,8 +621,9 @@ myproject/ ├── mcpp.lock # 整个工作空间一份锁文件 └── target/// # 以配置的谱系命名(§5.5) ├── build.ninja, compile_commands.json # 每个配置一张图、一份数据库 - ├── obj// # 各包的中间产物 - ├── gcm.cache// # 各包的 BMI(clang 为 pcm.cache) + ├── obj/// # 各包实例的中间产物 + ├── gcm.cache/// # 各实例的 BMI(clang 为 pcm.cache) + ├── instances.tsv # 构建最近一次使用各实例的时间 ├── modmap/ # 导入它们的编译单元所读的模块映射 └── bin/ ├── server/ # 成员的产物:bin/<包名>/ diff --git a/docs/zh/09-commands-by-scenario.md b/docs/zh/09-commands-by-scenario.md index b8bf4850a..27f448183 100644 --- a/docs/zh/09-commands-by-scenario.md +++ b/docs/zh/09-commands-by-scenario.md @@ -35,8 +35,10 @@ 写明它使用的生成源码目录(`.build-mcpp/out/`)与构建程序组 (`.build-mcpp/host/`);`--stale` 同时删除保留的配置都未写明的这两类目录 (2026.10.31.3+)。一个配置的新指纹接管其谱系的目录,不另开目录(docs/04 §2.9);构建或测试成功后,每天至多 -一次,`--stale` 以 `[build] keep_days`(config.toml;默认 3 天)为窗口自动执行。`mcpp clean --report` 给出每个配置的对象、BMI 与产物以及 -`.build-mcpp` 各区的大小,不删除任何东西: +一次,`--stale` 以 `[build] keep_days`(config.toml;默认 3 天)为窗口自动执行。在保留的配置中,`--stale` 删除该窗口内没有构建 +使用过的包实例(`obj/<包>/<摘要>/` 及其 BMI);写出该配置构建图的那次规划所用的实例无论窗口都保留(2026.11.19.1+)。 +`mcpp clean --report` 给出每个配置的对象、BMI 与产物、每个包实例的大小及距上次使用的天数,以及 `.build-mcpp` 各区的大小, +不删除任何东西: ``` $ mcpp clean --stale --dry-run diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 184699023..a0ff70c9b 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -2745,6 +2745,8 @@ std::optional run_ninja_fast(const std::string& ninjaProgram, // report attached to one path only appears or not depending on whether // build.ninja was up to date. mcpp::build::advice::report_and_clear(outputDir); + // The build used the instances of the plan that wrote this graph (§11.1). + mcpp::build::touch_last_instances(outputDir); if (elapsedOut) { *elapsedOut = std::chrono::duration_cast( @@ -8105,6 +8107,23 @@ export int clean_project(bool wipe_bmi) { // `target/` before that: none of them is read by a workspace build. A // member's `target/.build-mcpp/`, where its build program's outputs are, is // kept. +// The package instances of one configuration directory, each with its +// objects, BMIs and the days since a plan last used it (design 2026-11-19 +// §11.1), indented below the configuration's line. +void report_instances(const std::filesystem::path& fpDir) { + using mcpp::bmi_cache::dir_size, mcpp::bmi_cache::human_bytes; + const auto now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + for (auto const& [dir, use] : mcpp::build::read_instances(fpDir)) { + const auto obj = dir_size(fpDir / "obj" / std::filesystem::path(dir)); + std::uintmax_t bmi = 0; + for (auto const* area : {"gcm.cache", "pcm.cache", "ifc.cache"}) + bmi += dir_size(fpDir / area / std::filesystem::path(dir)); + std::println(" {:<42} {:>10} {:>10} {:>10} used {}d ago{}", "obj/" + dir, human_bytes(obj), "", + human_bytes(bmi), (now - use.when) / 86400, use.last ? ", by the last plan" : ""); + } +} + // `mcpp clean --report` (mcpp#874): where target/ holds its bytes. Per // configuration directory, its objects, their split debug information, its BMIs // and its products; then the @@ -8149,6 +8168,7 @@ export int clean_report() { std::println("{:<44} {:>10} {:>10} {:>10} {:>10} {:>10}", triple + "/" + f->path().filename().string(), human_bytes(obj), human_bytes(debug), human_bytes(bmi), human_bytes(bin), human_bytes(all)); + report_instances(f->path()); } } const auto bm = target / ".build-mcpp"; @@ -8266,6 +8286,54 @@ export void sweep_after_build() { } } +// THE PACKAGE INSTANCES OF ONE CONFIGURATION (design 2026-11-19 §11.1): an +// instance a plan has not used within the window -- its objects below +// `obj///` and its BMIs below the BMI directory -- is +// removed, and its line leaves `instances.tsv`. The window is the one the +// configurations are kept by; the last plan's instances are always kept. +struct InstanceSweep { + std::uintmax_t bytes = 0; + std::size_t removed = 0, failed = 0; +}; +InstanceSweep sweep_instances(const std::filesystem::path& fpDir, const std::string& shown, bool dryRun, + std::int64_t keepWithinSecs, + const std::function& line) { + namespace fs = std::filesystem; + InstanceSweep out; + auto instances = mcpp::build::read_instances(fpDir); + const auto now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + // The last plan's instances are kept as the recorded configuration is, + // whatever the window. + bool changed = false; + for (auto it = instances.begin(); it != instances.end();) { + if (it->second.last || now - it->second.when < keepWithinSecs) { ++it; continue; } + std::uintmax_t size = 0; + std::error_code rec; + for (auto const* area : {"obj", "gcm.cache", "pcm.cache", "ifc.cache"}) { + const auto dir = fpDir / area / fs::path(it->first); + size += mcpp::bmi_cache::dir_size(dir); + if (!dryRun) fs::remove_all(dir, rec); + } + if (rec) { + std::println(libc::stderr(), "error: cannot remove {}/obj/{}: {}", shown, it->first, rec.message()); + ++out.failed; + ++it; + continue; + } + line(std::format("{} {}/obj/{} ({}; no build used this instance within the window)", + dryRun ? "would remove" : "removed", shown, it->first, + mcpp::bmi_cache::human_bytes(size))); + out.bytes += size; + ++out.removed; + if (dryRun) { ++it; continue; } + it = instances.erase(it); + changed = true; + } + if (changed) mcpp::build::write_instances(fpDir, instances); + return out; +} + export int clean_stale(bool dryRun, std::int64_t keepWithinSecs, bool quiet) { // Quiet: the sweep after a build (design 2026-10-31 §8.3) says one line // when it removed something, and nothing about what it kept. @@ -8330,7 +8398,16 @@ export int clean_stale(bool dryRun, std::int64_t keepWithinSecs, bool quiet) { if (!fpIt->is_directory(fec) || fec) continue; const fs::path dir = fpIt->path(); const std::string fp = dir.filename().string(); - if (recorded->second.contains(fp)) continue; + if (recorded->second.contains(fp)) { + // A current configuration keeps its directory; its package + // instances no plan has used within the window go (§11.1). + const auto swept = sweep_instances(dir, std::format("target/{}/{}", triple, fp), + dryRun, keepWithinSecs, line); + bytes += swept.bytes; + removed += swept.removed; + failed += swept.failed; + continue; + } const std::string shown = std::format("target/{}/{}", triple, fp); // build.ninja is rewritten by every build; the directory's own // mtime only moves when an entry is added or removed. diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 8c09b50e9..588a4c736 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -5582,6 +5582,11 @@ void write_module_maps(const BuildPlan& plan) { write_if_changed(plan.outputDir / scope.argsFile, module_map_arguments(scope.arguments, msvc)); } + // The plan that writes the maps names the instances this configuration + // uses now (design 2026-11-19 §11.1); the others age in `instances.tsv`. + std::vector used; + for (auto const& [pkg, dir] : plan.instanceDirs) used.push_back(dir); + record_instances(plan.outputDir, used); } void report_failed_drive(const BuildError& error) { diff --git a/src/build/plan.cppm b/src/build/plan.cppm index 7c25c0a31..93c963d7f 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -617,6 +617,14 @@ struct BuildPlan { // the root package's, and then the BMI directory is flat and no command // names a map. std::map> moduleScopes; + // The package instances this plan compiles, as `/` below + // `obj/` and the BMI directory (design 2026-11-19 §11.1). Recorded with the + // time a plan last used each (`instances.tsv`), which `mcpp clean --stale` + // reads to remove an instance no build has used within its window. + // Keyed by the package's qualified name; the value is `/`. + // A GCC BMI records its imports at their instance paths, so the build + // cache keys a package's entry by its instance too (`bmiPathConvention`). + std::map instanceDirs; std::filesystem::path stdCompatObjectPath; // absolute path to prebuilt std.compat.o std::filesystem::path scanDepsPath; // clang-scan-deps binary (Clang only) // THE TARGET'S PLATFORM (SPEC-013): the merged statement, its flags for @@ -900,6 +908,23 @@ TargetFileAnswer resolve_target_file(const BuildPlan& plan, std::string_view nam std::filesystem::path objects_list_path(const BuildPlan& plan, std::string_view output, std::string_view suffix); +// The record of a configuration's package instances (design 2026-11-19 §11.1): +// `/instances.tsv`, one `/\t\t` +// line per instance: the time a build last used it, and whether the plan that +// wrote the build graph uses it. `record_instances` marks a plan's set as the +// last one and stamps it; `touch_last_instances` restamps that set for a build +// that runs the graph without planning (the fast path), writing only when a +// stamp is an hour old. `mcpp clean` keeps the last set whatever the window. +struct InstanceUse { + std::int64_t when = 0; + bool last = false; +}; +std::map read_instances(const std::filesystem::path& outputDir); +void write_instances(const std::filesystem::path& outputDir, + const std::map& instances); +void record_instances(const std::filesystem::path& outputDir, const std::vector& used); +void touch_last_instances(const std::filesystem::path& outputDir); + // Replaces `${mcpp.objects:}` and `${mcpp.objects_rsp:}` in `s` // with `objects_list_path` of the unit `resolve_target_file` names, records the // unit in `units` once, and a name no unit has, or several members define, in @@ -4621,6 +4646,100 @@ make_plan(const mcpp::manifest::Manifest& manifest, && !target.is_freestanding(); } + // A PACKAGE INSTANCE'S OBJECTS LIE IN A DIRECTORY ITS CONFIGURATION NAMES + // (design 2026-11-19 §11.1, mcpp#953 D5): `obj///`, and its + // BMIs in `///`. A separate directory level and + // not `@`: ninja shell-quotes an output holding a + // character outside `A-Za-z0-9_+,-./`, and a rule that writes `"$out.d"` + // then names a file with quotes in it (measured). A build directory is named by + // the configuration's lineage, which holds no package's features, and one + // command unifies a shared package's features over its selection + // (docs/07): `mcpp test` adds what the dev-dependencies enable, `-p X` and + // `-p Y` request different unions. One object path then stood for two + // command lines, and each switch compiled the package again (measured: + // 43 such edges between two members of a 31-member workspace; the + // fixture of e2e 953 recompiled its shared package at every switch). The + // digest covers what decides a unit's command line -- the package's own + // include directories and flags, its features' defines among them -- and + // the digests of the packages whose modules it imports, whose BMI paths + // its command names. It holds no version and no compiler build, so an + // upgrade keeps the paths. The root's units keep their place: one plan + // has one root, and its command line is the configuration's. + const auto rootPackageName = manifest.package.virtualRoot + ? std::string{} : qualified_package_name(manifest); + std::map instanceDirs; // package -> `/` + { + std::map own; // package -> its units' flags + std::map> importsOf; // package -> providers + // A directory is spelled relative to the store or the project that + // holds it: the digest names a directory a cached BMI records, so it + // is the same on every machine that shares the entry. + auto portable = [&](const std::filesystem::path& d) { + for (auto const& store : storeRoots) + if (auto rel = d.lexically_relative(store); !rel.empty() && !rel.generic_string().starts_with("..")) + return "/" + rel.generic_string(); + if (auto rel = d.lexically_relative(projectRoot); !rel.empty() && !rel.generic_string().starts_with("..")) + return "/" + rel.generic_string(); + return d.generic_string(); + }; + auto flags_text = [&](const CompileUnit& cu) { + std::string t; + for (auto const& d : cu.localIncludeDirs) t += "I" + portable(d) + "\n"; + for (auto const& d : cu.localIncludeDirsAfter) t += "A" + portable(d) + "\n"; + for (auto const& f : cu.packageCflags) t += "C" + f + "\n"; + for (auto const& f : cu.packageCxxflags) t += "X" + f + "\n"; + for (auto const& f : cu.packageAsmflags) t += "S" + f + "\n"; + return t; + }; + auto providers_of = [&](const CompileUnit& cu) { + std::set out; + for (auto const& imp : cu.imports) + if (auto provider = mcpp::modgraph::resolve_provider(graph, cu.packageName, imp)) + out.insert(graph.units[*provider].packageName); + out.erase(cu.packageName); + return out; + }; + for (auto const& cu : plan.compileUnits) { + if (cu.packageName.empty() || cu.packageName == rootPackageName || cu.testUnit) continue; + own[cu.packageName] += cu.object.generic_string() + "\n" + flags_text(cu); + importsOf[cu.packageName].merge(providers_of(cu)); + } + std::map digest; + std::set visiting; + std::function digest_of = [&](const std::string& pkg) { + if (auto it = digest.find(pkg); it != digest.end()) return it->second; + if (!own.contains(pkg) || !visiting.insert(pkg).second) return std::string{}; + std::string text = own[pkg]; + for (auto const& dep : importsOf[pkg]) text += "D" + dep + "=" + digest_of(dep) + "\n"; + visiting.erase(pkg); + return digest[pkg] = mcpp::toolchain::hash_string(text).substr(0, 8); + }; + for (auto const& [pkg, text] : own) + instanceDirs[pkg] = sanitize(pkg) + "/" + digest_of(pkg); + plan.instanceDirs = instanceDirs; + } + { + std::map moved; + auto instance_object = [&](const CompileUnit& cu) -> std::optional { + auto it = instanceDirs.find(cu.packageName); + if (it == instanceDirs.end()) return std::nullopt; + auto part = cu.object.begin(); + if (part == cu.object.end() || *part != "obj" || ++part == cu.object.end() + || *part != std::filesystem::path(sanitize(cu.packageName))) + return std::nullopt; + return std::filesystem::path("obj") / it->second / cu.object.lexically_relative( + std::filesystem::path("obj") / sanitize(cu.packageName)); + }; + for (auto& cu : plan.compileUnits) + if (auto to = instance_object(cu)) { + moved.emplace(cu.object, *to); + cu.object = *to; + } + for (auto& lu : plan.linkUnits) + for (auto& o : lu.objects) + if (auto it = moved.find(o); it != moved.end()) o = it->second; + } + // EVERY PACKAGE'S BMIS LIE BELOW ITS OWN DIRECTORY, EXCEPT THE ROOT'S (pack // drive and selection design 2026-10-01, B1), as object files have since // mcpp#233. A module name identifies one module within one program @@ -4653,8 +4772,9 @@ make_plan(const mcpp::manifest::Manifest& manifest, return out; }; auto bmi_file = [&](std::string_view provider, std::string_view name) { - return provider == rootPackage ? basename(name) - : std::string(provider) + "/" + basename(name); + if (provider == rootPackage) return basename(name); + auto it = instanceDirs.find(std::string(provider)); + return (it == instanceDirs.end() ? std::string(provider) : it->second) + "/" + basename(name); }; auto bmi_path = [&](std::string_view provider, std::string_view name) { return std::string(traits.bmiDir) + "/" + bmi_file(provider, name); @@ -4846,6 +4966,60 @@ bool substitute_objects_tokens(const BuildPlan& plan, std::string& s, std::strin return used; } +std::map read_instances(const std::filesystem::path& outputDir) { + std::map out; + std::ifstream in(outputDir / "instances.tsv", std::ios::binary); + for (std::string line; std::getline(in, line);) { + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + const auto tab2 = line.find('\t', tab + 1); + InstanceUse use; + std::from_chars(line.data() + tab + 1, + line.data() + (tab2 == std::string::npos ? line.size() : tab2), use.when); + use.last = tab2 != std::string::npos && line.compare(tab2 + 1, std::string::npos, "1") == 0; + out[line.substr(0, tab)] = use; + } + return out; +} + +void write_instances(const std::filesystem::path& outputDir, + const std::map& instances) { + std::string text; + for (auto const& [dir, use] : instances) + text += std::format("{}\t{}\t{}\n", dir, use.when, use.last ? 1 : 0); + std::error_code ec; + std::filesystem::create_directories(outputDir, ec); + const auto file = outputDir / "instances.tsv"; + auto tmp = file; + tmp += std::format(".{:x}.tmp", std::random_device{}()); + { std::ofstream(tmp, std::ios::binary | std::ios::trunc) << text; } + std::filesystem::rename(tmp, file, ec); + if (ec) std::filesystem::remove(tmp, ec); +} + +namespace { +std::int64_t now_seconds() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); +} +} // namespace + +void record_instances(const std::filesystem::path& outputDir, const std::vector& used) { + auto instances = read_instances(outputDir); + for (auto& [dir, use] : instances) use.last = false; + for (auto const& dir : used) instances[dir] = {now_seconds(), true}; + if (!instances.empty()) write_instances(outputDir, instances); +} + +void touch_last_instances(const std::filesystem::path& outputDir) { + auto instances = read_instances(outputDir); + const auto now = now_seconds(); + bool changed = false; + for (auto& [dir, use] : instances) + if (use.last && now - use.when >= 3600) { use.when = now; changed = true; } + if (changed) write_instances(outputDir, instances); +} + std::filesystem::path objects_list_path(const BuildPlan& plan, std::string_view output, std::string_view suffix) { auto p = plan.outputDir / "objects" / std::filesystem::path(std::string(output)); diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index 75d64984c..ec7bb69f3 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -1830,15 +1830,19 @@ static std::expected step13_dependency_cache(PrepareState& st state.storeRoots.empty() ? std::filesystem::path{} : state.storeRoots.front()); pa.sources = pkgSources[idx]; + const auto qname = mcpp::build::qualified_package_name(state.packages[idx].manifest); // A GCC BMI records its imports' paths as the module map spells - // them, and the map names the BMI directory as its repository - // (`plan.cppm`, the module scopes). See `bmiPathConvention`. - if (pkgProvidesModule[idx] && state.tc->compiler == mcpp::toolchain::CompilerId::GCC) + // them: the BMI directory and the package's instance, whose digest + // folds in its imports' (`plan.cppm`). See `bmiPathConvention`. + if (pkgProvidesModule[idx] && state.tc->compiler == mcpp::toolchain::CompilerId::GCC) { + auto inst = ctx.plan.instanceDirs.find(qname); pa.bmiPathConvention = "$root=bmi-dir"; + if (inst != ctx.plan.instanceDirs.end()) + pa.bmiPathConvention += ";instance=" + inst->second; + } // What `scope_lto` appended to this package's units is part of // their command, so it is part of the entry's identity. - if (state.ltoWithheld.contains( - mcpp::build::qualified_package_name(state.packages[idx].manifest))) { + if (state.ltoWithheld.contains(qname)) { pa.cflags.push_back(state.ltoOffWord); pa.cxxflags.push_back(state.ltoOffWord); } diff --git a/tests/e2e/deps/path.cpp b/tests/e2e/deps/path.cpp index 5399aa6c6..b9ac42aff 100644 --- a/tests/e2e/deps/path.cpp +++ b/tests/e2e/deps/path.cpp @@ -51,7 +51,8 @@ E2E path{"path", [] { for (const auto& f : s::find_files(b.at("target"), R"(build\.ninja)")) if (!f.generic_string().contains("/.build-mcpp/")) ninja += s::read_file(f); expect(s::has_line(ninja, "mylibA.*greet.cppm")) << "ninja missing dep package source"; - expect(s::has_line(ninja, "gcm.cache/[^ /]*mylibA/mylibA.greet.gcm")) << "ninja missing dep BMI"; + // Below the package's instance directory (design 2026-11-19 §11.1). + expect(s::has_line(ninja, "gcm.cache/[^ /]*mylibA/[0-9a-f]{8}/mylibA.greet.gcm")) << "ninja missing dep BMI"; // A path whose manifest names another package. const auto wrong = s::case_dir() / "elsewhere/wrongname"; fs::create_directories(wrong.parent_path()); diff --git a/tests/e2e/test/workspace.cpp b/tests/e2e/test/workspace.cpp index 2b83150cc..51e57db6e 100644 --- a/tests/e2e/test/workspace.cpp +++ b/tests/e2e/test/workspace.cpp @@ -150,7 +150,7 @@ kind = "lib" }; auto core_edges = [&](const fs::path& dir) { return std::ranges::count_if(s::ninja_log_outputs(dir), [](const std::string& o) { - return std::regex_search(o, std::regex(R"((^|/)obj/core/src/core\.m\.o$)")); + return std::regex_search(o, std::regex(R"((^|/)obj/core/[0-9a-f]{8}/src/core\.m\.o$)")); }); }; auto reset = [&] { for (auto d : {"target", "core/target", "a/target", "b/target"}) ws.remove(d); }; diff --git a/tests/e2e/workspace/graph.cpp b/tests/e2e/workspace/graph.cpp index 5d7588b31..1fae1aec4 100644 --- a/tests/e2e/workspace/graph.cpp +++ b/tests/e2e/workspace/graph.cpp @@ -102,8 +102,17 @@ std::map, std::vector> units(st return out; } +// A word with the package instance it names (`obj///`, and the module map whose +// name hashes the instances it lists) replaced: what remains when only the instance differs. +std::string without_instance(const std::string& word) { + static const std::regex instance{R"(/[0-9a-f]{8}/)"}, map{R"((modmap/[^/ ]+)-[0-9a-f]{8}\.)"}; + return std::regex_replace(std::regex_replace(word, instance, "//"), map, "$1-."); +} + // 872's comparison: the units both databases hold, `member`'s among them, have the same arguments, -// except for words that all contain `allowed`. What differs, or "" when nothing does. +// except for words that all contain `allowed`. A package's instance follows its features (mcpp#953), +// so where a difference is allowed, words that differ only by the instance are part of it. What +// differs, or "" when nothing does. std::string differences(std::string_view a, std::string_view b, std::string_view member, std::string_view allowed = "") { const auto ua = units(a), ub = units(b); bool any = false; @@ -115,7 +124,13 @@ std::string differences(std::string_view a, std::string_view b, std::string_view if (args == other->second) continue; std::set sa(args.begin(), args.end()), sb(other->second.begin(), other->second.end()), diff; std::ranges::set_symmetric_difference(sa, sb, std::inserter(diff, diff.end())); - if (!allowed.empty() && !diff.empty() && std::ranges::all_of(diff, [&](const std::string& t) { return t.contains(allowed); })) continue; + if (!allowed.empty() && !diff.empty()) { + std::set na, nb, rest; + for (const auto& t : sa) na.insert(without_instance(t)); + for (const auto& t : sb) nb.insert(without_instance(t)); + std::ranges::set_symmetric_difference(na, nb, std::inserter(rest, rest.end())); + if (std::ranges::all_of(rest, [&](const std::string& t) { return t.contains(allowed); })) continue; + } out += k.second + " (" + k.first + ") differs:"; for (const auto& t : diff) out += " " + t; out += "\n"; @@ -335,7 +350,7 @@ export int consumer_flag_v() { in >> v >> x >> f; return std::tuple{v, x, f}; }; - const std::regex lib_object{R"((^|/)lib/src/lib\.m\.o$)"}; + const std::regex lib_object{R"((^|/)lib/[0-9a-f]{8}/src/lib\.m\.o$)"}; auto r = ws.mcpp({"build", "--workspace"}); expect(fatal(e2e::succeeded(r))) << "A: the workspace did not build: " << s::text(r); const auto dirs = graph_dirs(ws.dir()); diff --git a/tests/e2e/workspace/inheritance.cpp b/tests/e2e/workspace/inheritance.cpp index 3b3b8ceaf..aa7648927 100644 --- a/tests/e2e/workspace/inheritance.cpp +++ b/tests/e2e/workspace/inheritance.cpp @@ -600,6 +600,50 @@ main = "src/main.cpp" expect(fatal(e2e::succeeded(r))) << s::text(r); expect(s::lacks(s::text(r), "shared.cppm") && s::lacks(s::text(r), "mid.cppm")) << "B: back at the first member a package was compiled again: " << s::text(r); }; + "953 a package compiled under two feature sets keeps both instances"_test[s::gcc][s::elf] = [] { + // mcpp#953 D5, design 2026-11-19 §11.1. `mcpp test` enables a feature of a shared package + // that `mcpp build` does not. Each instance has its own directory, `obj///`, + // so alternating the two commands compiles each instance once (A). `clean --report` lists + // the instances (B); `clean --stale --older-than 0` removes the ones the last plan did not + // use and keeps the others (C). + const auto tmp = s::case_dir(); + s::project ws(tmp / "ws", { + {"mcpp.toml", "[workspace]\nmembers = [\"util\", \"core\", \"app\"]\n"}, + {"util/mcpp.toml", "[package]\nname = \"util\"\nversion = \"0.1.0\"\n\n[features]\nextra = { defines = [\"UTIL_EXTRA=1\"] }\n\n[targets.util]\nkind = \"lib\"\n"}, + {"util/src/util.cppm", "export module ws953.util;\nexport int util_value() {\n#ifdef UTIL_EXTRA\n return 2;\n#else\n return 1;\n#endif\n}\n"}, + {"core/mcpp.toml", "[package]\nname = \"core\"\nversion = \"0.1.0\"\n\n[dependencies]\nutil = { path = \"../util\" }\n\n[targets.core]\nkind = \"lib\"\n"}, + {"core/src/core.cppm", "export module ws953.core;\nimport ws953.util;\nexport int core_value() { return util_value() + 10; }\n"}, + {"app/mcpp.toml", "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\ncore = { path = \"../core\" }\n\n" + "[dev-dependencies]\nutil = { path = \"../util\", features = [\"extra\"] }\n\n[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"app/src/main.cpp", "import ws953.core;\nint main() { return core_value() == 11 ? 0 : 1; }\n"}, + {"app/tests/t.cpp", "import ws953.core;\nint main() { return core_value() == 12 ? 0 : 1; }\n"}, + }); + auto stamps = [&] { + std::vector> out; + for (auto const& f : s::find_files(ws.at("target"), R"(util\.m\.o)")) + out.emplace_back(f.generic_string(), fs::last_write_time(f).time_since_epoch().count()); + std::ranges::sort(out); + return out; + }; + expect(fatal(e2e::succeeded(ws.mcpp({"build", "-p", "app"})))) << "A: the first build failed"; + auto r = ws.mcpp({"test", "-p", "app"}); + expect(fatal(e2e::succeeded(r))) << "A: the first test failed (the feature did not reach the test build): " << s::text(r); + const auto before = stamps(); + expect(fatal(before.size() == 2u)) << "A: the shared package does not have two instances: " << before.size(); + for (int round = 0; round < 2; ++round) { + expect(fatal(e2e::succeeded(ws.mcpp({"build", "-p", "app"})))) << "A: a build failed"; + expect(fatal(e2e::succeeded(ws.mcpp({"test", "-p", "app"})))) << "A: a test failed"; + } + expect(stamps() == before) << "A: alternating build and test compiled an instance again"; + r = ws.mcpp({"clean", "--report"}); + expect(fatal(e2e::succeeded(r))) << "B: clean --report failed: " << s::text(r); + expect(s::count_lines(s::text(r), "obj/util/") == 2u) << "B: the report does not list both instances: " << s::text(r); + r = ws.mcpp({"clean", "--stale", "--older-than", "0"}); + expect(fatal(e2e::succeeded(r))) << "C: clean --stale failed: " << s::text(r); + expect(stamps().size() == 1u) << "C: the instance the last plan did not use was kept, or the used one removed: " << s::text(r); + expect(fatal(e2e::succeeded(ws.mcpp({"test", "-p", "app"})))) << "C: the build after the sweep failed"; + }; + "954 the workspace's abi table is the root's statement"_test[s::gcc][s::elf] = [] { // mcpp#954. A workspace whose root is only `[workspace]` states the build's abi once, in // `[workspace.target..abi]`; its members inherit the row and no member is told diff --git a/tests/e2e/workspace/members.cpp b/tests/e2e/workspace/members.cpp index 23c36eca6..654fcef90 100644 --- a/tests/e2e/workspace/members.cpp +++ b/tests/e2e/workspace/members.cpp @@ -271,7 +271,7 @@ E2E members{"members", [] { ws.write(m + "/src/" + m + ".cppm", std::format("export module sel852_{0};\nexport int {0}_value() {{ return 1; }}\n", m)); } auto object_edges = [](const fs::path& dir, const std::string& m) { - const std::regex re{"(^|/)obj/" + m + "/src/" + m + "\\.m\\.o$"}; + const std::regex re{"(^|/)obj/" + m + "/[0-9a-f]{8}/src/" + m + "\\.m\\.o$"}; return std::ranges::count_if(s::ninja_log_outputs(dir), [&](const std::string& o) { return std::regex_search(o, re); }); }; auto r = ws.mcpp({"build", "-p", "a", "-p", "nosuch"}); diff --git a/tests/e2e/workspace/statements.cpp b/tests/e2e/workspace/statements.cpp index e4ccd6eb3..51c49e7cd 100644 --- a/tests/e2e/workspace/statements.cpp +++ b/tests/e2e/workspace/statements.cpp @@ -71,15 +71,20 @@ E2E statements{"statements", [] { const auto tmp = s::case_dir(); auto profile = [](std::string_view mark) { return std::format("[profile.release]\ncxxflags = [\"-DMARK_PROFILE_{}\"]\n", mark); }; // The variables of the graph and the build statement of `object` with its indented lines. + // A member's objects lie below its instance directory, `obj///` + // (design 2026-11-19 §11.1): `object` names the path without it. auto edge = [](const fs::path& root, std::string_view object) { const auto g = graphs(root); expect(fatal(!g.empty())) << "no build.ninja"; std::string vars, stmt; bool in = false; + const auto slash = object.find('/', object.find('/') + 1); + const std::regex named{"^build " + std::string(object.substr(0, slash + 1)) + "([0-9a-f]{8}/)?" + + std::string(object.substr(slash + 1))}; std::istringstream text(s::read_file(g.front())); for (std::string l; std::getline(text, l);) { if (std::regex_search(l, std::regex{"^(cxxflags|cflags) *="})) vars += l + "\n"; - if (l.starts_with("build " + std::string(object))) { in = true; stmt += l + "\n"; continue; } + if (std::regex_search(l, named)) { in = true; stmt += l + "\n"; continue; } if (in && l.starts_with(" ")) { stmt += l + "\n"; continue; } in = false; } diff --git a/tests/unit/test_object_address.cpp b/tests/unit/test_object_address.cpp index aab2cb265..b425f7a2a 100644 --- a/tests/unit/test_object_address.cpp +++ b/tests/unit/test_object_address.cpp @@ -30,6 +30,24 @@ using namespace mcpp::build; namespace { +// A dependency's objects lie below its instance directory, `obj// +// /` (design 2026-11-19 §11.1); the address inside the package is +// what these cases are about, so the digest is taken out, after asserting it +// is there. +std::string without_instance(const std::filesystem::path& object) { + std::vector parts; + for (auto const& p : object) parts.push_back(p.generic_string()); + const auto hex = [](const std::string& s) { + return s.size() == 8 && std::ranges::all_of(s, [](char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + }); + }; + if (parts.size() > 3 && parts[0] == "obj" && hex(parts[2])) parts.erase(parts.begin() + 2); + std::string out; + for (auto const& p : parts) out += (out.empty() ? "" : "/") + p; + return out; +} + struct Tmp { std::filesystem::path path; Tmp() { @@ -173,7 +191,8 @@ UTest dependency_cache_address_is_package_internal{"ObjectAddress/DependencyCach // The build path does partition by package — that is what makes the // cross-package census unnecessary. auto obj = built->object.generic_string(); - expect(eq(obj, "obj/compat_zlib/zlib-1.3.2/compress.o")) << obj; + expect(ne(without_instance(obj), obj)) << "no instance directory: " << obj; + expect(eq(without_instance(obj), "obj/compat_zlib/zlib-1.3.2/compress.o")) << obj; }}; // `path_is_under_any` decides both whether a package may be cached and where @@ -300,12 +319,12 @@ std::optional addressOfDeclaredDependencySource( for (auto& cu : plan->compileUnits) { if (cu.source == depRoot / "src" / "own.cpp") { // The dependency's own source keeps the mirrored address. - if (cu.object.generic_string() != "obj/dep/src/own.o") { + if (without_instance(cu.object) != "obj/dep/src/own.o") { if (err) *err = "dependency's own source moved: " + cu.object.generic_string(); return std::nullopt; } } - if (cu.source == helper) found = cu.object.generic_string(); + if (cu.source == helper) found = without_instance(cu.object); } if (!found && err) *err = "helper compile unit not found"; return found; @@ -361,7 +380,7 @@ UTest an_unowned_escaping_source_is_addressed_by_a_hash_of_its_directory{"Object std::string rootObj, outsideObj; for (auto& cu : plan->compileUnits) { if (cu.source == projectRoot / "src" / "gen.cpp") rootObj = cu.object.generic_string(); - if (cu.source == outside) outsideObj = cu.object.generic_string(); + if (cu.source == outside) outsideObj = without_instance(cu.object); } expect(eq(rootObj, "obj/app/src/gen.o")); expect(fatal(outsideObj.starts_with("obj/dep/__ext/"))) << outsideObj; @@ -455,5 +474,5 @@ UTest a_members_source_in_another_members_directory_is_immune_to_the_selection{" const auto beside = plan_with(true); expect(!alone.empty()); expect(eq(alone, beside)); - expect(ne(alone.generic_string().find("obj/app/__ws/lib/src/"), std::string::npos)) << alone; + expect(ne(without_instance(alone).find("obj/app/__ws/lib/src/"), std::string::npos)) << alone; }}; From 4e96031546505647bef5b2dd23bc8b9babae8dba Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 15:41:53 +0900 Subject: [PATCH 10/30] A build interrupted in another process space continues, and a start refused for its size says so (mcpp#952, mcpp#958) An observation of a compiler unit has four answers: empty, occupied, not yet decidable, and not observable from this process space (openkal 0.26, clause 7.18). An unobservable unit of a generation whose owner and helper locks are free -- locks of the file system, shared by every process space that shares the directory -- is settled as abandoned and reported once (compiler-run/unobservable). A refusal for occupied or undecidable units names their number, the state directory and the way out [compiler-run-occupied]. mcpp clean, --stale and --older-than retire a configuration's compiler runs before removing it. The owner's refusal of an admission names its condition, and every helper return other than the compiler's status or a cancellation names the step that ended it. Where the system gives nothing to declare a unit from, the build compiles without units, bounded by the generation's cap, and says so once (compiler-run/no-unit). mcpp builds against mcpplibs.os 0.4.5. A start the system refuses for the size of its arguments and environment reads "the request is larger than the system admits (N arguments, A bytes; M variables, E bytes) [spawn-too-large]". --- CHANGELOG.md | 20 +++ docs/04-mcpp-toml.md | 25 ++- docs/50-machine-output.md | 9 ++ docs/zh/04-mcpp-toml.md | 16 +- docs/zh/50-machine-output.md | 8 + mcpp.toml | 2 +- modules/invocation/mcpp.toml | 2 +- modules/manifest/mcpp.toml | 2 +- modules/toolchain-model/mcpp.toml | 2 +- src/build/execute.cppm | 26 +++ src/build/schedule/compiler_transaction.cppm | 93 ++++++++--- src/build/schedule/detach_codegen.cppm | 158 ++++++++++++++++--- src/run/run.cpp | 37 ++++- tests/e2e/build/basics.cpp | 47 ++++++ 14 files changed, 389 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30255338d..2c336b0b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,26 @@ ### Fixed +- A build interrupted in another process space -- a container job that reuses the working + directory, a rebuilt development container, an agent's sandbox session -- could never be + continued, `mcpp clean` included: `previous compiler unit is occupied or unknown` (mcpp#952). + An observation now has four answers: a unit is empty, occupied, not yet decidable, or not + observable from this process space (openkal 0.26, clause 7.18). An unobservable unit of a + generation whose owner and helper locks are free -- locks of the file system, shared by every + process space that shares the directory -- is settled as abandoned, its unpublished outputs + discarded, and the build reports `degraded` (`compiler-run/unobservable`) once. A refusal for + occupied or undecidable units names their number, the state directory and the way out + (`[compiler-run-occupied]`). `mcpp clean`, `--stale` and `--older-than` retire a + configuration's compiler runs before removing it. The owner's refusal of an admission names its + condition; every helper return other than the compiler's status or a cancellation names the + step that ended it. Where the system gives nothing to declare a unit from, the build compiles + without units and says so once (`compiler-run/no-unit`). +- A start the system refused for the size of its arguments and environment reported `Cannot + allocate memory`, from a bound of 512 elements in the kernel layer: 520 `[build] defines` + could not compile (mcpp#958). mcpp builds against mcpplibs.os 0.4.5 and openkal 0.26, which + keep no bound of their own, and the refusal reads `cannot start '': the request is + larger than the system admits (N arguments, A bytes; M variables, E bytes) + [spawn-too-large]`. - `mcpp publish` and `mcpp emit xpkg` wrote a descriptor that nothing read: a table of modules, version dependencies and run-time requirements with `manifest = "mcpp.toml"`. `mcpp xpkg parse` refused it for its missing `sources`, a consumer could not depend on diff --git a/docs/04-mcpp-toml.md b/docs/04-mcpp-toml.md index 5d3883ca1..f145ad41b 100644 --- a/docs/04-mcpp-toml.md +++ b/docs/04-mcpp-toml.md @@ -1109,8 +1109,29 @@ following build waits for their cleanup before starting another compiler; partial outputs are rebuilt. A missing object from a completed module is recreated by that build. An unchanged build still performs no compilation. If a forcibly ended build leaves a compiler unit whose completion cannot be -verified, the next build reports an occupied or unknown unit and retains its -outputs; it does not start a replacement compiler in that slot. +verified, the next build waits for it within a bound (2026.11.19.1+): + +- a unit that is empty is recovered, as before; +- a unit that is occupied, or not yet decidable, makes the build refuse with + the number of such units, the state directory and the way out + (`[compiler-run-occupied]`): when no compiler of the configuration is + running, `mcpp clean`, then build again; +- a unit that cannot be observed from this process space at all (the build + was interrupted in another container or sandbox session, or the system did + not keep the unit's identity) is settled as abandoned when the old build's + owner and helper locks are free, which holds across process spaces that + share the directory; its unpublished outputs are discarded and the build + reports `degraded` (`compiler-run/unobservable`) once. + +`mcpp clean`, `--stale` and `--older-than` retire a configuration's compiler +runs before removing its directory, and leave a configuration whose run a +build holds, naming it. An admission the build's owner refuses names its +condition (`outside-generation`, `record-unreadable`, `actor-outside`, +`actor-not-held`, `controller-ended`, `unit-unobservable`, `owner-stopping`), +and every helper that ends a compile for any reason but the compiler's status +or a cancellation says which step ended it. Where the system gives a program +nothing to declare a compiler unit from, the build compiles without declaring +units and reports `degraded` (`compiler-run/no-unit`) once. | value | | |---|---| diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index 779820e34..cca86097c 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -686,6 +686,15 @@ graph does not supply what the member asked for" from "the member did not build" on exactly this token, and that distinction decides whether a member counts against a compatibility figure. +**Two tokens are printed by a build that fails while it runs** (2026.11.19.1+), +in brackets, in the failure's own message; they are not refusals of the target +and do not appear as `reason`: + +| token | | +|---|---| +| `compiler-run-occupied` | an interrupted build of the configuration left compiler units that are occupied or not yet decidable; the message names their number, the state directory and the way out | +| `spawn-too-large` | the system refused to start a program for the size of its arguments and environment; the message names both counts and byte sizes | + **Exit 0 whenever the question was answered, including "refused".** "Would this build, and if not why" is answered successfully by "no, because the row's pin is a capability". A non-zero exit means the query itself could not run. diff --git a/docs/zh/04-mcpp-toml.md b/docs/zh/04-mcpp-toml.md index 43263d206..a7ccdf34f 100644 --- a/docs/zh/04-mcpp-toml.md +++ b/docs/zh/04-mcpp-toml.md @@ -1015,8 +1015,20 @@ bmi_schedule = "off" # auto (default, = off) | on | off 取消构建会结束本轮普通编译器和分离的编译器。紧接着的构建等待它们完成清理后 才启动新的编译器;未完成的输出会重新构建。已完成模块的 object 缺失时, 该次构建会重新生成它。输入和输出均不变的构建仍不进行编译。 -若被强制结束的构建留下了无法确认已经完成的编译单元,下一次构建报告该单元 -仍被占用或状态未知,并保留其输出;该名额不会启动替代编译器。 +若被强制结束的构建留下了无法确认已经完成的编译单元,下一次构建在一个界限内等待它(2026.11.19.1+): + +- 单元为空:照常恢复; +- 单元被占用或暂时无法判定:构建拒绝,写出这类单元的个数、状态目录与出路(`[compiler-run-occupied]`): + 该配置没有编译器在运行时,执行 `mcpp clean` 后重新构建; +- 单元从当前进程空间不可观察(构建在另一个容器或沙箱会话中被中断,或系统没有保留单元的标识):在旧构建的 + 所有者锁与辅助进程锁都空闲时按被放弃处理,这些锁在共享该目录的各进程空间中同样成立;未发布的输出被丢弃, + 构建报告一次 `degraded`(`compiler-run/unobservable`)。 + +`mcpp clean`、`--stale` 与 `--older-than` 在删除一个配置的目录前先退役它的编译运行;某个构建持有其运行的配置 +不被删除,消息写出它。构建的所有者拒绝准入时写出条件(`outside-generation`、`record-unreadable`、 +`actor-outside`、`actor-not-held`、`controller-ended`、`unit-unobservable`、`owner-stopping`);辅助进程因编译器 +状态与取消以外的任何原因结束一次编译时,写出结束它的步骤。系统不给程序任何可据以声明编译单元的东西时,构建不声明 +单元而编译,并报告一次 `degraded`(`compiler-run/no-unit`)。 | 取值 | | |---|---| diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index 9bfe64eeb..7335ec324 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -612,6 +612,14 @@ replaced}` —— `origin` 与构建的状态行使用的是同一句话 「这个图没有供给这个成员所要求的」与「这个成员没能构建」区分开,而这个区分 决定了一个成员算不算进兼容率。 +**运行中失败的构建打印两个记号**(2026.11.19.1+),用方括号写在失败消息本身里;它们不是对目标的拒绝, +不作为 `reason` 出现: + +| 记号 | | +|---|---| +| `compiler-run-occupied` | 该配置一次被中断的构建留下了被占用或暂时无法判定的编译单元;消息写出它们的个数、状态目录与出路 | +| `spawn-too-large` | 系统因参数与环境的大小拒绝启动一个程序;消息写出两者的个数与字节数 | + **只要问题被回答了就退出 0,包括答案是「拒绝」这种情况。** 「这次构建能不能 成立,不能的话为什么」被「不能,因为这一行的 pin 是一项能力陈述」完整地 回答了。非零退出的含义是这次查询本身没能运行起来。 diff --git a/mcpp.toml b/mcpp.toml index b280b4382..2145b065b 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -125,7 +125,7 @@ mcpplibs.mwrap.xz = "5.8.3.1" # The operating system is mcpplibs.os's (design 2026-10-26 §25): mcpp keeps no # layer of its own. -mcpplibs.os = "0.4.1" +mcpplibs.os = "0.4.5" # `modules/` holds what gets linked into this binary; `src/` is what has not # been separated yet, and shrinking it is the direction. Each entry is an diff --git a/modules/invocation/mcpp.toml b/modules/invocation/mcpp.toml index 1bb29ed19..efbf6fa0b 100644 --- a/modules/invocation/mcpp.toml +++ b/modules/invocation/mcpp.toml @@ -13,4 +13,4 @@ description = "The environment of one invocation of mcpp: statements over the st license = "Apache-2.0" [dependencies] -mcpplibs.os = "0.4.1" +mcpplibs.os = "0.4.5" diff --git a/modules/manifest/mcpp.toml b/modules/manifest/mcpp.toml index c3be033e2..1421be750 100644 --- a/modules/manifest/mcpp.toml +++ b/modules/manifest/mcpp.toml @@ -24,7 +24,7 @@ mcpp.source-kind = { path = "../source-kind" } mcpp.toolchain-model = { path = "../toolchain-model" } mcpp.versioning = { path = "../versioning" } -mcpplibs.os = "0.4.1" +mcpplibs.os = "0.4.5" # Subsystem-level tests. They see only this package, which is what lets them # state its contract in isolation; the root `tests/unit/` states the facts two diff --git a/modules/toolchain-model/mcpp.toml b/modules/toolchain-model/mcpp.toml index 21924bc76..a6cdc8862 100644 --- a/modules/toolchain-model/mcpp.toml +++ b/modules/toolchain-model/mcpp.toml @@ -23,7 +23,7 @@ license = "Apache-2.0" mcpp.invocation = { path = "../invocation" } mcpp.versioning = { path = "../versioning" } -mcpplibs.os = "0.4.1" +mcpplibs.os = "0.4.5" # Subsystem-level tests. They see only this package, which is what lets them # state its contract in isolation; the root `tests/unit/` states the facts two diff --git a/src/build/execute.cppm b/src/build/execute.cppm index a0ff70c9b..3359fae86 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -8040,10 +8040,30 @@ export void run_workspace_tests(std::span passthrough, } // `mcpp clean` driver. +// The configuration directories below `target/`: `//`. +std::vector configuration_dirs(const std::filesystem::path& target) { + std::vector out; + std::error_code ec; + for (std::filesystem::directory_iterator t(target, ec), end; !ec && t != end; t.increment(ec)) { + std::error_code fec; + if (t->path().filename().string().starts_with(".") || !t->is_directory(fec)) continue; + for (std::filesystem::directory_iterator f(t->path(), fec), fend; !fec && f != fend; f.increment(fec)) + if (std::error_code dec; f->is_directory(dec)) out.push_back(f->path()); + } + return out; +} + export int clean_project(bool wipe_bmi) { auto root = mcpp::project::find_manifest_root(std::filesystem::current_path()); if (!root) { std::println(libc::stderr(), "error: not in an mcpp package"); return 2; } std::error_code ec; + // Each configuration's compiler runs are retired first (§10.3 D); one a + // build holds keeps target/ as it is. + for (const auto& dir : configuration_dirs(*root / "target")) + if (auto retired = mcpp::build::schedule::detach::retire_compiler_runs(dir); !retired) { + std::println(libc::stderr(), "error: {}", retired.error()); + return 1; + } std::filesystem::remove_all(*root / "target", ec); if (ec) { std::println(libc::stderr(), "error: cannot remove target/: {}", ec.message()); @@ -8426,6 +8446,12 @@ export int clean_stale(bool dryRun, std::int64_t keepWithinSecs, bool quiet) { } } const auto size = mcpp::bmi_cache::dir_size(dir); + if (!dryRun) + if (auto retired = mcpp::build::schedule::detach::retire_compiler_runs(dir); !retired) { + std::println(libc::stderr(), "error: {}", retired.error()); + ++failed; + continue; + } swept.insert(dir); if (dryRun) { line(std::format("would remove {} ({})", shown, mcpp::bmi_cache::human_bytes(size))); diff --git a/src/build/schedule/compiler_transaction.cppm b/src/build/schedule/compiler_transaction.cppm index 1444beb55..9b275d8ce 100644 --- a/src/build/schedule/compiler_transaction.cppm +++ b/src/build/schedule/compiler_transaction.cppm @@ -302,19 +302,30 @@ std::expected certified_quiet(const Attempt& attempt) { if(*admitted && !(**admitted).empty())return std::unexpected("invalid compiler admission record"); return !*admitted; // publication/ACK precede admission; no compiler otherwise } -std::expected observed(const Attempt& attempt,ObserverMap* held=nullptr) { +// WHAT AN OBSERVATION OF AN ATTEMPT'S UNIT ANSWERS (design 2026-11-19 §10.2). +// Three facts and a fourth that may pass: the unit is empty, it is occupied, +// it cannot be observed from this process space at all (openkal 0.26 clause +// 7.18: another space, or a native identity not kept), or it cannot be +// decided now. Waiting changes the last only; `unobservable' was folded into +// `unknown', and a build interrupted in another process space was refused for +// ever, `mcpp clean' included. +enum class Observation { empty, occupied, unknown, unobservable }; +std::expected observed(const Attempt& attempt,ObserverMap* held=nullptr) { const auto quiet=certified_quiet(attempt);if(!quiet)return std::unexpected(quiet.error()); - if(*quiet)return os::process::unit_state::empty; + if(*quiet)return Observation::empty; auto unit=read(attempt.directory/"unit","mcpp-unit-v1");if(!unit)return std::unexpected(unit.error()); if(!*unit||(**unit).size()!=4)return std::unexpected("missing admitted compiler unit reference"); + auto answer=[](const os::result& value) { + if(!value)return Observation::unknown; + return *value==os::process::unit_state::empty ? Observation::empty + : *value==os::process::unit_state::occupied ? Observation::occupied : Observation::unknown; + }; const auto key=spelling(attempt.directory); - if(held)if(const auto found=held->find(key);found!=held->end()) { - auto value=found->second.observe(); - return value ? *value : os::process::unit_state::unknown; - } + if(held)if(const auto found=held->find(key);found!=held->end())return answer(found->second.observe()); auto observer=os::process::unit_observer::open((**unit)[0]); - if(!observer)return os::process::unit_state::unknown; - auto value=observer->observe();return value ? *value : os::process::unit_state::unknown; + if(!observer)return observer.error().code==os::errc::not_supported ? Observation::unobservable + : Observation::unknown; + return answer(observer->observe()); } std::expected reservation_available(const fs::path& reservation) { auto previous=read(reservation,"mcpp-reservation-v1");if(!previous)return std::unexpected(previous.error()); @@ -322,7 +333,7 @@ std::expected reservation_available(const fs::path& reservatio if((**previous).size()!=1)return std::unexpected("invalid compiler reservation"); auto attempt=Attempt::load(named((**previous)[0]));if(!attempt)return std::unexpected(attempt.error()); const auto state=observed(*attempt);if(!state)return std::unexpected(state.error()); - return *state==os::process::unit_state::empty; + return *state==Observation::empty; } bool release_reservation(const fs::path& reservation,const fs::path& attempt) { auto current=read(reservation,"mcpp-reservation-v1"); @@ -402,7 +413,16 @@ std::expected Attempt::register_unit(os::process::owned_unit& if(!connection||!send(*connection,spelling(directory)))return std::unexpected("cannot register the compiler unit"); auto ack=receive(*connection,std::chrono::seconds(2)); if(ack&&*ack=="ACK")return {}; - if(ack)return std::unexpected("compiler-run owner refused unit admission"); + // THE OWNER SAYS WHY (§10.3 F): `REFUSED `. A record it + // could not yet read is retried within the startup grace, as an + // unanswered registration is; every other condition is final. + if(ack) { + const auto condition=ack->starts_with("REFUSED ") ? ack->substr(8) : std::string("unstated"); + if(condition!="record-unreadable"||std::chrono::steady_clock::now()>=deadline) + return std::unexpected("compiler-run owner refused unit admission: "+condition + +" (state: "+spelling(directory)+")"); + continue; + } if(std::chrono::steady_clock::now()>=deadline)return std::unexpected(ack.error()); } } @@ -494,29 +514,48 @@ private: auto request=receive(connection,std::chrono::milliseconds(500)); if(!request||stopping_.load())continue; const auto attempt=named(*request); - if(attempt.parent_path()!=generation_/"attempts") {send(connection,"REFUSED");continue;} + auto refuse=[&](std::string_view condition) {send(connection,"REFUSED "+std::string(condition));}; + if(attempt.parent_path()!=generation_/"attempts") {refuse("outside-generation");continue;} auto prepared=Attempt::load(attempt); auto unit=read(attempt/"unit","mcpp-unit-v1"); - if(!prepared||!unit||!*unit||(**unit).size()!=4) {send(connection,"REFUSED");continue;} + if(!prepared||!unit||!*unit||(**unit).size()!=4) {refuse("record-unreadable");continue;} const auto actor=named((**unit)[3]); - if(actor.parent_path()!=generation_/"actors") {send(connection,"REFUSED");continue;} + if(actor.parent_path()!=generation_/"actors") {refuse("actor-outside");continue;} auto lease=os::fs::lock::acquire(actor,os::fs::lock::kind::exclusive,false); // The actor's lock is held, so its controller is running; a record // still settling does not refuse the registration. - if(lease||lease.error().code!=os::errc::again||controller_state(*prepared)==Controller::ended) {send(connection,"REFUSED");continue;} + if(lease||lease.error().code!=os::errc::again) {refuse("actor-not-held");continue;} + if(controller_state(*prepared)==Controller::ended) {refuse("controller-ended");continue;} auto retained=os::process::unit_observer::open((**unit)[0]); - if(!retained) {send(connection,"REFUSED");continue;} + if(!retained) {refuse("unit-unobservable");continue;} std::lock_guard guard(admission_); - if(stopping_.load()||failed_.load()) {send(connection,"REFUSED");continue;} + if(stopping_.load()||failed_.load()) {refuse("owner-stopping");continue;} held_.insert_or_assign(*request,std::move(*retained)); send(connection,"ACK"); } } }; -std::expected recover(const Attempt& attempt,ObserverMap* held=nullptr) { +// What the recovery of one generation met (§10.3 C, E): attempts it could not +// settle, by their observation, and attempts it settled as abandoned because +// their units cannot be observed from here. +struct Recovery { + std::size_t occupied = 0, unknown = 0, unobservable = 0; + std::size_t pending() const {return occupied+unknown;} +}; +// `abandonUnobservable': the caller holds the generation's owner lock and has +// found every actor lock idle. Those are locks of the file system, not of a +// process space, so the generation's controllers, helpers and writers have +// ended in every space that shares this directory, and an attempt whose unit +// cannot be observed is settled as an abandoned one. +std::expected recover(const Attempt& attempt,ObserverMap* held=nullptr, + Recovery* met=nullptr,bool abandonUnobservable=false) { const auto state=observed(attempt,held);if(!state)return std::unexpected(state.error()); - if(*state!=os::process::unit_state::empty)return false; + if(*state==Observation::unobservable&&abandonUnobservable) {if(met)++met->unobservable;} + else if(*state!=Observation::empty) { + if(met)++(*state==Observation::occupied ? met->occupied : met->unknown); + return false; + } auto retired=read(attempt.directory/"retired","mcpp-retired-v1"); auto result=attempt.ordinary ? read(attempt.directory/"result","mcpp-result-v1") : Record{std::optional{}}; auto published=read(attempt.directory/"published","mcpp-publication-v1"); @@ -620,12 +659,14 @@ std::expected,std::string> cached_result(const fs::path& slot return std::optional{1}; } -std::expected recover_generation(const fs::path& generation,ObserverMap* held=nullptr) { +std::expected recover_generation(const fs::path& generation,ObserverMap* held=nullptr, + bool abandonUnobservable=false) { const auto limit=std::chrono::steady_clock::now()+std::chrono::seconds(5); for(;;) { bool pending=false;std::error_code ec; + Recovery met; const auto directory=generation/"attempts"; - if(!fs::exists(directory,ec))return ec ? std::unexpected("cannot inspect compiler transactions") : std::expected{}; + if(!fs::exists(directory,ec))return ec ? std::unexpected("cannot inspect compiler transactions") : std::expected{met}; fs::directory_iterator entries(directory,ec);if(ec)return std::unexpected("cannot enumerate compiler transactions"); for(const auto& entry:entries) { auto prepared=read(entry.path()/"prepared","mcpp-attempt-v1"); @@ -637,11 +678,17 @@ std::expected recover_generation(const fs::path& generation,Ob continue; } auto attempt=Attempt::load(entry.path());if(!attempt)return std::unexpected(attempt.error()); - const auto done=recover(*attempt,held);if(!done)return std::unexpected(done.error()); + const auto done=recover(*attempt,held,&met,abandonUnobservable);if(!done)return std::unexpected(done.error()); if(!*done)pending=true; } - if(!pending)return {}; - if(std::chrono::steady_clock::now()>=limit)return std::unexpected("previous compiler unit is occupied or unknown; reservation and outputs were retained"); + if(!pending)return met; + // THE REFUSAL NAMES THE STATE AND THE WAY OUT (§10.3 E). + if(std::chrono::steady_clock::now()>=limit)return std::unexpected(std::format( + "{} compiler unit{} of an interrupted build of this configuration may still be running " + "({} occupied, {} unknown) [compiler-run-occupied]\n" + " state: {}\n" + " When no compiler of this configuration is running: mcpp clean, then build again.", + met.pending(),met.pending()==1?"":"s",met.occupied,met.unknown,spelling(generation))); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm index 49ee36cdf..4027fa44e 100644 --- a/src/build/schedule/detach_codegen.cppm +++ b/src/build/schedule/detach_codegen.cppm @@ -76,6 +76,7 @@ import mcpp.run; import mcpp.build.stage; import mcpp.buildopt.keep; import mcpp.build.schedule.compiler_transaction; +import mcpp.diag; import mcpp.home; import mcpp.home.files; import mcpp.invocation; @@ -93,6 +94,14 @@ inline constexpr std::string_view kCapEnv = "MCPP_COMPILER_CAP"; // The generation carries the compiler cap: build.ninja names no number of // compilers, so the graph is the same whatever the job count (e2e 861), and // every executor of the generation reads the cap from its owner record. +// `mcpp clean` (design 2026-11-19 §10.3 D): before a configuration's output +// directory is removed, its compiler-run generations are retired under the +// gate a build takes. Its outputs are being removed, so there is nothing an +// old generation's recovery would protect and nothing is observed. A +// generation whose owner or actor lock is held belongs to a running build: +// the error names it, and the configuration is not cleaned. +std::expected retire_compiler_runs(const std::filesystem::path& outputDir); + class RunScope { public: static std::expected begin(const std::filesystem::path& outputDir, int compilerCap); @@ -617,6 +626,12 @@ std::expected retire_owned(mcpplibs::os::process::owned_unit& } // namespace struct RunScope::State { + // False where this program cannot declare a compiler unit (§10.3 H): the + // build then compiles as an unmanaged helper does, and there is no + // generation to recover, export or retire; the helpers read the cap from + // `kCapEnv`, as an executor outside a generation does. + bool managed = true; + int cap = 0; std::filesystem::path root, generation; std::optional gate; std::optional owner; @@ -653,27 +668,71 @@ RunScope::RunScope(RunScope&&) noexcept = default; RunScope& RunScope::operator=(RunScope&&) noexcept = default; RunScope::~RunScope() = default; -std::expected RunScope::begin(const std::filesystem::path& outputDir, int compilerCap) { - if (compilerCap <= 0) return std::unexpected("a compiler-run generation needs a compiler cap"); - const auto self = mcpplibs::os::process::self(); - if (!self || self->id == 0 || self->started_ns == 0) - return std::unexpected("cannot identify the compiler-run owner and its birth time"); +namespace { +// Where a configuration's compiler-run state lies: +// `/.mcpp/compiler-runs//`. +struct RunsLocation { std::filesystem::path canonical, work, root; }; +std::expected locate_runs(const std::filesystem::path& outputDir) { std::error_code ec; const auto spelled = std::filesystem::absolute(outputDir, ec).lexically_normal(); if (ec) return std::unexpected("cannot identify the compiler-run output directory"); - const auto canonical = std::filesystem::weakly_canonical(spelled, ec); + RunsLocation at; + at.canonical = std::filesystem::weakly_canonical(spelled, ec); if (ec) return std::unexpected("cannot identify the compiler-run output directory"); - auto state = std::make_unique(); // Locate work BEFORE resolving output symlinks: target/ may point // at a disposable cache. Locks remain beside target/, independently of // clean and MCPP_HOME; the canonical output still identifies the slots. const auto target = spelled.parent_path().parent_path(); - const auto work = std::filesystem::weakly_canonical( + at.work = std::filesystem::weakly_canonical( target.filename() == "target" ? target.parent_path() : spelled.parent_path(), ec); if (ec) return std::unexpected("cannot identify the compiler-run work directory"); - mcpp::home::mark_project_state(work / ".mcpp"); - state->root = work / ".mcpp" / "compiler-runs" - / mcpp::toolchain::hash_string(mcpp::home::spelling(canonical)); + at.root = at.work / ".mcpp" / "compiler-runs" / mcpp::toolchain::hash_string(mcpp::home::spelling(at.canonical)); + return at; +} +} // namespace + +std::expected retire_compiler_runs(const std::filesystem::path& outputDir) { + auto located = locate_runs(outputDir); + if (!located) return std::unexpected(located.error()); + std::error_code ec; + if (!std::filesystem::is_directory(located->root, ec)) return {}; + auto gate = mcpp::home::FileLock::acquire(located->root, "gate.lock", "", + std::chrono::milliseconds(300), {}); + if (!gate) return std::unexpected(std::format( + "an mcpp is building {}: it holds the compiler-run gate of {}; it was not cleaned", + mcpp::home::spelling(outputDir), mcpp::home::spelling(located->root))); + std::filesystem::directory_iterator runs(located->root, ec); + if (ec) return std::unexpected("cannot inspect the compiler runs of " + mcpp::home::spelling(outputDir)); + for (const auto& entry : runs) { + if (!entry.path().filename().native().starts_with(std::filesystem::path("run-").native())) continue; + { + // Released before the directory goes: a lock held on a file of + // it keeps it on Windows. + auto owner = mcpplibs::os::fs::lock::acquire(entry.path() / "owner.lock", + mcpplibs::os::fs::lock::kind::exclusive, false); + if (!owner || !actors_idle(entry.path())) return std::unexpected(std::format( + "an mcpp is building {}: the helpers of {} hold its locks; it was not cleaned", + mcpp::home::spelling(outputDir), mcpp::home::spelling(entry.path()))); + } + std::filesystem::remove_all(entry.path(), ec); + if (ec) return std::unexpected(std::format("cannot retire {}: {}", + mcpp::home::spelling(entry.path()), ec.message())); + } + return {}; +} + +std::expected RunScope::begin(const std::filesystem::path& outputDir, int compilerCap) { + if (compilerCap <= 0) return std::unexpected("a compiler-run generation needs a compiler cap"); + const auto self = mcpplibs::os::process::self(); + if (!self || self->id == 0 || self->started_ns == 0) + return std::unexpected("cannot identify the compiler-run owner and its birth time"); + std::error_code ec; + auto located = locate_runs(outputDir); + if (!located) return std::unexpected(located.error()); + const auto canonical = located->canonical; + auto state = std::make_unique(); + mcpp::home::mark_project_state(located->work / ".mcpp"); + state->root = located->root; std::filesystem::create_directories(state->root, ec); if (ec) return std::unexpected("cannot create compiler-run runtime state: " + ec.message()); state->gate = mcpp::home::FileLock::acquire(state->root, "gate.lock", "", @@ -691,8 +750,21 @@ std::expected RunScope::begin(const std::filesystem::path if (oldOwner && actors_idle(entry.path())) { free = true; break; } std::this_thread::sleep_for(std::chrono::milliseconds(10)); } while (std::chrono::steady_clock::now() < limit); - if (!free) return std::unexpected("previous compiler helpers are still active; their state was retained"); - if(auto recovered=transactions::recover_generation(entry.path());!recovered)return std::unexpected(recovered.error()); + if (!free) return std::unexpected(std::format( + "an mcpp is building this configuration: the helpers of {} hold its locks, from this " + "process space or another that shares the directory; its state was retained\n" + " When that build has ended, build again.", mcpp::home::spelling(entry.path()))); + // The owner lock and every actor lock are free: what cannot be observed + // from here is settled as abandoned (§10.3 C). + auto recovered=transactions::recover_generation(entry.path(),nullptr,true); + if(!recovered)return std::unexpected(recovered.error()); + if(recovered->unobservable>0) + mcpp::diag::degraded("compiler-run/unobservable", std::format( + "{} compiler unit{} of an interrupted build cannot be observed from this process space " + "(state: {}); their unpublished outputs were discarded", + recovered->unobservable, recovered->unobservable==1?"":"s", mcpp::home::spelling(entry.path())), + "a compiler that outlived its controller in that process space is not found; a late " + "write is caught by the next build, which compiles that unit again"); std::filesystem::remove_all(entry.path(), ec); if (ec) return std::unexpected("cannot retire previous compiler-run state"); } @@ -700,6 +772,22 @@ std::expected RunScope::begin(const std::filesystem::path // no Ninja reader from this generation has started yet. if(auto admitted=transactions::admit_cached_outputs(canonical);!admitted) return std::unexpected(admitted.error()); + // WHETHER A UNIT CAN BE DECLARED HERE IS A PROPERTY OF THE ENVIRONMENT + // (§10.3 H), asked once for the build. Where it cannot, every compile of + // the build runs as an unmanaged helper does -- bounded by the slots, not + // declared as a unit -- and this is said once, rather than each compile + // failing without a word (an Android app sandbox, 2026-11-18). + if (!mcpplibs::os::process::units_observable()) { + mcpp::diag::degraded("compiler-run/no-unit", + "this environment gives a program nothing to declare a compiler unit from; " + "the compilers of this build are not declared as units", + "an interrupted build's compilers are not found by the next build, and a compiler " + "that outlives its controller may write an output after it"); + state->managed = false; + state->cap = compilerCap; + state->gate.reset(); + return RunScope(std::move(state)); + } static std::atomic sequence{0}; state->generation = state->root / std::format("run-{}-{}-{}", self->id, self->started_ns, sequence++); std::filesystem::create_directories(state->generation, ec); @@ -719,6 +807,11 @@ std::expected RunScope::begin(const std::filesystem::path void RunScope::environment(std::vector>& env) const { std::erase_if(env, [](const auto& entry) { return entry.first == kRunEnv; }); + if (!state_->managed) { + std::erase_if(env, [](const auto& entry) { return entry.first == kCapEnv; }); + env.emplace_back(kCapEnv, std::to_string(state_->cap)); + return; + } env.emplace_back(kRunEnv, mcpp::home::spelling(state_->generation)); } std::expected RunScope::finish() { return state_->close(); } @@ -1159,7 +1252,7 @@ int run_capped(const std::vector& argv, const std::filesystem::path& semaphore, int maxCompilers, const std::filesystem::path& output, const std::filesystem::path& depfile, const std::filesystem::path& bmi, bool object) { - if (argv.empty()) return 2; + if (argv.empty()) { std::println(std::cerr, "mcpp: __compiler-cap: no compiler command was given"); return 2; } auto actor = std::make_shared(); if (!actor->ready) return 130; Previous previousBmi{Previous::Kind::Bmi, bmi}; @@ -1168,30 +1261,43 @@ int run_capped(const std::vector& argv, // certificate names the output as it stands. auto settle = [&](bool compiled) { previousBmi.settle(compiled); previousObject.settle(compiled); }; if(actor->owner.managed) { + // EVERY RETURN BUT THE COMPILER'S OWN STATUS AND A CANCELLATION SAYS + // WHICH STEP ENDED IT AND WHY (§10.3 G): ninja shows the line, and a + // helper that ended with 1 or 2 and nothing on its error stream left a + // failed build with no cause (Termux, 2026-11-18). + const auto out_name=output.empty() ? std::string("a compile") : mcpp::home::spelling(output); + auto ended=[&](int status,std::string_view step,std::string_view why) { + std::println(std::cerr,"mcpp: {} for {}: {}",step,out_name,why);return status; + }; auto prepared=transactions::Attempt::prepare(actor->owner.generation,true,{},output,{},depfile); - if(!prepared) {std::println(std::cerr,"mcpp: {}",prepared.error());return 1;} + if(!prepared)return ended(1,"preparing the compiler transaction",prepared.error()); auto attempt=std::move(*prepared); const auto who=mcpplibs::os::process::self(); - if(!who||!who->id||!who->started_ns - ||!transactions::publish(attempt.directory/"controller","mcpp-controller-v1",{ - std::to_string(who->id),std::to_string(who->started_ns),transactions::spelling(actor->path)}))return 1; + if(!who||!who->id||!who->started_ns) + return ended(1,"identifying the compiler controller","this process's identifier or birth cannot be read"); + if(!transactions::publish(attempt.directory/"controller","mcpp-controller-v1",{ + std::to_string(who->id),std::to_string(who->started_ns),transactions::spelling(actor->path)})) + return ended(1,"recording the compiler controller","its record cannot be written"); auto permit=acquire_permit(*actor,semaphore,maxCompilers);if(!permit)return permit.error(); auto created=mcpplibs::os::process::owned_unit::create(); - if(!created)return 2; + if(!created)return ended(2,"declaring the compiler unit",created.error().message()); auto unit=std::make_shared(std::move(*created)); InterruptCleanup cleanup{mcpp::run::add_cleanup([actor,unit] {unit->terminate();})}; - if(auto registered=attempt.register_unit(*unit,actor->path,permit->reservation);!registered) { - std::println(std::cerr,"mcpp: {}",registered.error());return 2; - } + if(auto registered=attempt.register_unit(*unit,actor->path,permit->reservation);!registered) + return ended(2,"registering the compiler unit",registered.error()); if(!actor->owner.active())return 130; - if(!transactions::publish(attempt.directory/"admitted","mcpp-admitted-v1",{}))return 2; + if(!transactions::publish(attempt.directory/"admitted","mcpp-admitted-v1",{})) + return ended(2,"recording the compiler's admission","its record cannot be written"); const auto result=mcpp::run::execute(argv,{ .input=mcpp::run::Input::inherit,.streams=mcpp::run::Streams::inherit, .should_stop=[&] {return !actor->owner.active();},.declared_unit=unit.get()}); - if(auto retired=retire_owned(*unit,actor->owner);!retired) {std::println(std::cerr,"mcpp: {}",retired.error());return 1;} + if(auto retired=retire_owned(*unit,actor->owner);!retired) + return ended(1,"retiring the compiler unit",retired.error()); settle(result.exit_code==0&&actor->owner.active()); - if(!attempt.retired(result.exit_code)||!attempt.result(result.exit_code))return 1; - if(!transactions::release_reservation(permit->reservation,attempt.directory))return 1; + if(!attempt.retired(result.exit_code)||!attempt.result(result.exit_code)) + return ended(1,"certifying the compiler's result","its record cannot be written"); + if(!transactions::release_reservation(permit->reservation,attempt.directory)) + return ended(1,"releasing the compiler slot","its reservation cannot be released"); return result.exit_code; } InterruptCleanup cleanup{mcpp::run::add_cleanup([actor, output] { diff --git a/src/run/run.cpp b/src/run/run.cpp index 22e944ebb..568531054 100644 --- a/src/run/run.cpp +++ b/src/run/run.cpp @@ -158,6 +158,9 @@ int errno_of(os::errc c) { case os::errc::not_directory: return static_cast(std::errc::not_a_directory); case os::errc::not_supported: return static_cast(std::errc::not_supported); case os::errc::too_large: return static_cast(std::errc::argument_list_too_long); + // At a start, the system refused it for the size of what it was given + // (openkal 0.26, clause 7.6). + case os::errc::no_space: return static_cast(std::errc::argument_list_too_long); default: return static_cast(std::errc::io_error); } } @@ -174,6 +177,32 @@ std::string spawn_failure(std::string_view program, int error, std::string_view std::generic_category().message(error), detail.empty() ? "" : "; ", detail); } +// A START REFUSED FOR ITS SIZE (design 2026-11-19 §9.2): the two vectors' +// counts and bytes, which is what a caller acts upon, and the reason token. +// Before openkal 0.26 this read "Cannot allocate memory" at 512 elements, a +// bound of the kernel layer and not of the system (520 `[build] defines`). +std::string too_large_failure(std::string_view program, std::span args, + const std::vector>>& changes) { + std::size_t argumentBytes = program.size() + 1; + for (auto const& a : args) argumentBytes += a.size() + 1; + std::map environment; + for (auto const& entry : os::env::variables()) { + const auto eq = entry.find('='); + environment[entry.substr(0, eq)] = eq == std::string::npos ? std::string{} : entry.substr(eq + 1); + } + for (auto const& [k, v] : changes) { + if (v) environment[k] = *v; + else environment.erase(k); + } + std::size_t variableBytes = 0; + for (auto const& [k, v] : environment) variableBytes += k.size() + v.size() + 2; + return std::format("cannot start '{}': the request is larger than the system admits ({} arguments, " + "{} bytes; {} variables, {} bytes) [spawn-too-large]\n" + " Give the program fewer or shorter arguments (a response file, `@file`), or " + "a smaller environment.\n", + program, args.size() + 1, argumentBytes, environment.size(), variableBytes); +} + // The environment a started program receives: this invocation's statements, // the inherited loader search paths without mcpp's private glibc (an outer // `mcpp run` points them at a payload a host tool must not load), and the @@ -286,7 +315,9 @@ Outcome launch(const Spec& s) { if (!p) { out.spawn_error = errno_of(p.error().code); out.spawn_detail = detail_of(p.error()); - out.spawn_message = spawn_failure(s.argv.front(), out.spawn_error, out.spawn_detail); + out.spawn_message = p.error().code == os::errc::no_space + ? too_large_failure(s.argv.front(), o.args, o.env) + : spawn_failure(s.argv.front(), out.spawn_error, out.spawn_detail); return out; } Registered guard(&*p); @@ -526,6 +557,10 @@ int run_foreground(const std::vector& argv, const Env& extraEnv, in auto r = os::process::exec(o); // returns only when the program could not be started error = r ? static_cast(std::errc::io_error) : errno_of(r.error().code); if (!r) detail = detail_of(r.error()); + if (!r && r.error().code == os::errc::no_space && !spawn_error) { + std::fputs(too_large_failure(argv.front(), o.args, o.env).c_str(), libc::stderr()); + return 127; + } } if (spawn_error) *spawn_error = error; else std::fputs(spawn_failure(argv.front(), error, detail).c_str(), libc::stderr()); diff --git a/tests/e2e/build/basics.cpp b/tests/e2e/build/basics.cpp index abf1fdf15..16c0a4d68 100644 --- a/tests/e2e/build/basics.cpp +++ b/tests/e2e/build/basics.cpp @@ -896,6 +896,53 @@ a = { path = "../a" } expect(fatal(e2e::contains(s::text(r), "reaches two variants of 'compat.zlib'"))) << "the refusal does not name the package\n" << s::text(r); expect(s::count_lines(s::text(r), "compat.zlib+p") >= 2u) << "the refusal does not name both variants\n" << s::text(r); }; + "952 a build interrupted in another process space continues"_test[s::gcc][s::elf] = [] { + // mcpp#952, design 2026-11-19 §10.3 C and D. A build killed inside a pid namespace of its + // own leaves admitted compiler units this process space can never observe. The next build + // continues, settles them as abandoned and says so once (`compiler-run/unobservable`, whose + // text names the process space); + // `mcpp clean` then leaves no generation of the configuration behind. + const auto unshare = s::on_path("unshare"); + const auto timeout = s::on_path("timeout"); + if (!unshare || !timeout || !e2e::succeeded(s::run(*unshare, {"-Urpf", "--mount-proc", "true"}))) + skip("952 -- no new pid namespace on this machine"); + const auto root = s::case_dir(); + std::vector> seed{ + {"mcpp.toml", "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}}; + std::string main_cpp = "import std;\n", body = "int main() { int s = 0;"; + for (int i = 0; i < 24; ++i) { + seed.emplace_back(std::format("src/u{}.cpp", i), std::format( + "import std;\nint f{0}(int x) {{ std::vector v; for (int i = 0; i < x; ++i) " + "v.push_back(std::format(\"{{}}\", i)); std::ranges::sort(v); return (int)v.size(); }}\n", i)); + main_cpp += std::format("int f{}(int);\n", i); + body += std::format(" s += f{}(3);", i); + } + seed.emplace_back("src/main.cpp", main_cpp + body + " std::println(\"{}\", s); }\n"); + auto pending = [](const fs::path& dir) { + std::size_t n = 0; + for (auto const& a : s::find_files(dir / ".mcpp", "admitted")) + if (!fs::exists(a.parent_path() / "retired")) ++n; + return n; + }; + std::optional hit; + for (auto const* t : {"0.8", "1.0", "1.4", "1.8", "2.2", "2.6", "3.2"}) { + s::project p(root / t, {}); + for (auto const& [name, text] : seed) p.write(name, text); + e2e::command killed{.argv = {unshare->string(), "-Urpf", "--mount-proc", timeout->string(), "-s", "KILL", t, + s::mcpp_path(), "build"}, + .cwd = p.dir(), .env = {{"MCPP_JOBS", "8"}}, .timeout = std::chrono::minutes{2}}; + (void)e2e::run(killed); + if (pending(p.dir()) > 0) { hit.emplace(std::move(p)); break; } + } + if (!hit) skip("952 -- no kill landed while a compiler was admitted"); + auto r = hit->mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << "the build after the interrupted one was refused: " << s::text(r); + expect(s::count_lines(s::text(r), "cannot be observed from this process space") == 1u) + << "the abandoned units were not reported once: " << s::text(r); + r = hit->mcpp({"clean"}); + expect(fatal(e2e::succeeded(r))) << "mcpp clean failed: " << s::text(r); + expect(s::find_files(hit->dir() / ".mcpp", "owner").empty()) << "a compiler-run generation survived mcpp clean"; + }; }}; } // namespace From 7313dee5bc63f4a2579a74ad5ad01ed1628df17e Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 18:07:54 +0900 Subject: [PATCH 11/30] =?UTF-8?q?A=20compile=20writes=20only=20its=20own?= =?UTF-8?q?=20attempt,=20and=20the=20executor=20publishes=20what=20it=20wr?= =?UTF-8?q?ote=20(design=202026-11-19=20=C2=A710.6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Where a toolchain's outputs are relocatable, a compile writes below /.mcpp-attempts// and the executor publishes by rename once the compiler has ended, a module's BMI the moment it is complete; an object or BMI equal to the published one is kept. The attempt is recorded before its compiler is admitted, so recovery discards an interrupted build's unpublished attempts without observing them: a compiler that outlives its build writes nothing the next build trusts. GCC 11 and later relocate through -dumpdir/-dumpbase and a per-attempt module map, the root package's module units included; a clang driver relocates when it takes -dumpdir, which detection asks of it once and keeps (driver_takes_dumpdir); CodeView targets, NASM and the resource compilers write in place. SPEC-011 §5.1 clause 8. Under clang's two-phase schedule a cache fill now asks for every BMI of its package: a module no unit imported had no BMI and the fill was refused. The tests follow the package instance layout (mcpp#953) and the link record (mcpp#931): the module address test, graph on Windows, the 847 map lookup, the 669 and 667 packs, and runners/330. The upgrade lane records 2026.11.19.1 as the last BMI spelling change. The role refusals of the action graph move into one function, so the step stays within the prepare size bound. --- ...11-19-ecosystem-rule-convergence-design.md | 82 +++- .github/tools/upgrade_lane.sh | 10 +- CHANGELOG.md | 28 +- docs/04-mcpp-toml.md | 23 +- docs/specs/concurrency.md | 41 +- docs/zh/04-mcpp-toml.md | 12 +- modules/buildopt/src/keep/keep.cppm | 20 +- modules/toolchain-model/src/model.cppm | 50 ++ .../tests/test_output_relocation.cpp | 50 ++ src/build/ninja_backend.cppm | 49 +- src/build/plan.cppm | 80 ++- src/build/prepare/plan.cpp | 61 +-- src/build/schedule/compiler_transaction.cppm | 50 +- src/build/schedule/detach_codegen.cppm | 194 ++++++-- src/build/schedule/relocation.cppm | 462 ++++++++++++++++++ src/cli.cppm | 10 +- src/cli/cmd_build.cppm | 6 +- src/toolchain/clang.cpp | 3 + src/toolchain/probe.cppm | 43 ++ .../667_an_android_pack_stages_its_closure.sh | 29 +- tests/e2e/build/basics.cpp | 125 ++++- tests/e2e/modules/programs.cpp | 14 +- tests/e2e/pack/modes.cpp | 28 +- tests/e2e/run/runners.cpp | 12 +- tests/e2e/workspace/graph.cpp | 2 +- tests/unit/test_compile_relocation.cpp | 161 ++++++ tests/unit/test_compiler_transaction.cpp | 39 ++ tests/unit/test_module_address.cpp | 31 +- tests/unit/test_ninja_backend.cpp | 59 ++- 29 files changed, 1579 insertions(+), 195 deletions(-) create mode 100644 modules/toolchain-model/tests/test_output_relocation.cpp create mode 100644 src/build/schedule/relocation.cppm create mode 100644 tests/unit/test_compile_relocation.cpp diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md index 81575ab9e..9ac831c18 100644 --- a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md @@ -5,8 +5,8 @@ status: active # 生态规则收敛方案:2026.11.18.1 之后十个 issue 的五类成因、七项决定与实施范围 -**日期:** 2026-11-19 · **版本:** v3(待评审,未实施。用户决定:全部在本轮实施,每个仓库一个拉取请求,提交作为检查点, -从成因解决而不留变通。第 15 节为七项决定的依据,第 16 节为两轮自评,第 17 节为已定事项与仍需决定的三项) · **基线:** mcpp 2026.11.18.1(`3b886163`)、openkal 0.25.3、openkal-linux 0.26.3(`363ccbb`)、 +**日期:** 2026-11-19 · **版本:** v4(实施中。用户决定:全部在本轮实施,每个仓库一个拉取请求,提交作为检查点, +从成因解决而不留变通;N1 至 N3 按建议。第 15 节为七项决定的依据,第 16 节为两轮自评,第 17 节为已定事项,第 19 节为实施的状态与测量) · **基线:** mcpp 2026.11.18.1(`3b886163`)、openkal 0.25.3、openkal-linux 0.26.3(`363ccbb`)、 openkal-macos 0.23.5(`9dbc36f`,索引中为 0.23.4)、openkal-platform 0.29.2(引擎默认)与 0.30.1、mcpplibs.os 0.4.4、 mcpp.plugins 0.26.0、mcpp-index `3ee5f08`、xlings `4f5df092` · **前文:** [第二部分之后的补充方案](2026-11-17-ninja-cascade-publish-supplement-design.md)、 @@ -343,10 +343,10 @@ mcpp 是否应当为编译器裁剪环境是另一个问题(“默认封闭” | 标识单元的进程号现在属于另一个出生时刻的进程 | `EMPTY` | POSIX:存在以某进程号为组号的进程组时,该进程号不被重新分配 | | 引用产生于同一次启动的另一个进程空间 | `kal_process_job_open` 返回 `kal_err_not_supported`(现状),规范写明这是对该调用者不变的回答 | 调用者无从枚举另一空间的进程 | | 进程空间的标识 | 规范:引用**必须**精确标识其进程空间,实现**不得**以另一空间的进程回答。openkal-linux 的引用在命名空间的编号之外记入该空间 1 号进程的出生时刻(格式 `L3`;`L2` 的引用按不可观察处理) | 编号会被复用(实测);同一次启动内编号与 1 号进程的出生时刻不会同时相同 | -| 能否声明单元 | `KAL_PROCESS_PROP_JOB` 只在实现能够产生引用的环境中声明(启动标识、进程空间与出生时刻均可读) | 条款 6.1:能力的缺失是使用者得知的事实 | +| 能否声明单元 | `KAL_PROCESS_PROP_JOB_OBSERVE` 只在实现能够产生引用的环境中声明(启动标识、进程空间与出生时刻均可读) | 条款 6.1:能力的缺失是使用者得知的事实 | | 其余不能判定的情形 | `UNKNOWN`,含义限定为暂时 | — | -**B. mcpplibs.os。** `unit_observer::open` 的错误区分“不可观察”(`not_supported`)与其余失败;提供对 `KAL_PROCESS_PROP_JOB` 的查询。 +**B. mcpplibs.os。** `unit_observer::open` 的错误区分“不可观察”(`not_supported`)与其余失败;提供对 `KAL_PROCESS_PROP_JOB_OBSERVE` 的查询。 **C. 引擎的恢复(D3)。** 未退役的尝试有三种结果: @@ -625,7 +625,7 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 | 语义与一致性 | 与 D3 同一条规定:观察能力缺失时按许可与锁运行,并写明 | 两处行为不一致 | | 安全 | 失去的是对比控制方活得更久的编译器后代的发现,即引入单元之前的行为 | 没有损失,也没有构建 | -结论:降级并报告;依据是环境不声明 `KAL_PROCESS_PROP_JOB`,声明了而失败照常拒绝。 +结论:降级并报告;依据是环境不声明 `KAL_PROCESS_PROP_JOB_OBSERVE`,声明了而失败照常拒绝。 ### 15.5 D5 同一路径的两种配置:配置计入对象目录 @@ -690,7 +690,7 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 | 8 | 引擎的事实一个记号;插件携带自己的程序 | 成因 | — | | 9 | 规范条款与实现 | 成因 | — | | 10 A | 两种可证明为空的情形 | 不完整:进程空间以名字识别,而名字被复用(实测) | 引用精确标识其进程空间,写入规范;实现记入 1 号进程的出生时刻 | -| 10 H | 调用失败后降级 | 表现:能力的缺失在每次编译失败时才被发现 | 由 `KAL_PROCESS_PROP_JOB` 在构建开始时决定一次;实现只在能产生引用的环境声明它 | +| 10 H | 调用失败后降级 | 表现:能力的缺失在每次编译失败时才被发现 | 由 `KAL_PROCESS_PROP_JOB_OBSERVE` 在构建开始时决定一次;实现只在能产生引用的环境声明它 | | 10 C | 不可观察时按锁恢复并报告 | 表现:它规定了就地写入之下的行为,而需要观察的原因是编译器直接写已发布的路径 | 增加 10.6,产物的事务化发布,前提在检查点内测量;列为 N1 | | 11.1 | 配置计入对象目录 | 成因 | — | | 11.2 | 载荷带 zlib 与 zstd | 成因;但生态中已有的绕过未撤销 | FreeBSD 基座恢复上游的字节 | @@ -751,3 +751,73 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 - xlings 与 mcpp-index 的工具不读取描述符中的 `mcpp` 表,依据是对两个仓库的检索,未运行(15.6)。 - 14.2 中其余实现的仓库(openkal-freebsd、-uefi、-dos、-svr4)是否实现进程接口,未读。 - #957 未在本机复现,依据为源码与报告中的输出。 + +## 19. 实施的状态与测量 + +### 19.1 各项的落点 + +| 项 | 落点 | +|---|---| +| 3(#960) | mcpp 检查点 2;SPEC-001 §3.5 | +| 4(#956) | 检查点 3;SPEC-013 §7.5,SPEC-004 §4.8 | +| 5(#957) | 检查点 4;SPEC-016 §6 | +| 6(#931) | 检查点 5;SPEC-004 §8.1 | +| 7(#954) | 检查点 6 | +| 8(#959、#955) | 检查点 7、8;SPEC-007 R3.16、R3.17、R6.7,协议 20;mcpp-plugins 的 `steps` 以 mcpp 的发布为引擎下界 | +| 9(#958) | openkal 0.26.0 条款 7.6 与各实现,mcpplibs.os 0.4.5,openkal-musl 0.30.0,mcpp 检查点 10 | +| 10(#952) | openkal 0.26.0 条款 7.18 与各实现,mcpplibs.os 0.4.5,mcpp 检查点 10;SPEC-011 §5.1 第 6、9、10 条 | +| 10.6(N1) | mcpp 检查点 10b;SPEC-011 §5.1 第 8 条 | +| 11.1(#953 D5) | 检查点 9;SPEC-004 §9 第 17 条 | +| 11.2(#953) | xim-pkgindex#1196:llvm 23.1.3 修订 4,FreeBSD 基座修订 3 | +| 12(超时) | mtest 0.10.0(mcpp-community/mtest#25),openkal-image 0.7.0(okrun 的结束请求记录) | + +### 19.2 测量 + +- **链接器的报告(6.2 规则 B)。** GNU ld 2.42、gold 1.16 与 ld.lld 23.1.3 接受 `--dependency-file`;三者在回答 + `--version` 之前拒绝未知选项,所以探测以驱动的一次真实链接(经驱动链接时)或该选项加 `--version`(直接调用链接器时) + 回答,答案按工具链缓存。 +- **包实例(11.1)。** 修正前,31 个成员的工作区中两个成员之间切换一次重新编译 43 条边;修正后交替执行 `mcpp test` + 与 `mcpp build`,共享包的两个实例都保留,交替时不再编译。`<包>@<摘要>` 经测量不采纳:ninja 对含 `@` 的输出加引号, + 规则中的 `"$out.d"` 因而命名一个带引号的文件。GCC 的 BMI 按映射的写法记录所导入模块的路径,构建缓存中按旧布局编译的 + BMI 在新布局下报 `CRC mismatch`、`Bad import dependency`:缓存条目的键因此计入实例(`bmiPathConvention`)。 +- **不可观察的单元(10.3)。** e2e 952:在新的 pid 命名空间中被结束的构建,从当前空间继续成功;`mcpp clean` 之后没有 + 运行状态留下。只以编号命名进程空间的 `L2` 引用由 openkal-linux 0.27 回答为另一空间,事务套件以它构造不可观察的就地尝试。 +- **产物的事务化发布(10.6)。** P1:GCC 16.1 以尝试自己的映射(相对于 `$root` 的 `../.mcpp-attempts/<尝试>/out/...`) + 写出自己的 BMI,改名后导入者读取成功;BMI 记录源文件与所导入的模块(`import: x x.gcm`),不记录自己的路径。GCC 的 + 文件映射对未列出的名字不回退(`unknown compiled module interface`),所以根包中提供模块的 GCC 单元也被给予映射。 + P2:clang 23.1.3 的 `-fmodule-output=` 与 `-o` 指向私有路径,结果相同;目标文件的字节在 ELF、Mach-O 与 wasm 上与路径 + 无关(`-O2`,带与不带 `-g`),在 CodeView 目标(`*-windows-msvc`)上随路径变化,不带 `-g` 亦然。emcc 6.0.9 接受 + `-dumpdir`,目标文件与路径无关。驱动自己的编号不回答它是否接受 `-dumpdir`(Apple clang 16 与 emcc 6.0.9 接受, + LLVM 17 之前拒绝这个词),按名字区分驱动又违反引擎不点名实现的规则:工具链在探测时以一次空单元的编译询问驱动, + 回答按驱动的身份保存(`driver_takes_dumpdir`,`Toolchain::takesDumpdir`)。P3:mcpp 实际产生的附属输出为 `-gsplit-dwarf` 的 `.dwo`(dev 配置下的本地单元), + 用户的标志可加入 `--coverage` 的 `.gcno`。`-fdebug-prefix-map` 与 `-ffile-prefix-map` 都不改写 `.dwo` 的记录名, + 编译器在哪里写出就记录哪里;所以以 `-dumpdir`、`-dumpbase`(GCC)与 `-dumpdir`(clang)取最终名字并就地写出。clang + 不接受 `-dumpbase`,其名字为 `<目标文件主名>.<源文件主名>.dwo`。实现中另测得:GCC 的依赖文件把模块映射文件列为前提, + 尝试的映射在发布后删除,下一次构建因而判定每个带映射的单元为脏;依赖文件因此以发布后的名字发布。e2e 952(构建被切断 + 而两个编译器继续运行):下一次构建不等待、不拒绝,孤儿编译器之后写出的文件不触及已发布的目标文件与 BMI。 + 冷构建(mcpp 自身,同一引擎,以临时开关在两种写法间切换,交替各两次,`MCPP_JOBS=16`):重定向 162.4 s、169.3 s,就地 198.4 s、162.8 s;同一写法的两次相差可达 36 s(机器上另有负载),两种写法之差在这一波动之内。 +- **Windows 的 LLVM 载荷(11.2)。** 原因不是 lld 不带 zlib:23.1.3 修订 3 的 `lld-link.exe` 读写 zlib 与 zstd,但归档中 + 没有 `ld.lld.exe`,clang 的驱动于是运行宿主 PATH 上不带 zlib 的 LLD 20.1.8。修订 4 补上上游的 `ld.lld.exe`、 + `ld64.lld.exe` 与 `wasm-ld.exe`,载荷的 CI 在四种宿主上链接带 zlib 与 zstd 压缩段的对象。20.1.7 与 22.1.8 的 Windows + 载荷是上游不带 zlib 的构建,配方写明;mcpp 的默认为 23.1.3。 +- **openkal-musl。** 级联中测得 `okm_spawn` 自带 512 项的界限,并把 `kal_err_no_space` 报为 `ENOSPC`,与 #958 同一成因; + 0.30.0 删除该界限、以 `E2BIG` 报告大小的拒绝,子进程探针以 2,000 个参数与一个 4 MiB 的参数核对,两项在 0.29.10 上均失败。 + +### 19.3 第 18 节中已测量的事项 + +- 11.1 的两种交替、10.6 的 P1 至 P3:见 19.2。 +- 规则 B:ld.lld 经 mcpp 的链接边运行(e2e 931 两例与 emscripten 一例);mold 未核对。 +- mtest 0.9.0 的超时直接结束程序;0.10.0 先记录进程、请求结束,宽限之后再结束。 + +### 19.4 实施中发现、单列的事项 + +- mcpp#968:GCC 16.1 在 `mcpp.pm.descriptor` 的接口单元带定义时于 `import mcpp.cli;` 段错误;以实现单元定义绕过(检查点 2)。 +- mcpp#969:`emit build-database` 的工作目录从不回收(一个 home 中 3.4 GB)。 +- clang 的两阶段调度下,依赖的缓存条目只在构建导入了包的每个模块时才被填充:BMI 由独立的边编译,目标文件不等它, + 无人导入的模块没有 BMI,填充被拒(mcpp 自身的 clang 构建,`openkal.time`)。填充改为按名字请求包的每个 BMI,并入检查点 10b。 +- 包实例(11.1)改变了 BMI 的拼写,缓存键随之改变:升级通道(`upgrade_lane.sh modules`)的“拼写未变时由缓存提供” + 以 2026.11.19.1 为最后一次拼写变化。 +- mcpp.plugins 0.22.1 以精确版本依赖 mcpplibs.os 0.4.4(openkal 0.24 至 0.25),与 openkal 0.26 一族不能共存于一个构建, + 而 openkal-platform 的构建依赖正是它。发布顺序因此为:mcpp.plugins 0.26.1(仅 mcpplibs.os 0.4.5,引擎下界不变)→ + openkal-platform 0.31.0 → mcpp 2026.11.19.1(缺省平台 0.31.0)→ mcpp.plugins 0.27.0(`steps`,引擎下界为该 mcpp)。 + 后者的引擎下界使两者不能合为一次发布。 diff --git a/.github/tools/upgrade_lane.sh b/.github/tools/upgrade_lane.sh index a0ec2db31..a6dbcfd84 100644 --- a/.github/tools/upgrade_lane.sh +++ b/.github/tools/upgrade_lane.sh @@ -256,13 +256,15 @@ fi if [ "$VARIANT" = modules ]; then # A BMI of another spelling is not read: the dependency is rebuilt once (its # cache key carries the spelling), then served from the cache. The spelling - # moved in 2026.10.26.2 (the module map's `$root`); a previous release from - # then on spells as this one, and its entry is served at once. + # moved in 2026.10.26.2 (the module map's `$root`) and in 2026.11.19.1 (a + # package instance's directory, mcpp#953); a previous release from the last + # move on spells as this one, and its entry is served at once. + spelled_since=2026.11.19.1 old_version=$("$OLD" --version | grep -Eo '[0-9]{4}\.[0-9]+\.[0-9]+\.[0-9]+' | head -1) - same_spelling=$(printf '%s\n2026.10.26.2\n' "$old_version" | sort -t. -k1,1n -k2,2n -k3,3n -k4,4n | head -1) + same_spelling=$(printf '%s\n%s\n' "$old_version" "$spelled_since" | sort -t. -k1,1n -k2,2n -k3,3n -k4,4n | head -1) modules_project m-new1 run new-modules1 "$W/m-new1" "$NEW" build -v || fail "the module project does not build over the previous release's cache" - if [ "$same_spelling" = 2026.10.26.2 ]; then + if [ "$same_spelling" = "$spelled_since" ]; then grep -q "STAGE gcm.cache/mcpplibs.openkal/" "$W/new-modules1.log" \ || fail "a dependency BMI of the same spelling ($old_version) was not served from the cache" else diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c336b0b2..51a35695b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,23 @@ and `mcpp clean --report` lists them. The root package's objects keep their place. A `@` name was measured and not taken: ninja quotes an output that holds `@`, and a rule writing `"$out.d"` then names a file with quotes in it. The objects of the - previous layout are not read again and go with `mcpp clean`. + previous layout are not read again and go with `mcpp clean`. The build cache keys a + dependency's BMIs by its instance too, so each is compiled once more after the upgrade and + served from the cache from then on. +- A compile writes its outputs below an attempt of its own, `/.mcpp-attempts//`, + and the executor publishes them by rename once the compiler has ended, a module's BMI the + moment it is complete, where the toolchain's outputs are relocatable: GCC 11 and later, and a + clang driver that takes `-dumpdir`, asked of the driver once (LLVM 17 and later; Apple clang 16 + and emcc 6.0.9 take it under numbers of their own), CodeView targets excepted (design 2026-11-19 §10.6, + SPEC-011 §5.1). A compiler that outlives its build writes only its own attempt: the next build + continues at once, and the attempts the interrupted build did not publish are discarded without + being observed. `compiler-run-occupied` remains for the toolchains that write in place. Measured + first: a GCC BMI written where its module map names it and then renamed is read by its importers, + and records its imports, not itself; an object's bytes do not depend on its path on ELF, Mach-O + and wasm, and do on CodeView, at `-O2` without `-g`. A GCC module unit of the root package is + given a map too, which names each module at its name. The files named after the object (`.dwo`, + `.gcno`) keep their names through `-dumpdir` and `-dumpbase`; clang's are named + `..`. ### Fixed @@ -103,8 +119,9 @@ declares is an input (SPEC-004 §8.1). Where the linker writes what it read (`--dependency-file`: GNU ld from 2.35, gold, ld.lld), the link edge records it as a compile edge records the compiler's report, so a library found through `-L`/`-l` relinks - the program when it changes. Whether a linker accepts the option is measured once and kept - in the probe cache. + the program when it changes, and one that is gone links again and is named by the linker. As + for a compile, a program changed outside the build is linked again. Whether a linker accepts + the option is measured once and kept in the probe cache. - Every member of a workspace whose root states `[workspace.target..abi]` was warned that it declares `[target..abi]`, which only the root decides: a build of about 60 members for wasm32-emscripten printed the warning 262 times (mcpp#954). A @@ -119,6 +136,11 @@ the feature active (SPEC-007 R6.3), so a multi-step action is a program the plugin carries. A tool no edge and no feature declares is reported (`build-program/undeclared-tool`) with the line that declares it. +- Under clang's two-phase schedule a dependency's cache entry was filled only when the build + imported every module of the package: the BMI is compiled on an edge of its own, which the + object does not wait for, so a module no unit imported had no BMI, and the fill was refused + (`bmi cache populate failed for openkal@0.26.0: expected build output missing: + .../openkal.time.pcm` in mcpp's own build). A fill asks for every BMI of its package. ## [2026.11.18.1] - 2026-11-18 diff --git a/docs/04-mcpp-toml.md b/docs/04-mcpp-toml.md index f145ad41b..dae71dc9b 100644 --- a/docs/04-mcpp-toml.md +++ b/docs/04-mcpp-toml.md @@ -1108,8 +1108,27 @@ Cancelling the build ends its ordinary and detached compilers. An immediately following build waits for their cleanup before starting another compiler; partial outputs are rebuilt. A missing object from a completed module is recreated by that build. An unchanged build still performs no compilation. -If a forcibly ended build leaves a compiler unit whose completion cannot be -verified, the next build waits for it within a bound (2026.11.19.1+): + +A compile writes its outputs below an attempt of its own, +`/.mcpp-attempts//`, and the executor publishes them by +rename once the compiler has ended, a module's BMI the moment it is complete, +where the toolchain's outputs are relocatable: GCC 11 and later, and a clang +driver that takes `-dumpdir`, which mcpp asks of the driver once (LLVM 17 and +later; Apple clang 16 and emcc 6.0.9 take it under numbers of their own), +except for CodeView targets (`*-windows-msvc`), whose objects record their own +path (2026.11.19.1+). An object or a BMI equal to +the published one leaves the published file as it was. A compiler that +outlives its build (the build killed, its container or sandbox gone) writes +only its own attempt, and the next build continues at once: the attempts the +interrupted build did not publish are discarded without being observed. The +auxiliary files a compiler names after the object (`.dwo`, `.gcno`) keep their +names and are written in place; clang names them +`..`. + +Where a toolchain writes in place (cl.exe, clang for `*-windows-msvc`, NASM and +the resource compilers), a forcibly ended build that leaves a compiler unit +whose completion cannot be verified makes the next build wait for it within a +bound (2026.11.19.1+): - a unit that is empty is recovered, as before; - a unit that is occupied, or not yet decidable, makes the build refuse with diff --git a/docs/specs/concurrency.md b/docs/specs/concurrency.md index bbf7cfcfb..52afd6e72 100644 --- a/docs/specs/concurrency.md +++ b/docs/specs/concurrency.md @@ -4,10 +4,10 @@ |---|---| | 规范编号 | SPEC-011 | | 标题 | 同一工程上的并发运行:构建目录锁、原子发布、乐观写入与取消 | -| 状态 | 评审中 v1.1 | -| 版本 | 1.1 | -| 最后修改 | 2026-11-06 | -| 对应实现 | 基础保证: mcpp >= 2026.10.26.1; §5.1: mcpp >= 2026.11.7.1 | +| 状态 | 评审中 v1.2 | +| 版本 | 1.2 | +| 最后修改 | 2026-11-19 | +| 对应实现 | 基础保证: mcpp >= 2026.10.26.1; §5.1: mcpp >= 2026.11.7.1;§5.1 第 6 条的不可观察单元与第 8 至 10 条: mcpp >= 2026.11.19.1 | | 相关设计文档 | `.agents/docs/2026-10-26-control-contract-runtime-and-ide-ecosystem-design.md` §3、§4、§5.4 | | 相关 issue | #839 | | 使用文档 | [50 — 机器可读输出](../zh/50-machine-output.md) §9、§11 | @@ -88,7 +88,8 @@ CI。在 2026.10.24.1 上,同一配置的两次冷构建相隔 0.1–0.5 s 启 ### 5.1 分离编译的执行单元与恢复 -以下保证适用于 GCC 的 `bmi_schedule = "on"`(2026.11.7.1+): +以下保证适用于经编译执行器运行的编译(带编译上限的构建图,2026.11.7.1+),GCC 的 `bmi_schedule = "on"` +的分离代码生成在其中: 1. 普通编译与分离代码生成**必须**共用实际的 `jobs` 总上限。等待任务与调度任务 **禁止**作为编译名额的替代。内核锁已释放而原编译单元仍被占用或状态未知时, @@ -106,13 +107,36 @@ CI。在 2026.10.24.1 上,同一配置的两次冷构建相隔 0.1–0.5 s 启 读取 BMI 之前失效对应 BMI,使该次构建正常重建。损坏证书、目录符号链接或 已重定向的 BMI 父目录**禁止**授权删除其他输出。损坏记录、非法阶段顺序或 写入失败**禁止**猜测成功。 -6. 原单元被占用或不能被确认时,恢复**必须**在有限等待后明确失败并保留名额及证据。 - 所有原生引用持有者在认证完成之前都被强制结束时,平台可能只能报告未知; - 此边界**不保证**自动恢复,**禁止**把名字消失或数值标识复用解释为空单元。 +6. 原单元被占用或暂时不能判定时,恢复**必须**在有限等待后明确失败并保留名额及证据; + 拒绝**必须**写出这类单元的个数、两种状态各自的个数、状态目录与出路(原因标识 + `compiler-run-occupied`)。所有原生引用持有者在认证完成之前都被强制结束时,平台可能只能 + 报告未知;此边界**不保证**自动恢复,**禁止**把名字消失或数值标识复用解释为空单元。 + 原单元从当前进程空间不可观察(引用产生于另一个进程空间,或系统未保留原生标识;openkal + 0.26 条款 7.18)而旧代的所有者锁与全部 actor 锁空闲时,恢复**必须**把该尝试作为被放弃的 + 尝试处理,丢弃其未发布的输出,并经 `degraded`(`compiler-run/unobservable`)报告一次。 + 这些锁属于文件系统而不属于进程空间,在共享该目录的每个进程空间中同样成立。 7. 本节的恢复范围是文件系统持续工作时的进程异常退出;**不保证**断电或存储服务 故障后的持久性。运行记录**必须**完整写入并检查关闭结果后才原子替换最终名字, 确认消息**禁止**先于该替换。OS 库的通用持久化写入接口保持其既有语义。 +8. 输出可重定向的工具链(工具链模型的 `OutputRelocation`:GCC 11 起,以及探测时接受 + `-dumpdir` 的 clang 驱动;CodeView 目标除外,其目标文件记录自己的路径)上,编译**必须** + 只写该次尝试私有的路径(`<构建目录>/.mcpp-attempts/<尝试>/`),由执行器在编译器结束后以 + 改名发布,模块的 BMI 在完整时发布;与已发布文件相同的目标文件与 BMI **必须**保留已发布者。 + 依赖文件以发布后的名字发布。目标文件记录其名字的附属输出(`.dwo`、`.gcno`)以编译器的 + 映射选项(`-dumpdir`、`-dumpbase`)取最终名字,就地写出。一次尝试在其编译器被准入之前 + **必须**记录为已重定向;恢复对这样的尝试**禁止**等待观察,直接丢弃它未发布的文件并使它 + 已发布而未完成的输出失效,第 6 条的等待、拒绝与报告不适用。一次编译的输出只能全部重定向 + 或全部就地写出。 +9. `mcpp clean`(含 `--stale`、`--older-than`)删除一个配置的输出目录之前**必须**在构建所用的 + 闸锁之下退役该配置的各代运行状态;所有者锁或 actor 锁被持有时该配置**禁止**删除,消息 + **必须**写出持有它的运行。 +10. 所有者拒绝一次准入时**必须**写出条件(`outside-generation`、`record-unreadable`、 + `actor-outside`、`actor-not-held`、`controller-ended`、`unit-unobservable`、 + `owner-stopping`);`record-unreadable` 在启动宽限内重试,其余为最终结果。执行器以编译器 + 状态与取消以外的任何原因结束时**必须**写出结束它的步骤与原因。环境不能声明执行单元时 + (`units_observable()` 为假),整次构建的编译不声明单元,仍受同一编译上限约束,并经 + `degraded`(`compiler-run/no-unit`)报告一次。 ## 6. 全局一侧 @@ -128,3 +152,4 @@ CI。在 2026.10.24.1 上,同一配置的两次冷构建相隔 0.1–0.5 s 启 |---|---|---| | 1.0 | 2026-10-26 | 首版:三段、构建目录锁、原子发布、`mcpp.lock` 的乐观写入、取消(#839) | | 1.1 | 2026-11-06 | 分离编译的单元观察、共享名额、绝对输出事务与有限恢复边界;对齐取消请求的 OS 接收方式。 | +| 1.2 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#952,设计 2026-11-19 §10):§5.1 第 6 条不可观察单元的恢复与报告、拒绝写出个数与出路;第 8 条输出可重定向的工具链上编译写入尝试私有的路径并由执行器发布;第 9 条清理退役运行状态;第 10 条准入的条件、执行器的结束原因与不能声明单元的环境。 | diff --git a/docs/zh/04-mcpp-toml.md b/docs/zh/04-mcpp-toml.md index a7ccdf34f..3eb50f653 100644 --- a/docs/zh/04-mcpp-toml.md +++ b/docs/zh/04-mcpp-toml.md @@ -1015,7 +1015,17 @@ bmi_schedule = "off" # auto (default, = off) | on | off 取消构建会结束本轮普通编译器和分离的编译器。紧接着的构建等待它们完成清理后 才启动新的编译器;未完成的输出会重新构建。已完成模块的 object 缺失时, 该次构建会重新生成它。输入和输出均不变的构建仍不进行编译。 -若被强制结束的构建留下了无法确认已经完成的编译单元,下一次构建在一个界限内等待它(2026.11.19.1+): + +工具链的输出可重定向时(GCC 11 起,以及接受 `-dumpdir` 的 clang 驱动,由 mcpp 向驱动询问一次:LLVM 17 起, +Apple clang 16 与 emcc 6.0.9 以各自的编号接受它;CodeView 目标 `*-windows-msvc` 除外,其目标文件记录自己的路径),一次编译把输出写在该次尝试自己的目录 +`<构建目录>/.mcpp-attempts/<尝试>/` 下,由执行器在编译器结束后以改名发布,模块的 BMI 在完整时发布 +(2026.11.19.1+)。与已发布者相同的目标文件或 BMI 保留已发布的文件。比其构建活得更久的编译器(构建被结束, +其容器或沙箱已不存在)只写自己的尝试,下一次构建立即继续:被中断的构建未发布的尝试不经观察即被丢弃。 +编译器以目标文件命名的附属文件(`.dwo`、`.gcno`)保持其名字并就地写出;clang 将其命名为 +`<目标文件主名>.<源文件主名>.<扩展名>`。 + +工具链就地写出时(cl.exe、面向 `*-windows-msvc` 的 clang、NASM 与资源编译器),若被强制结束的构建留下了 +无法确认已经完成的编译单元,下一次构建在一个界限内等待它(2026.11.19.1+): - 单元为空:照常恢复; - 单元被占用或暂时无法判定:构建拒绝,写出这类单元的个数、状态目录与出路(`[compiler-run-occupied]`): diff --git a/modules/buildopt/src/keep/keep.cppm b/modules/buildopt/src/keep/keep.cppm index a4cd896d9..95fec6b0a 100644 --- a/modules/buildopt/src/keep/keep.cppm +++ b/modules/buildopt/src/keep/keep.cppm @@ -53,14 +53,18 @@ inline constexpr std::string_view kObjectsFile = "objects-same.list"; // What an edge asks after compiling an interface: whether the BMI before it // stays. Reads the cutoff and records the difference in the current -// directory. -bool keeps_previous(const std::filesystem::path& bmi, const std::filesystem::path& previous); +// directory. `recorded`, when given, is the name the record states: an +// attempt that wrote its BMI at a path of its own compares that file with the +// published one and records the published name (design 2026-11-19 §10.6). +bool keeps_previous(const std::filesystem::path& bmi, const std::filesystem::path& previous, + const std::filesystem::path& recorded = {}); // What an edge asks after compiling an object: whether the object before it // stays. The bytes decide and nothing else does -- an object has no reader and // no cutoff, and one that differs in a single byte is linked. Records an // object that stays in the current directory. -bool keeps_previous_object(const std::filesystem::path& object, const std::filesystem::path& previous); +bool keeps_previous_object(const std::filesystem::path& object, const std::filesystem::path& previous, + const std::filesystem::path& recorded = {}); // Before a run of the graph in `dir`: states the cutoff, starts the record of // changes, and, where the cutoff is exact, removes every BMI an earlier run @@ -106,11 +110,12 @@ Difference difference(const std::filesystem::path& a, const std::filesystem::pat return same_bytes(a, b) ? Difference::Same : Difference::Changed; } -bool keeps_previous_object(const std::filesystem::path& object, const std::filesystem::path& previous) { +bool keeps_previous_object(const std::filesystem::path& object, const std::filesystem::path& previous, + const std::filesystem::path& recorded) { // The standard build (MCPP_BUILDOPT=off): what the compiler wrote stands. if (registry::standard_only()) return false; if (!same_bytes(object, previous)) return false; - const auto line = object.generic_string() + "\n"; + const auto line = (recorded.empty() ? object : recorded).generic_string() + "\n"; std::ofstream out(std::filesystem::path{kObjectsFile}, std::ios::app | std::ios::binary); out.write(line.data(), static_cast(line.size())); return true; @@ -126,7 +131,8 @@ std::vector read_lines(const std::filesystem::path& p) { } } // namespace -bool keeps_previous(const std::filesystem::path& bmi, const std::filesystem::path& previous) { +bool keeps_previous(const std::filesystem::path& bmi, const std::filesystem::path& previous, + const std::filesystem::path& recorded) { if (registry::standard_only()) return false; const auto found = keep::difference(bmi, previous); if (found == Difference::Same) return true; @@ -136,7 +142,7 @@ bool keeps_previous(const std::filesystem::path& bmi, const std::filesystem::pat keep = !cutoff.empty() && cutoff.front() == kIgnoreLocations; } // One write of one line, appended: the edges of a run do not interleave. - const auto line = std::format("{}\t{}\t{}\n", bmi.generic_string(), + const auto line = std::format("{}\t{}\t{}\n", (recorded.empty() ? bmi : recorded).generic_string(), found == Difference::LocationsOnly ? "locations" : "changed", keep ? "kept" : "replaced"); std::ofstream out(std::filesystem::path{kChangesFile}, std::ios::app | std::ios::binary); out.write(line.data(), static_cast(line.size())); diff --git a/modules/toolchain-model/src/model.cppm b/modules/toolchain-model/src/model.cppm index 2a4e3a748..a227c65d5 100644 --- a/modules/toolchain-model/src/model.cppm +++ b/modules/toolchain-model/src/model.cppm @@ -243,6 +243,12 @@ struct Toolchain { // Why no std module was found, when the compiler did not answer the // question (its probe failed); empty when it answered that it has none. std::string stdModuleUnanswered; + // Whether the clang driver takes `-dumpdir` and names a compile's + // auxiliary outputs from it, asked of the driver at detection + // (`mcpp::toolchain::driver_takes_dumpdir`). A driver's own number does + // not say: Apple clang 16 and every emcc number were measured to take it, + // and LLVM before 17 refuses the word. Read by `output_relocation`. + bool takesDumpdir = false; // Something about HOW this toolchain was resolved that the user has to be // told, but which is not a failure. Non-empty ⇒ the caller MUST surface it. @@ -516,6 +522,39 @@ BmiTraits bmi_traits(const Toolchain& tc); // units without module flags. bool compiles_modules(const Toolchain& tc); +// WHETHER A COMPILE MAY WRITE ITS OUTPUTS AT PATHS OF ITS OWN ATTEMPT, which +// the compiler executor then publishes by rename (design 2026-11-19 §10.6). +// A compiler that writes the published path directly lets one that outlives +// its build rewrite an output the next build trusts; one that writes paths of +// its own attempt cannot, and an interrupted build is recovered by discarding +// what it did not publish. Three premises, measured on GCC 16.1 and clang +// 23.1.3 (2026-11-19): +// +// P1 a GCC BMI written where its compile's module map names it, then +// renamed, is read by its importers; a BMI records the paths of its +// imports as the map spells them, and not its own; +// P2 clang's `-fmodule-output=` and `-o` likewise; an object's bytes do not +// depend on its path on ELF, Mach-O and wasm (with and without `-g`); +// P3 the auxiliary outputs named after the object (`.dwo`, `.gcno`) keep +// their names when the compile is told the object's final base name +// (`auxNaming`); no prefix map rewrites the name a skeleton records. +// +// An object of a CodeView target names its own path (clang for +// `*-windows-msvc` at `-O2` without `-g`, measured; cl.exe the same by its +// `S_OBJNAME`), so its bytes depend on where it was written: those compilers +// write in place, and an interrupted build is recovered by observation. So +// does a clang driver that refuses `-dumpdir` (`Toolchain::takesDumpdir`), +// whose auxiliary outputs would be named after the attempt. +struct OutputRelocation { + bool relocatable = false; + // How the executor tells the compile its object's final base name: + // "dumpbase" adds `-dumpdir / -dumpbase ` (GCC 11+, the names + // GCC derives from `-o`); "dumpdir" adds `-dumpdir /.` (clang, + // which then names an auxiliary output `..`). + std::string_view auxNaming; +}; +OutputRelocation output_relocation(const Toolchain& tc); + } // namespace mcpp::toolchain namespace mcpp::toolchain { @@ -635,6 +674,17 @@ bool target_supports_full_static(std::string_view targetTriple, bool hostCapabil return t->os == "linux" || t->c_library_by_platform(); } +OutputRelocation output_relocation(const Toolchain& tc) { + if (tc.compiler == CompilerId::MSVC || is_msvc_target(tc)) return {}; + // GCC's number is its own: `-dumpbase` and `-dumpdir` name the auxiliary + // outputs as in place from GCC 11 (its auxiliary-output naming change). + if (is_gcc(tc)) { + const int major = std::atoi(tc.version.c_str()); + return major == 0 || major >= 11 ? OutputRelocation{true, "dumpbase"} : OutputRelocation{}; + } + return is_clang(tc) && tc.takesDumpdir ? OutputRelocation{true, "dumpdir"} : OutputRelocation{}; +} + bool compiles_modules(const Toolchain& tc) { if (tc.compiler != CompilerId::GCC) return true; const int major = std::atoi(tc.version.c_str()); diff --git a/modules/toolchain-model/tests/test_output_relocation.cpp b/modules/toolchain-model/tests/test_output_relocation.cpp new file mode 100644 index 000000000..77858a9e6 --- /dev/null +++ b/modules/toolchain-model/tests/test_output_relocation.cpp @@ -0,0 +1,50 @@ +import std; +import mcpplibs.mtest; +import mcpp.toolchain.model; + +using namespace mcpplibs::mtest; + +// Whether a compile writes its outputs below an attempt of its own (design +// 2026-11-19 §10.6) is a fact of the toolchain: GCC's by its own number, a +// clang driver's by what it answered at detection, and never a CodeView +// target's, whose objects record their path. + +namespace { +mcpp::toolchain::Toolchain toolchain(mcpp::toolchain::CompilerId id, std::string version, + std::string triple = "x86_64-linux-gnu") { + mcpp::toolchain::Toolchain tc; + tc.compiler = id; + tc.version = std::move(version); + tc.targetTriple = std::move(triple); + return tc; +} +} // namespace + +UTest gcc_relocates_from_its_auxiliary_naming_change{"OutputRelocation/GccRelocatesFromItsAuxiliaryNamingChange", [] { + using mcpp::toolchain::CompilerId; + const auto gcc16 = mcpp::toolchain::output_relocation(toolchain(CompilerId::GCC, "16.1.0")); + expect(gcc16.relocatable); + expect(eq(gcc16.auxNaming, std::string_view("dumpbase"))); + expect(!mcpp::toolchain::output_relocation(toolchain(CompilerId::GCC, "10.5.0")).relocatable); +}}; + +UTest a_clang_driver_relocates_when_it_takes_dumpdir{"OutputRelocation/AClangDriverRelocatesWhenItTakesDumpdir", [] { + using mcpp::toolchain::CompilerId; + // The driver's number does not decide: a driver numbered 6 (emcc 6.0.9) + // takes the word, and the answer is what was asked of it. + auto emcc = toolchain(CompilerId::Clang, "6.0.9", "wasm32-unknown-emscripten"); + expect(!mcpp::toolchain::output_relocation(emcc).relocatable); + emcc.takesDumpdir = true; + const auto relocated = mcpp::toolchain::output_relocation(emcc); + expect(relocated.relocatable); + expect(eq(relocated.auxNaming, std::string_view("dumpdir"))); +}}; + +UTest a_codeview_target_writes_in_place{"OutputRelocation/ACodeViewTargetWritesInPlace", [] { + using mcpp::toolchain::CompilerId; + auto clang = toolchain(CompilerId::Clang, "23.1.3", "x86_64-pc-windows-msvc"); + clang.takesDumpdir = true; + expect(!mcpp::toolchain::output_relocation(clang).relocatable); + expect(!mcpp::toolchain::output_relocation(toolchain(CompilerId::MSVC, "19.44", "x86_64-pc-windows-msvc")) + .relocatable); +}}; diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 588a4c736..48c26ec90 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -1644,6 +1644,22 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, const std::string compilerPrefix = plan.scheduleCompilerCap > 0 ? std::string("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d -- ") : std::string{}; + // A COMPILE WRITES PATHS OF ITS OWN ATTEMPT where the toolchain's outputs + // are relocatable, and the executor publishes them (design 2026-11-19 + // §10.6): a compiler that outlives its build cannot rewrite a published + // output. The C and C++ drivers, their assembler, and the dependency scans; + // the NASM and resource compilers write in place, not having been measured. + const auto relocation = mcpp::toolchain::output_relocation(plan.toolchain); + const std::string relocateWord = relocation.relocatable + ? std::format(" --relocate {}", relocation.auxNaming.empty() ? std::string_view("none") : relocation.auxNaming) + : std::string{}; + const std::string driverPrefix = plan.scheduleCompilerCap > 0 + ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{} -- ", relocateWord) + : std::string{}; + const std::string scanPrefix = plan.scheduleCompilerCap > 0 + ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{} -- ", + relocation.relocatable ? " --relocate none" : "") + : std::string{}; // The same executor for an edge that writes a BMI: it holds the previous // BMI while the compiler runs and puts it back when the new one is // equivalent (`mcpp.buildopt.keep`), which is what lets `restat` end the rebuild at @@ -1658,11 +1674,12 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, auto bmi_keeping_prefix = [&](std::string_view bmi) { return plan.scheduleCompilerCap > 0 ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{}" - " --bmi \"{}\" -- ", bmi == "$out" ? "" : " --object", bmi) + " --bmi \"{}\"{} -- ", bmi == "$out" ? "" : " --object", bmi, relocateWord) : std::string{}; }; const std::string objectPrefix = plan.scheduleCompilerCap > 0 - ? std::string("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d --object -- ") + ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d --object{} -- ", + relocateWord) : std::string{}; // The other split shape. Clang publishes no BMI early — it writes the BMI // at the end of the compile — so the GCC trick of releasing the file @@ -2302,9 +2319,9 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // rules — the assumption "one flag element == one argv token" has been // wrong in this file before. append("rule cxx_module_bmi\n"); - append(" command = $mcpp bmi-compile --bmi $bmi_out --slot $slot" + append(std::format(" command = $mcpp bmi-compile --bmi $bmi_out --slot $slot" " --self $mcpp --sem $sched_sem --object $obj_out" - " --command-file $out.cmd --dep-from $scan_dep --dep-to $out.d\n"); + " --command-file $out.cmd --dep-from $scan_dep --dep-to $out.d{}\n", relocateWord)); append(std::format( " rspfile = $out.cmd\n" " rspfile_content = $cxx $local_includes $cxxflags $unit_cxxflags{}{} {} $in {}$obj_out\n", @@ -2439,7 +2456,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, const std::string payload = " $local_includes"; append("rule asm_object\n"); // .S — preprocessed, tracks #include append(std::format(" command = {}$cc{} $asmflags $unit_asmflags {}{}\n", - compilerPrefix, rsp_ref(payload), c_mmd_flag, compile_tail)); + driverPrefix, rsp_ref(payload), c_mmd_flag, compile_tail)); append_rspfile(payload); append(" description = AS $out\n"); append_cxx_deps(); @@ -2447,7 +2464,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, append("rule asm_object_raw\n"); // .s — not preprocessed, no depfile append(std::format(" command = {}$cc{} $asmflags $unit_asmflags {}\n", - compilerPrefix, rsp_ref(payload), compile_tail)); + driverPrefix, rsp_ref(payload), compile_tail)); append_rspfile(payload); append(" description = AS $out\n\n"); } @@ -2791,7 +2808,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // only — no codegen); /TP because our module units are .cppm. append(std::format(" command = {}$cxx_driver{} $cxxflags $unit_cxxflags " "/scanDependencies $out /TP /c $in /Fo:$compile_target\n", - compilerPrefix, rsp_ref(scanPayload))); + scanPrefix, rsp_ref(scanPayload))); // No $unit_lang here: /TP above already applies to every input, // and cl has no per-suffix recognition problem to fix — a module // interface is identified by /interface at COMPILE time, which @@ -2810,7 +2827,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, "-fdeps-format=p1689r5 " "-fdeps-file=$out -fdeps-target=$deps_target " "-M -MM -MF $out.dep $unit_lang -E $in -o $out.scan\n", - compilerPrefix, rsp_ref(scanPayload))); + scanPrefix, rsp_ref(scanPayload))); } else { // Clang path: clang-scan-deps writes the P1689 JSON itself via -o // (LLVM 17+), like the GCC and MSVC branches above write theirs @@ -2823,7 +2840,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, " command = {}$scan_deps -format=p1689 -o $out -- " "$cxx_driver{} $cxxflags $unit_cxxflags $unit_lang -c $in " "-o $compile_target\n", - compilerPrefix, rsp_ref(scanPayload))); + scanPrefix, rsp_ref(scanPayload))); } append_rspfile(scanPayload); append(" description = SCAN $out\n\n"); @@ -4296,10 +4313,18 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // (for example a C ABI provider). Once cache fills or build actions emit // an explicit `default`, ninja no longer selects the other terminal edges. // A cache fill must also finish even when no link or importer asks for it: - // the entry is its package's every artifact or nothing. - for (auto const& cu : plan.compileUnits) - if (plan.linkUnits.empty() || (cu.fillsCache && !cu.servedFromCache)) + // the entry is its package's every artifact or nothing. A two-phase module + // unit writes its BMI on an edge of its own that its object does not wait + // for, so the BMI is asked for by name: a module no unit of the build + // imports was otherwise never compiled to a BMI, and the fill was refused + // for it (`openkal.time` in mcpp's own clang build). + for (auto const& cu : plan.compileUnits) { + const bool fills = cu.fillsCache && !cu.servedFromCache; + if (plan.linkUnits.empty() || fills) actionDefaults += " " + escape_ninja_path(cu.object); + if (fills && !cu.providesModule.empty()) + actionDefaults += " " + unit_bmi(cu); + } if (!plan.linkUnits.empty() || !actionDefaults.empty()) { std::string defaults; diff --git a/src/build/plan.cppm b/src/build/plan.cppm index 93c963d7f..6b6ec4420 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -944,6 +944,17 @@ std::expected refuse_unresolved_references( const std::map>& ambiguousTargets, const std::set& unresolvedArtifacts); +// The references an action's arguments made that its role or this run cannot +// answer: `${mcpp.stage_dir}` where no tree is staged (`stageDirNoPass`; the +// package several packed members reach is `stageDirShared`, the stage's own +// reason `stageDirWhy`), and `${mcpp.stage_dir}` or `${mcpp.objects:}` +// used by a role that runs before what it names. Each set holds action ids. +struct ActionRoleReferences { + std::set stageDirNoPass, stageDirWrongRole, objectsWrongRole; + std::string stageDirWhy, stageDirShared; +}; +std::expected refuse_action_roles(const ActionRoleReferences& r); + // The objects a package contributes to an image that links it whole: its // module units, which link unconditionally, then its implementation units, in // plan order. A shared library that carries a private copy of a graph C++ @@ -4784,6 +4795,11 @@ make_plan(const mcpp::manifest::Manifest& manifest, cu.bmiFile = bmi_file(cu.packageName, cu.providesModule); const bool gcc = traits.moduleFileUsePrefix.empty(); + // A relocated GCC compile names its own BMI through a copy of its map + // (design 2026-11-19 §10.6), and GCC's mapper answers only what it + // lists: a unit that provides a module is given a map wherever its BMI + // lies, the root's included. + const bool mapsOwn = gcc && mcpp::toolchain::output_relocation(tc).relocatable; std::string_view prefix = traits.moduleFileUsePrefix; while (!prefix.empty() && prefix.front() == ' ') prefix.remove_prefix(1); const bool separate = !prefix.empty() && prefix.back() == ' '; @@ -4818,7 +4834,8 @@ make_plan(const mcpp::manifest::Manifest& manifest, const bool importsQualified = std::ranges::any_of(reach, [&](auto const& e) { return !e.second.first.empty() && e.second.first != rootPackage; }); - if (!ownQualified && !importsQualified) continue; // every BMI at its name + if (!ownQualified && !importsQualified && !(mapsOwn && !cu.providesModule.empty())) + continue; // every BMI at its name std::map> entries; // name -> BMI path for (auto const& [name, at] : reach) entries.emplace(name, at.second); @@ -5043,6 +5060,67 @@ TargetFileAnswer resolve_target_file(const BuildPlan& plan, std::string_view nam return answer; } +std::expected refuse_action_roles(const ActionRoleReferences& r) { + if (!r.stageDirNoPass.empty()) { + std::string ids; + for (auto const& n : r.stageDirNoPass) ids += (ids.empty() ? "" : ", ") + n; + if (!r.stageDirShared.empty()) + return std::unexpected(std::format( + "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}}, and {}.\n" + " A staged tree belongs to one member, and one run of this package's " + "program serves all of them.\n" + " use: provide the format from each member's own build program, or pack " + "the members one at a time", ids, r.stageDirShared)); + if (!r.stageDirWhy.empty()) { + return std::unexpected(std::format( + "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}}, and no " + "tree could be staged for this target.\n" + " {}\n" + " The format was requested and the provider was reached; what is " + "missing is the staged\n" + " closure itself. A member that names a built file with " + "${{mcpp.target_file:}} instead\n" + " of reading the tree is unaffected on this target.", + ids, r.stageDirWhy)); + } + return std::unexpected(std::format( + "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}}, and this " + "build is not packaging.\n" + " The staged tree is produced by `mcpp pack` after the link, so " + "it does not exist during\n" + " a plain build and there is nothing for the placeholder to name.\n" + " Gate the submission on the format you provide:\n" + " mcpp::provides_pack_format(\"\"); // always\n" + " if (std::string_view(mcpp::pack_format()) == \"\") " + "// then submit\n" + " and reach the tree with `mcpp pack --format `.", ids)); + } + if (!r.stageDirWrongRole.empty()) { + std::string ids; + for (auto const& n : r.stageDirWrongRole) ids += (ids.empty() ? "" : ", ") + n; + return std::unexpected(std::format( + "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}} with a role " + "other than \"artifact\".\n" + " Only an artifact action runs after the link, and the staged tree " + "is a link output's\n" + " successor: a source, object or check action is scheduled before " + "there is anything to stage.\n" + " use: role = \"artifact\"", ids)); + } + if (!r.objectsWrongRole.empty()) { + std::string ids; + for (auto const& n : r.objectsWrongRole) ids += (ids.empty() ? "" : ", ") + n; + return std::unexpected(std::format( + "build.mcpp action(s) [{}] reference ${{mcpp.objects:}} or " + "${{mcpp.objects_rsp:}} with a role other than \"artifact\" or \"check\".\n" + " The list names the objects a link takes; a source, object or prepare action " + "runs before\n" + " that link or feeds it, so the list it would read is not yet decided.\n" + " use: role = \"artifact\" (or \"check\")", ids)); + } + return {}; +} + std::expected refuse_unresolved_references( const BuildPlan& plan, const std::set& unresolvedTargets, const std::map>& ambiguousTargets, diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index ec7bb69f3..43a0b3440 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -1355,63 +1355,10 @@ static std::expected step13_build_graph_actions(PrepareState& std::ranges::unique(ctx.plan.providedPackFormats).begin(), ctx.plan.providedPackFormats.end()); ctx.plan.packFormat = state.overrides.pack_format; - if (!stageDirNoPass.empty()) { - std::string ids; - for (auto const& n : stageDirNoPass) ids += (ids.empty() ? "" : ", ") + n; - if (!stageDirShared.empty()) - return std::unexpected(std::format( - "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}}, and {}.\n" - " A staged tree belongs to one member, and one run of this package's " - "program serves all of them.\n" - " use: provide the format from each member's own build program, or pack " - "the members one at a time", ids, stageDirShared)); - if (!stageDirWhy.empty()) { - return std::unexpected(std::format( - "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}}, and no " - "tree could be staged for this target.\n" - " {}\n" - " The format was requested and the provider was reached; what is " - "missing is the staged\n" - " closure itself. A member that names a built file with " - "${{mcpp.target_file:}} instead\n" - " of reading the tree is unaffected on this target.", - ids, stageDirWhy)); - } - return std::unexpected(std::format( - "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}}, and this " - "build is not packaging.\n" - " The staged tree is produced by `mcpp pack` after the link, so " - "it does not exist during\n" - " a plain build and there is nothing for the placeholder to name.\n" - " Gate the submission on the format you provide:\n" - " mcpp::provides_pack_format(\"\"); // always\n" - " if (std::string_view(mcpp::pack_format()) == \"\") " - "// then submit\n" - " and reach the tree with `mcpp pack --format `.", ids)); - } - if (!stageDirWrongRole.empty()) { - std::string ids; - for (auto const& n : stageDirWrongRole) ids += (ids.empty() ? "" : ", ") + n; - return std::unexpected(std::format( - "build.mcpp action(s) [{}] reference ${{mcpp.stage_dir}} with a role " - "other than \"artifact\".\n" - " Only an artifact action runs after the link, and the staged tree " - "is a link output's\n" - " successor: a source, object or check action is scheduled before " - "there is anything to stage.\n" - " use: role = \"artifact\"", ids)); - } - if (!objectsWrongRole.empty()) { - std::string ids; - for (auto const& n : objectsWrongRole) ids += (ids.empty() ? "" : ", ") + n; - return std::unexpected(std::format( - "build.mcpp action(s) [{}] reference ${{mcpp.objects:}} or " - "${{mcpp.objects_rsp:}} with a role other than \"artifact\" or \"check\".\n" - " The list names the objects a link takes; a source, object or prepare action " - "runs before\n" - " that link or feeds it, so the list it would read is not yet decided.\n" - " use: role = \"artifact\" (or \"check\")", ids)); - } + if (auto refused = mcpp::build::refuse_action_roles( + {stageDirNoPass, stageDirWrongRole, objectsWrongRole, stageDirWhy, stageDirShared}); + !refused) + return std::unexpected(refused.error()); if (auto refused = mcpp::build::refuse_unresolved_references( ctx.plan, unresolvedTargets, ambiguousTargets, unresolvedArtifacts); !refused) diff --git a/src/build/schedule/compiler_transaction.cppm b/src/build/schedule/compiler_transaction.cppm index 9b275d8ce..67adc8257 100644 --- a/src/build/schedule/compiler_transaction.cppm +++ b/src/build/schedule/compiler_transaction.cppm @@ -240,6 +240,19 @@ struct Attempt { return {}; } bool publication() const { return publish(directory/"published","mcpp-publication-v1",{}); } + // THE ATTEMPT WRITES PATHS OF ITS OWN (design 2026-11-19 §10.6), stated + // before its compiler is admitted: `directory` holds what it writes, and + // nothing it did not publish is where a build reads it. + bool relocate(const fs::path& outputs) const { + return publish(directory/"relocated","mcpp-relocated-v1",{spelling(outputs)}); + } + std::expected,std::string> relocation() const { + auto record=read(directory/"relocated","mcpp-relocated-v1"); + if(!record)return std::unexpected(record.error()); + if(!*record)return std::optional{}; + if((**record).size()!=1||!named((**record)[0]).is_absolute())return std::unexpected("invalid compiler relocation record"); + return std::optional{named((**record)[0])}; + } bool result(int rc) const { return publish(directory/"result","mcpp-result-v1",{std::to_string(rc)}); } bool retired(int rc) const { auto output=identity(object);if(!output)output=Fields{"missing","missing"}; @@ -266,7 +279,7 @@ std::expected validate(const Attempt& attempt) { Shape{"controller","mcpp-controller-v1",3}, Shape{"reservation","mcpp-reservation-pointer-v1",1}, Shape{"admitted","mcpp-admitted-v1",0}, Shape{"published","mcpp-publication-v1",0}, Shape{"retired","mcpp-retired-v1",3}, - Shape{"result","mcpp-result-v1",1}}) { + Shape{"result","mcpp-result-v1",1}, Shape{"relocated","mcpp-relocated-v1",1}}) { auto record=read(attempt.directory/shape.file,shape.tag); if(!record)return std::unexpected(record.error()); if(!*record)continue; @@ -281,7 +294,7 @@ std::expected validate(const Attempt& attempt) { else if(file=="unit")valid=!fields[0].empty()&&decimal(fields[1])&&fields[1]!="0" &&decimal(fields[2])&&fields[2]!="0"&&named(fields[3]).is_absolute(); else if(file=="baseline")valid=fields[0]==spelling(attempt.bmi)&&digest(fields[1]); - else if(file=="reservation")valid=named(fields[0]).is_absolute(); + else if(file=="reservation"||file=="relocated")valid=named(fields[0]).is_absolute(); if(!valid)return std::unexpected("invalid compiler transaction record value"); } const auto has=[&](std::string_view file){return present.contains(file);}; @@ -548,13 +561,21 @@ struct Recovery { // process space, so the generation's controllers, helpers and writers have // ended in every space that shares this directory, and an attempt whose unit // cannot be observed is settled as an abandoned one. +// A RELOCATED ATTEMPT IS NOT OBSERVED (§10.6): its compiler, running or not, +// writes only below the attempt, and what it published is complete. It is +// settled at once, its unpublished files discarded. std::expected recover(const Attempt& attempt,ObserverMap* held=nullptr, Recovery* met=nullptr,bool abandonUnobservable=false) { - const auto state=observed(attempt,held);if(!state)return std::unexpected(state.error()); - if(*state==Observation::unobservable&&abandonUnobservable) {if(met)++met->unobservable;} - else if(*state!=Observation::empty) { - if(met)++(*state==Observation::occupied ? met->occupied : met->unknown); - return false; + const auto relocation=attempt.relocation();if(!relocation)return std::unexpected(relocation.error()); + if(*relocation) { + if(auto valid=validate(attempt);!valid)return std::unexpected(valid.error()); + } else { + const auto state=observed(attempt,held);if(!state)return std::unexpected(state.error()); + if(*state==Observation::unobservable&&abandonUnobservable) {if(met)++met->unobservable;} + else if(*state!=Observation::empty) { + if(met)++(*state==Observation::occupied ? met->occupied : met->unknown); + return false; + } } auto retired=read(attempt.directory/"retired","mcpp-retired-v1"); auto result=attempt.ordinary ? read(attempt.directory/"result","mcpp-result-v1") : Record{std::optional{}}; @@ -585,7 +606,17 @@ std::expected recover(const Attempt& attempt,ObserverMap* held } } std::error_code ec; - if(!committed) { + if(!committed&&*relocation) { + // What it published before its compile failed or was cut is withdrawn, + // so that its edges run again, as an in-place attempt's outputs are. + if(*published&&!attempt.bmi.empty()) {fs::remove(attempt.bmi,ec);if(ec)return std::unexpected("cannot invalidate an abandoned BMI");} + if(*admitted)for(const auto& path:{attempt.object,attempt.dep})if(!path.empty()) { + fs::remove(path,ec);if(ec)return std::unexpected("cannot invalidate an abandoned compiler output"); + } + if(!attempt.slot.empty())for(std::string_view tail:{".rc",".rc.tmp"}) { + fs::remove(suffix(attempt.slot,tail),ec);if(ec)return std::unexpected("cannot invalidate an abandoned compiler status"); + } + } else if(!committed) { const auto baseline=attempt.baseline();if(!baseline)return std::unexpected(baseline.error()); if(*baseline) { if(*published) { @@ -624,6 +655,9 @@ std::expected recover(const Attempt& attempt,ObserverMap* held fs::remove(suffix(attempt.bmi,".bak.cert"),ec);if(ec)return std::unexpected("cannot retire the carried baseline certificate"); fs::remove(suffix(attempt.bmi,".bak"),ec);if(ec)return std::unexpected("cannot retire the carried baseline"); } + // A compiler still running may hold a file of it, which the system does + // not let go: what remains goes with the build directory's next sweep. + if(*relocation)fs::remove_all(**relocation,ec); auto reservation=read(attempt.directory/"reservation","mcpp-reservation-pointer-v1"); if(!reservation)return std::unexpected(reservation.error()); if(*reservation) { diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm index 4027fa44e..79e32c2cd 100644 --- a/src/build/schedule/detach_codegen.cppm +++ b/src/build/schedule/detach_codegen.cppm @@ -76,6 +76,7 @@ import mcpp.run; import mcpp.build.stage; import mcpp.buildopt.keep; import mcpp.build.schedule.compiler_transaction; +import mcpp.build.schedule.relocation; import mcpp.diag; import mcpp.home; import mcpp.home.files; @@ -164,6 +165,10 @@ struct CompileRequest { // an edge whose depfile names anything but its own output spelling as // permanently dirty. Empty means `bmi` is already that spelling. std::string bmiTarget; + // The object as the command spells it, and how the compile is relocated + // (`--relocate`, design 2026-11-19 §10.6): empty writes in place. + std::string objectTarget; + std::string relocate; }; // Phase 1 — returns 0 as soon as the BMI is published, leaving code generation @@ -174,7 +179,8 @@ int compile_release_at_bmi(const CompileRequest& req); // then records the status. Never invoked directly by a build edge. int supervise(const std::filesystem::path& slot, const std::filesystem::path& semaphore, int maxCompilers, - const std::filesystem::path& object, std::string_view command); + const std::filesystem::path& object, std::string_view command, + const std::filesystem::path& relocation = {}); // Phase 2 — blocks until the compiler for `slot` finished, replays what it // wrote, and propagates its status. `object`, when given, must exist: a @@ -184,11 +190,16 @@ int await_unit(const std::filesystem::path& slot, const std::filesystem::path& o // Ordinary compiler invocations share the detached compilers' actual bound. // Their original argv and streams are retained; no shell is introduced. +// `relocate` non-empty: the compile writes its outputs below an attempt of +// its own and this executor publishes them (design 2026-11-19 §10.6); the +// value is the toolchain's auxiliary naming (`OutputRelocation::auxNaming`), +// or `none`. int run_capped(const std::vector& argv, const std::filesystem::path& semaphore, int maxCompilers, const std::filesystem::path& output = {}, const std::filesystem::path& depfile = {}, - const std::filesystem::path& bmi = {}, bool object = false); + const std::filesystem::path& bmi = {}, bool object = false, + std::string_view relocate = {}); } // namespace mcpp::build::schedule::detach @@ -197,6 +208,16 @@ int run_capped(const std::vector& argv, namespace mcpp::build::schedule::detach { namespace { namespace transactions = mcpp::build::schedule::transaction; +namespace relocation = mcpp::build::schedule::relocation; + +// A name no other attempt has: this process's identity and a sequence of its +// own (design 2026-11-19 §10.6). +std::optional attempt_name() { + const auto self = mcpplibs::os::process::self(); + if (!self || !self->id || !self->started_ns) return std::nullopt; + static std::atomic sequence{0}; + return std::format("{}-{}-{}", self->id, self->started_ns, sequence++); +} std::filesystem::path suffixed(const std::filesystem::path& base, std::string_view s) { return std::filesystem::path{base.string() + std::string(s)}; @@ -770,6 +791,9 @@ std::expected RunScope::begin(const std::filesystem::path } // Old owners, actors and units are quiet before cached output recovery; // no Ninja reader from this generation has started yet. + // What the recovered attempts wrote below themselves goes with them + // (§10.6); a file an orphan still holds stays until a later build. + relocation::sweep(canonical); if(auto admitted=transactions::admit_cached_outputs(canonical);!admitted) return std::unexpected(admitted.error()); // WHETHER A UNIT CAN BE DECLARED HERE IS A PROPERTY OF THE ENVIRONMENT @@ -827,13 +851,38 @@ int compile_release_at_bmi(const CompileRequest& req) { if(!req.slot.parent_path().empty())std::filesystem::create_directories(req.slot.parent_path(),ec); if(ec)return 1; CompileRequest actual = req; + // THE ATTEMPT'S OWN PATHS (§10.6): its command names them, through a map + // of its own for the BMI, and is handed to the supervisor in a file of the + // attempt's, which outlives this edge as the supervisor does. + relocation::Plan plan; + if (!req.relocate.empty()) { + const auto name = attempt_name(); + if (!name) { std::println(std::cerr, "mcpp: relocating the compile of {}: this process's identifier or birth cannot be read", req.objectTarget); return 1; } + auto planned = relocation::relocate(actual.command, + {req.objectTarget, "", req.bmiTarget.empty() ? req.bmi.string() : req.bmiTarget, true}, + *name, req.relocate == "none" ? std::string_view{} : std::string_view(req.relocate)); + if (!planned) { std::println(std::cerr, "mcpp: relocating the compile of {}: {}", req.objectTarget, planned.error()); return 1; } + plan = std::move(*planned); + if (plan.relocated()) { + actual.commandFile = plan.directory / "command"; + std::ofstream file(actual.commandFile, std::ios::binary | std::ios::trunc); + file << actual.command; + file.close(); + if (auto saved = relocation::save(plan); !saved || !file) { + std::println(std::cerr, "mcpp: relocating the compile of {}: its command cannot be written", req.objectTarget); + return 1; + } + } + } std::optional attempt; if(actor.owner.managed) { auto prepared=transactions::Attempt::prepare(actor.owner.generation,false,req.bmi,req.object,req.slot,req.depTo); if(!prepared) {std::println(std::cerr,"mcpp: {}",prepared.error());return 1;} attempt.emplace(std::move(*prepared)); actual.bmi=attempt->bmi;actual.object=attempt->object;actual.slot=attempt->slot;actual.depTo=attempt->dep; - if(auto copied=attempt->prepare_backup();!copied) {std::println(std::cerr,"mcpp: {}",copied.error());return 1;} + if(plan.relocated()) { + if(!attempt->relocate(std::filesystem::absolute(plan.directory))) {std::println(std::cerr,"mcpp: cannot record the relocation of {}",req.objectTarget);return 1;} + } else if(auto copied=attempt->prepare_backup();!copied) {std::println(std::cerr,"mcpp: {}",copied.error());return 1;} } std::filesystem::remove(suffixed(actual.slot, ".rc"), ec); if(attempt&&ec)return 1; @@ -850,8 +899,10 @@ int compile_release_at_bmi(const CompileRequest& req) { // IN PLACE — see BmiIdentity for why moving it away is what broke importers. // Snapshot through the SAME function in both cases, so "no previous BMI" // and "the BMI as it is now" are directly comparable. - const BmiIdentity before = bmi_identity(actual.bmi); - if (!attempt && !actual.bmi.empty()) { + // A relocated attempt watches its own BMI, which did not exist before it. + const auto watched = plan.relocated() ? relocation::written_bmi(plan) : actual.bmi; + const BmiIdentity before = plan.relocated() ? BmiIdentity{} : bmi_identity(actual.bmi); + if (!attempt && !plan.relocated() && !actual.bmi.empty()) { const auto backup = suffixed(actual.bmi, ".bak"); std::filesystem::remove(backup, ec); if (before.present) { @@ -872,6 +923,11 @@ int compile_release_at_bmi(const CompileRequest& req) { // The BMI this slot publishes, for phase 2 to withdraw if the compile // fails (see `await_unit`). if (!actual.bmi.empty()) std::ofstream(suffixed(actual.slot, ".bmi"), std::ios::trunc) << actual.bmi.string(); + // The attempt's directory, for phase 2 to discard once the compiler ended. + std::filesystem::remove(suffixed(actual.slot, ".relocation"), ec); + if (plan.relocated()) + std::ofstream(suffixed(actual.slot, ".relocation"), std::ios::trunc) + << mcpp::home::spelling(std::filesystem::absolute(plan.directory)); // The supervisor reads the SAME argv file rather than receiving the command // on its own command line: one representation, no re-quoting, and no limit @@ -882,10 +938,11 @@ int compile_release_at_bmi(const CompileRequest& req) { if (!actual.semaphore.empty()) { sup.push_back("--sem"); sup.push_back(actual.semaphore.string()); } if (actual.maxCompilers > 0) { sup.push_back("--cap"); sup.push_back(std::to_string(actual.maxCompilers)); } if (!actual.object.empty()) { sup.push_back("--object"); sup.push_back(actual.object.string()); } + if (plan.relocated()) { sup.push_back("--relocation"); sup.push_back(std::filesystem::absolute(plan.directory).string()); } if (!actor.owner.active()) return 130; std::optional supervisor; if (!spawn_detached(sup,actor.owner.managed,&supervisor)) { - if(!attempt)restore_backup(actual.bmi); + if(!attempt&&!plan.relocated())restore_backup(actual.bmi); return 2; } @@ -910,9 +967,12 @@ int compile_release_at_bmi(const CompileRequest& req) { // GCC unlinks an existing BMI immediately before renaming its new // temporary file. Absence is a transition, never publication: treating // it as a changed identity releases importers inside that unlink gap. - const auto published = bmi_identity(actual.bmi); + const auto published = bmi_identity(watched); if (published.present && published != before) { - if(attempt) { + if(plan.relocated()) { + if(attempt&&!attempt->publication())return 1; + if(auto done=relocation::publish_bmi(plan);!done) {std::println(std::cerr,"mcpp: {}",done.error());return 1;} + } else if(attempt) { if(!attempt->publication())return 1; if(auto settled=attempt->settle();!settled) {std::println(std::cerr,"mcpp: {}",settled.error());return 1;} } else settle_bmi(actual.bmi); @@ -926,7 +986,18 @@ int compile_release_at_bmi(const CompileRequest& req) { if (in) std::cerr << in.rdbuf(); // Nothing was published, so the previous BMI is the truth. Put // it back rather than leaving the unit with no BMI at all. - if(!attempt)restore_backup(actual.bmi); + if(!attempt&&!plan.relocated())restore_backup(actual.bmi); + } else if (plan.relocated()) { + // The compile ended within one poll: its BMI is published now. + if(attempt&&!attempt->publication())return 1; + auto done=relocation::publish_bmi(plan); + if(!done) {std::println(std::cerr,"mcpp: {}",done.error());return 1;} + if(!*done) { + std::println(std::cerr,"mcpp: compiler reported success but {} is missing",actual.bmi.string()); + return 1; + } + copy_first_rule(actual.depFrom, actual.depTo, + actual.bmiTarget.empty() ? actual.bmi.string() : actual.bmiTarget); } else { if (!actual.bmi.empty() && (!file_exists(actual.bmi) || (attempt && bmi_identity(actual.bmi)==before))) { @@ -987,14 +1058,14 @@ int compile_release_at_bmi(const CompileRequest& req) { "(no log after {}s) — restoring the previous BMI", actual.slot.string(), std::chrono::duration_cast(waited).count()); - if(!attempt)restore_backup(actual.bmi); + if(!attempt&&!plan.relocated())restore_backup(actual.bmi); return 1; } if (waited > kHardLimit) { std::println(std::cerr, "mcpp: timed out waiting for {} to publish its BMI", actual.slot.string()); - if(!attempt)restore_backup(actual.bmi); + if(!attempt&&!plan.relocated())restore_backup(actual.bmi); return 1; } std::this_thread::sleep_for(std::chrono::milliseconds(2)); @@ -1003,9 +1074,20 @@ int compile_release_at_bmi(const CompileRequest& req) { int supervise(const std::filesystem::path& slot, const std::filesystem::path& semaphore, int maxCompilers, - const std::filesystem::path& object, std::string_view command) { + const std::filesystem::path& object, std::string_view command, + const std::filesystem::path& relocationDir) { auto actor = std::make_shared(); if (!actor->ready) return 130; + // A relocated compile's outputs but its BMI, which phase 1 publishes the + // moment it is complete (§10.6): published once the compiler has ended. + auto publish_outputs = [&](int rc) { + if (relocationDir.empty() || rc != 0 || !actor->owner.active()) return rc; + auto plan = relocation::load(relocationDir); + auto done = plan ? relocation::publish(*plan, false) : std::expected(std::unexpected(plan.error())); + if (done) return rc; + record_supervisor_error(slot, "publishing the compile's outputs: " + done.error()); + return 1; + }; if(actor->owner.managed) { auto loaded=transactions::for_slot(actor->owner.generation,slot); if(!loaded||!*loaded) {std::println(std::cerr,"mcpp: missing or invalid prepared compiler attempt");return 2;} @@ -1032,10 +1114,11 @@ int supervise(const std::filesystem::path& slot, if(!transactions::publish(attempt.directory/"admitted","mcpp-admitted-v1",{})) { (void)write_rc(slot,2);return 2; } - const int rc=run_to_completion(command,suffixed(slot,".log"),actor->owner,unit.get()); + const int ran=run_to_completion(command,suffixed(slot,".log"),actor->owner,unit.get()); if(auto retired=retire_owned(*unit,actor->owner);!retired) { record_supervisor_error(slot,retired.error());return 1; } + const int rc=publish_outputs(ran); if(!attempt.retired(rc)) {record_supervisor_error(slot,"cannot publish compiler retirement certificate");return 1;} std::string status_failure; if(!write_rc(slot,rc,&status_failure)) { @@ -1079,7 +1162,7 @@ int supervise(const std::filesystem::path& slot, // Acquire in the supervisor: no lock-handle handoff, and no interval in // which an early-returning phase 1 could release the compiler's permit. if (!actor->owner.active()) return 130; - const int rc = run_to_completion(command, suffixed(slot, ".log"), actor->owner); + const int rc = publish_outputs(run_to_completion(command, suffixed(slot, ".log"), actor->owner)); settle(mcpp::run::ending() ? 130 : rc); return 0; // the supervisor's own status is not the compiler's } @@ -1155,6 +1238,15 @@ int await_unit(const std::filesystem::path& slot, const std::filesystem::path& o std::ifstream in(suffixed(slot, ".log")); if (in) std::cerr << in.rdbuf(); } + // The compiler has ended and phase 1 has returned: a relocated attempt's + // directory holds nothing more to publish (§10.6). + { + std::error_code ec; + std::string directory; + if (std::ifstream named(suffixed(slot, ".relocation")); named && std::getline(named, directory) && !directory.empty()) + relocation::discard(mcpp::home::from_utf8(directory)); + std::filesystem::remove(suffixed(slot, ".relocation"), ec); + } if(attempt) { if(*rc!=0)return *rc; auto retired=transactions::read(attempt->directory/"retired","mcpp-retired-v1"); @@ -1251,27 +1343,51 @@ struct Previous { int run_capped(const std::vector& argv, const std::filesystem::path& semaphore, int maxCompilers, const std::filesystem::path& output, const std::filesystem::path& depfile, - const std::filesystem::path& bmi, bool object) { + const std::filesystem::path& bmi, bool object, std::string_view relocate) { if (argv.empty()) { std::println(std::cerr, "mcpp: __compiler-cap: no compiler command was given"); return 2; } auto actor = std::make_shared(); if (!actor->ready) return 130; - Previous previousBmi{Previous::Kind::Bmi, bmi}; - Previous previousObject{Previous::Kind::Object, object ? output : std::filesystem::path{}}; + // EVERY RETURN BUT THE COMPILER'S OWN STATUS AND A CANCELLATION SAYS + // WHICH STEP ENDED IT AND WHY (§10.3 G): ninja shows the line, and a + // helper that ended with 1 or 2 and nothing on its error stream left a + // failed build with no cause (Termux, 2026-11-18). + const auto out_name=output.empty() ? std::string("a compile") : mcpp::home::spelling(output); + auto ended=[&](int status,std::string_view step,std::string_view why) { + std::println(std::cerr,"mcpp: {} for {}: {}",step,out_name,why);return status; + }; + // THE ATTEMPT'S OWN PATHS (§10.6), decided before anything of the + // compile is held or recorded. + std::vector command = argv; + relocation::Plan plan; + if (!relocate.empty()) { + const auto name = attempt_name(); + if (!name) return ended(1, "relocating the compile's outputs", "this process's identifier or birth cannot be read"); + auto planned = relocation::relocate(command, {output.string(), depfile.string(), bmi.string(), object}, + *name, relocate == "none" ? std::string_view{} : relocate); + if (!planned) return ended(1, "relocating the compile's outputs", planned.error()); + plan = std::move(*planned); + } + // An attempt in place holds what the compile wrote before (`Previous`); + // a relocated one leaves it where it is until it publishes. + std::optional previousBmi, previousObject; + if (!plan.relocated()) { + previousBmi.emplace(Previous::Kind::Bmi, bmi); + previousObject.emplace(Previous::Kind::Object, object ? output : std::filesystem::path{}); + } // Before the transaction certifies what the compiler retired: the // certificate names the output as it stands. - auto settle = [&](bool compiled) { previousBmi.settle(compiled); previousObject.settle(compiled); }; + auto settle = [&](bool compiled) -> std::expected { + if (!plan.relocated()) { previousBmi->settle(compiled); previousObject->settle(compiled); return {}; } + auto published = compiled ? relocation::publish(plan) : std::expected{}; + relocation::discard(plan.directory); + return published; + }; if(actor->owner.managed) { - // EVERY RETURN BUT THE COMPILER'S OWN STATUS AND A CANCELLATION SAYS - // WHICH STEP ENDED IT AND WHY (§10.3 G): ninja shows the line, and a - // helper that ended with 1 or 2 and nothing on its error stream left a - // failed build with no cause (Termux, 2026-11-18). - const auto out_name=output.empty() ? std::string("a compile") : mcpp::home::spelling(output); - auto ended=[&](int status,std::string_view step,std::string_view why) { - std::println(std::cerr,"mcpp: {} for {}: {}",step,out_name,why);return status; - }; auto prepared=transactions::Attempt::prepare(actor->owner.generation,true,{},output,{},depfile); if(!prepared)return ended(1,"preparing the compiler transaction",prepared.error()); auto attempt=std::move(*prepared); + if(plan.relocated()&&!attempt.relocate(std::filesystem::absolute(plan.directory))) + return ended(1,"recording the compile's relocation","its record cannot be written"); const auto who=mcpplibs::os::process::self(); if(!who||!who->id||!who->started_ns) return ended(1,"identifying the compiler controller","this process's identifier or birth cannot be read"); @@ -1288,39 +1404,39 @@ int run_capped(const std::vector& argv, if(!actor->owner.active())return 130; if(!transactions::publish(attempt.directory/"admitted","mcpp-admitted-v1",{})) return ended(2,"recording the compiler's admission","its record cannot be written"); - const auto result=mcpp::run::execute(argv,{ + const auto result=mcpp::run::execute(command,{ .input=mcpp::run::Input::inherit,.streams=mcpp::run::Streams::inherit, .should_stop=[&] {return !actor->owner.active();},.declared_unit=unit.get()}); if(auto retired=retire_owned(*unit,actor->owner);!retired) return ended(1,"retiring the compiler unit",retired.error()); - settle(result.exit_code==0&&actor->owner.active()); + if(auto settled=settle(result.exit_code==0&&actor->owner.active());!settled) + return ended(1,"publishing the compile's outputs",settled.error()); if(!attempt.retired(result.exit_code)||!attempt.result(result.exit_code)) return ended(1,"certifying the compiler's result","its record cannot be written"); if(!transactions::release_reservation(permit->reservation,attempt.directory)) return ended(1,"releasing the compiler slot","its reservation cannot be released"); return result.exit_code; } - InterruptCleanup cleanup{mcpp::run::add_cleanup([actor, output] { - if (!output.empty()) { - std::error_code ec; - std::filesystem::remove(output, ec); - } + // An interrupted compile in place leaves no output a later build would + // take for complete; a relocated one leaves its own directory only. + const auto withdrawn = plan.relocated() ? plan.directory : output; + InterruptCleanup cleanup{mcpp::run::add_cleanup([actor, withdrawn] { + std::error_code ec; + if (!withdrawn.empty()) std::filesystem::remove_all(withdrawn, ec); })}; auto permit = acquire_permit(*actor, semaphore, maxCompilers); if (!permit) return permit.error(); if (!actor->owner.active()) return 130; - const auto result = mcpp::run::execute(argv, { + const auto result = mcpp::run::execute(command, { .input = mcpp::run::Input::inherit, .streams = mcpp::run::Streams::inherit, .should_stop = [&] { return !actor->owner.active(); }}); if (!actor->owner.active() || result.exit_code != 0) { - if (!output.empty()) { - std::error_code ec; - std::filesystem::remove(output, ec); - } + std::error_code ec; + if (!withdrawn.empty()) std::filesystem::remove_all(withdrawn, ec); return actor->owner.active() ? result.exit_code : 130; } - settle(true); + if (auto settled = settle(true); !settled) return ended(1, "publishing the compile's outputs", settled.error()); return result.exit_code; } diff --git a/src/build/schedule/relocation.cppm b/src/build/schedule/relocation.cppm new file mode 100644 index 000000000..d98bffd0e --- /dev/null +++ b/src/build/schedule/relocation.cppm @@ -0,0 +1,462 @@ +// mcpp.build.schedule.relocation — a compile writes its outputs at paths of +// its own attempt, and the executor publishes them by rename (design +// 2026-11-19 §10.6). +// +// WHY. A compiler that writes the published path directly lets one that +// outlives its build -- its controller killed, its process space gone -- +// rewrite an output the next build has already trusted. Recovery then had to +// prove, by observing the compiler's unit, that no such writer remained, and +// where the unit could not be observed or was still occupied the next build +// was refused (mcpp#952). A compiler that writes only paths of its own attempt +// cannot reach a published output: the attempt is published by the executor +// after the compiler has ended, or discarded, and recovery needs no +// observation to protect what is published. +// +// HOW. The executor is told, on its command line, the files a compile writes +// (`--output`, `--depfile`, `--bmi`). Each word of the command that names one +// of them -- the whole word, or the part after `=` or `:` (`-fmodule-output=`, +// `/Fo:`), with nothing after it or a `.` suffix (`$out.d`, `$out.d.raw`) -- +// is spelled below the attempt's directory instead, which mirrors the build +// directory. GCC names its BMI through its module map, so the attempt is given +// a copy of the map in which the unit's own module names the attempt's BMI. +// The auxiliary outputs a compiler names after the object (`.dwo`, `.gcno`) are +// named after the final object (`-dumpdir`, `-dumpbase`), because the object +// records their names and no prefix map rewrites them (measured, GCC 16.1 and +// clang 23.1.3): they are written in place, and no build input is among them. +// +// A file the command names that cannot be spelled below the attempt (an +// absolute path outside the build directory) leaves the whole compile in +// place: an attempt is either relocated in every output or in none, because +// recovery relies on the answer. +export module mcpp.build.schedule.relocation; + +import std; +import mcpp.buildopt.keep; + +export namespace mcpp::build::schedule::relocation { + +// The attempts' directory below a build directory: below it, and not beside +// `target/` with the compiler-run state, because a rename publishes and a +// rename does not cross file systems, which `target/` may. +inline constexpr std::string_view kAttempts = ".mcpp-attempts"; + +// What the executor is told of a compile's outputs, as its command spells +// them (relative to the build directory, where the executor runs). +struct Named { + std::string output; // `--output` + std::string depfile; // `--depfile` + std::string bmi; // `--bmi` + bool object = false; // `--object`: `output` is an object +}; + +struct Output { + // A dependency file names what the compile read, the attempt's map among + // them (GCC lists its mapper), and the attempt's outputs as its targets: + // it is published with those names as published. + enum class Kind { object, bmi, depfile, other }; + Kind kind = Kind::other; + std::filesystem::path published, written; +}; + +// An attempt's relocation. Empty `outputs` is a compile in place. +struct Plan { + std::filesystem::path directory; // `.mcpp-attempts/` + std::filesystem::path root; // `/out`, the mirror of the build directory + std::vector outputs; + // GCC: the unit's module map as the command named it, and the attempt's. + std::string map, attemptMap; + bool relocated() const { return !outputs.empty(); } +}; + +// The attempt's directory for an attempt name. +std::filesystem::path attempt_directory(std::string_view attempt); + +// Rewrites `argv` (or one shell command line, whose words are separated by +// spaces) for the attempt `attempt`, with the auxiliary-name words of +// `auxNaming` ("dumpbase", "dumpdir", or "" for none) appended. On a command +// that cannot be relocated, the command is left as it was and the plan is +// empty; an error is a map that cannot be read or written. +std::expected relocate(std::vector& argv, const Named& named, + std::string_view attempt, std::string_view auxNaming); +std::expected relocate(std::string& command, const Named& named, + std::string_view attempt, std::string_view auxNaming); + +// The plan, written beside the attempt's outputs for a process that publishes +// them (the GCC schedule's supervisor), and read back. +std::expected save(const Plan& plan); +std::expected load(const std::filesystem::path& directory); + +// Publishes what the compile wrote: an object or a BMI equal to the published +// one leaves the published file as it is (its time unchanged, so `restat` +// ends the rebuild there), and every other file the attempt wrote replaces +// its published counterpart. `withBmi` false leaves the BMI, which the GCC +// schedule publishes the moment it is complete. Does not discard. +std::expected publish(const Plan& plan, bool withBmi = true); + +// The BMI alone: whether it was there to publish. +std::expected publish_bmi(const Plan& plan); + +// The attempt's BMI as written, or empty. +std::filesystem::path written_bmi(const Plan& plan); + +// Removes the attempt's directory. A compiler that outlived its build may +// still hold a file of it, which a system refuses to remove while it is open: +// what remains is removed by a later build (`sweep`). +void discard(const std::filesystem::path& directory); + +// Removes every attempt directory below `buildDir`: at a build's start, when +// no compile of the directory runs but one an interrupted build left. +void sweep(const std::filesystem::path& buildDir); + +// The words that name the auxiliary outputs after `object`. +std::vector aux_words(std::string_view auxNaming, std::string_view object); + +} // namespace mcpp::build::schedule::relocation + +namespace mcpp::build::schedule::relocation { +namespace { + +namespace fs = std::filesystem; + +// Where `spelling` lies below the attempt: the same spelling below `root`, so +// that a word that names the file with a suffix (`$out.d.raw`) names the +// attempt's file with the same suffix. Empty for an absolute path outside the +// build directory. +std::string below(const fs::path& root, std::string_view spelling) { + const fs::path p{std::string(spelling)}; + if (p.is_relative()) { + auto rel = p.lexically_normal(); + if (rel.empty() || rel.begin()->string() == "..") return {}; + return (root / rel).generic_string(); + } + std::error_code ec; + const auto cwd = fs::current_path(ec); + if (ec) return {}; + auto rel = p.lexically_normal().lexically_relative(cwd); + if (rel.empty() || rel.begin()->string() == "..") return {}; + return (cwd / root / rel).string(); +} + +// Where in `word` a file `spelling` names begins: the whole word, or the part +// after an `=` or `:`, followed by nothing or by a `.` suffix. +std::optional names_at(std::string_view word, std::string_view spelling) { + if (spelling.empty()) return std::nullopt; + auto at = [&](std::size_t start) { + if (!word.substr(start).starts_with(spelling)) return false; + const auto end = start + spelling.size(); + return end == word.size() || word[end] == '.'; + }; + if (at(0)) return 0; + for (std::size_t i = 0; i < word.size(); ++i) + if ((word[i] == '=' || word[i] == ':') && at(i + 1)) return i + 1; + return std::nullopt; +} + +struct Rewriter { + const Named& named; + Plan& plan; + bool outside = false; // a named file outside the build directory + std::set seen; + + Output::Kind kind_of(std::string_view file) const { + if (!named.bmi.empty() && file == named.bmi) return Output::Kind::bmi; + if (named.object && file == named.output) return Output::Kind::object; + // `$out.d`, the filtered `$out.d.raw`, a scan's `$out.dep`. + if (!named.depfile.empty() && file.starts_with(named.depfile)) return Output::Kind::depfile; + return Output::Kind::other; + } + // The word as the attempt spells it; the word itself where it names none. + std::string word(std::string_view w) { + for (const auto* spelling : {&named.output, &named.depfile, &named.bmi}) { + const auto start = names_at(w, *spelling); + if (!start) continue; + const auto file = w.substr(*start); + const auto written = below(plan.root, file); + if (written.empty()) { outside = true; return std::string(w); } + if (seen.insert(std::string(file)).second) + plan.outputs.push_back({kind_of(file), fs::path{std::string(file)}, fs::path{written}}); + return std::string(w.substr(0, *start)) + written; + } + return std::string(w); + } +}; + +// GCC's module map, for the attempt: the line of the module whose BMI is +// `bmi` names the attempt's file, relative to the map's repository. Returns +// the attempt's map spelling, or empty where the map names no such module. +std::expected attempt_map(std::string_view mapSpelling, const Named& named, + Plan& plan) { + const fs::path map{std::string(mapSpelling)}; + std::ifstream in(map, std::ios::binary); + if (!in) return std::unexpected(std::format("cannot read the module map {}", map.generic_string())); + std::string out, line; + fs::path repository; + bool found = false; + const auto bmi = fs::path{named.bmi}.lexically_normal(); + while (std::getline(in, line)) { + const auto space = line.find(' '); + if (space != std::string::npos && line.starts_with("$root ")) repository = line.substr(space + 1); + else if (space != std::string::npos && !found) { + const fs::path named_path = line.substr(space + 1); + const auto resolved = (named_path.is_absolute() ? named_path : repository / named_path).lexically_normal(); + if (resolved == bmi) { + const auto written = below(plan.root, named.bmi); + if (written.empty()) return std::string{}; + line = line.substr(0, space + 1) + fs::path(written).lexically_relative(repository).generic_string(); + plan.outputs.push_back({Output::Kind::bmi, fs::path{named.bmi}, fs::path{written}}); + found = true; + } + } + out += line + "\n"; + } + if (!found) return std::string{}; + const auto attemptMap = plan.directory / "module.map"; + std::error_code ec; + fs::create_directories(plan.directory, ec); + std::ofstream file(attemptMap, std::ios::binary | std::ios::trunc); + file << out; + if (!file) return std::unexpected(std::format("cannot write the attempt's module map {}", attemptMap.generic_string())); + return attemptMap.generic_string(); +} + +// The parent directories of what the attempt writes: a compiler does not +// create the directory of its `-o`. +std::expected make_directories(const Plan& plan) { + for (const auto& o : plan.outputs) { + std::error_code ec; + fs::create_directories(o.written.parent_path(), ec); + if (ec) return std::unexpected(std::format("cannot create {}: {}", o.written.parent_path().generic_string(), ec.message())); + } + return {}; +} + +// Relocates `words` in place; `rejoin` writes them back where they came from. +std::expected relocate_words(std::vector& words, const Named& named, + std::string_view attempt, std::string_view auxNaming) { + Plan plan; + plan.directory = attempt_directory(attempt); + plan.root = plan.directory / "out"; + Rewriter rewrite{named, plan}; + std::vector out; + out.reserve(words.size() + 4); + constexpr std::string_view mapper = "-fmodule-mapper="; + bool bmiByMap = false; + for (const auto& w : words) { + if (!named.bmi.empty() && w.starts_with(mapper)) { + auto map = attempt_map(std::string_view(w).substr(mapper.size()), named, plan); + if (!map) return std::unexpected(map.error()); + if (map->empty()) { rewrite.outside = true; out.push_back(w); continue; } + out.push_back(std::string(mapper) + *map); + plan.map = w.substr(mapper.size()); + plan.attemptMap = *map; + bmiByMap = true; + rewrite.seen.insert(named.bmi); + continue; + } + out.push_back(rewrite.word(w)); + } + auto has = [&](std::string_view file) { + return std::ranges::any_of(plan.outputs, [&](const Output& o) { return o.published == fs::path{std::string(file)}; }); + }; + // Every named file is written below the attempt, or the compile stays in + // place: a compiler that wrote one of them where it is published would + // void what recovery concludes from the relocation. + const bool complete = !rewrite.outside && has(named.output) + && (named.bmi.empty() || bmiByMap || has(named.bmi)); + if (!complete) { + std::error_code ec; + fs::remove_all(plan.directory, ec); + return Plan{}; + } + for (auto& w : aux_words(auxNaming, named.output)) out.push_back(std::move(w)); + if (auto made = make_directories(plan); !made) return std::unexpected(made.error()); + words = std::move(out); + return plan; +} + +std::string_view kind_name(Output::Kind k) { + return k == Output::Kind::object ? "object" : k == Output::Kind::bmi ? "bmi" + : k == Output::Kind::depfile ? "depfile" : "other"; +} + +// A dependency file with the attempt's names replaced by the published ones: +// the attempt's map by the unit's, and a path below the attempt's mirror by +// the same path below the build directory. Its other words are the compile's +// reads, as the compiler wrote them. +std::string published_text(const Plan& plan, std::string text) { + auto replace_all = [&](const std::string& from, const std::string& to) { + if (from.empty()) return; + for (std::size_t at = text.find(from); at != std::string::npos; at = text.find(from, at + to.size())) + text.replace(at, from.size(), to); + }; + replace_all(plan.attemptMap, plan.map); + replace_all(plan.root.generic_string() + "/", ""); + return text; +} + +// Moves `from` over `to`: one rename, so a reader sees the old file or the +// new one and never a part of either. +std::expected replace(const fs::path& from, const fs::path& to) { + std::error_code ec; + fs::create_directories(to.parent_path(), ec); + fs::rename(from, to, ec); + if (!ec) return {}; + return std::unexpected(std::format("cannot publish {} as {}: {}", from.generic_string(), + to.generic_string(), ec.message())); +} + +std::expected publish_one(const Plan& plan, const Output& o) { + std::error_code ec; + if (!fs::is_regular_file(o.written, ec)) return {}; + if (o.kind == Output::Kind::depfile) { + std::ifstream in(o.written, std::ios::binary); + const std::string text{std::istreambuf_iterator(in), {}}; + in.close(); + std::ofstream out(o.written, std::ios::binary | std::ios::trunc); + out << published_text(plan, text); + if (!out) return std::unexpected(std::format("cannot write {}", o.written.generic_string())); + } + bool keep = false; + if (fs::is_regular_file(o.published, ec)) { + if (o.kind == Output::Kind::object) + keep = mcpp::buildopt::keep::keeps_previous_object(o.written, o.published, o.published); + else if (o.kind == Output::Kind::bmi) + keep = mcpp::buildopt::keep::keeps_previous(o.written, o.published, o.published); + } + if (keep) { + fs::remove(o.written, ec); + return {}; + } + return replace(o.written, o.published); +} + +} // namespace + +fs::path attempt_directory(std::string_view attempt) { + return fs::path{std::string(kAttempts)} / std::string(attempt); +} + +std::vector aux_words(std::string_view auxNaming, std::string_view object) { + if (auxNaming.empty() || object.empty()) return {}; + const auto slash = object.find_last_of("/\\"); + const std::string dir = slash == std::string_view::npos ? "." : std::string(object.substr(0, slash)); + std::string stem(slash == std::string_view::npos ? object : object.substr(slash + 1)); + if (const auto dot = stem.rfind('.'); dot != std::string::npos && dot > 0) stem.resize(dot); + if (auxNaming == "dumpbase") return {"-dumpdir", dir + "/", "-dumpbase", stem}; + if (auxNaming == "dumpdir") return {"-dumpdir", dir + "/" + stem + "."}; + return {}; +} + +std::expected relocate(std::vector& argv, const Named& named, + std::string_view attempt, std::string_view auxNaming) { + return relocate_words(argv, named, attempt, auxNaming); +} + +std::expected relocate(std::string& command, const Named& named, + std::string_view attempt, std::string_view auxNaming) { + // The command's words as the backend joined them: separated by single + // spaces, a quoted word kept whole. A word that names an output is a path + // of the build directory, spelled without a character a shell quotes. + std::vector words; + std::string current; + char quote = 0; + for (const char c : command) { + if (quote == 0 && (c == '"' || c == '\'')) quote = c; + else if (c == quote) quote = 0; + if (c == ' ' && quote == 0) { words.push_back(std::move(current)); current.clear(); continue; } + current += c; + } + words.push_back(std::move(current)); + auto plan = relocate_words(words, named, attempt, auxNaming); + if (!plan || !plan->relocated()) return plan; + std::string joined; + for (const auto& w : words) joined += (joined.empty() ? "" : " ") + w; + command = std::move(joined); + return plan; +} + +std::expected save(const Plan& plan) { + std::string text = std::format("map\t{}\t{}\n", plan.map, plan.attemptMap); + for (const auto& o : plan.outputs) + text += std::format("{}\t{}\t{}\n", kind_name(o.kind), o.published.generic_string(), o.written.generic_string()); + std::error_code ec; + fs::create_directories(plan.directory, ec); + const auto file = plan.directory / "plan"; + std::ofstream out(file, std::ios::binary | std::ios::trunc); + out << text; + if (!out) return std::unexpected(std::format("cannot write the attempt's plan {}", file.generic_string())); + return {}; +} + +std::expected load(const fs::path& directory) { + Plan plan; + plan.directory = directory; + plan.root = directory / "out"; + std::ifstream in(directory / "plan", std::ios::binary); + if (!in) return std::unexpected(std::format("cannot read the attempt's plan in {}", directory.generic_string())); + for (std::string line; std::getline(in, line);) { + const auto a = line.find('\t'), b = line.rfind('\t'); + if (a == std::string::npos || a == b) return std::unexpected("invalid attempt plan"); + const auto kind = line.substr(0, a); + if (kind == "map") { + plan.map = line.substr(a + 1, b - a - 1); + plan.attemptMap = line.substr(b + 1); + continue; + } + plan.outputs.push_back({kind == "object" ? Output::Kind::object : kind == "bmi" ? Output::Kind::bmi + : kind == "depfile" ? Output::Kind::depfile : Output::Kind::other, + fs::path{line.substr(a + 1, b - a - 1)}, fs::path{line.substr(b + 1)}}); + } + return plan; +} + +std::expected publish(const Plan& plan, bool withBmi) { + for (const auto& o : plan.outputs) { + if (o.kind == Output::Kind::bmi && !withBmi) continue; + if (auto done = publish_one(plan, o); !done) return done; + } + // What else the compiler wrote below the attempt, named after a file it + // was given (clang's `-ftime-trace` beside the object): its counterpart. + std::error_code ec; + if (!fs::is_directory(plan.root, ec)) return {}; + std::set named; + for (const auto& o : plan.outputs) named.insert(o.written.lexically_normal()); + for (fs::recursive_directory_iterator it(plan.root, ec), end; !ec && it != end; it.increment(ec)) { + std::error_code fec; + if (!it->is_regular_file(fec) || named.contains(it->path().lexically_normal())) continue; + const auto rel = it->path().lexically_relative(plan.root); + if (auto done = replace(it->path(), rel); !done) return done; + } + return {}; +} + +std::expected publish_bmi(const Plan& plan) { + for (const auto& o : plan.outputs) { + if (o.kind != Output::Kind::bmi) continue; + std::error_code ec; + if (!fs::is_regular_file(o.written, ec)) return false; + if (auto done = publish_one(plan, o); !done) return std::unexpected(done.error()); + return true; + } + return false; +} + +fs::path written_bmi(const Plan& plan) { + for (const auto& o : plan.outputs) + if (o.kind == Output::Kind::bmi) return o.written; + return {}; +} + +void discard(const fs::path& directory) { + std::error_code ec; + fs::remove_all(directory, ec); +} + +void sweep(const fs::path& buildDir) { + std::error_code ec; + const auto attempts = buildDir / std::string(kAttempts); + for (fs::directory_iterator it(attempts, ec), end; !ec && it != end; it.increment(ec)) discard(it->path()); + fs::remove(attempts, ec); +} + +} // namespace mcpp::build::schedule::relocation diff --git a/src/cli.cppm b/src/cli.cppm index 3b635c10c..38ff30dfa 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -282,6 +282,7 @@ int run(int argc, char** argv) { std::filesystem::path output; std::filesystem::path depfile; std::filesystem::path bmi; + std::string relocate; bool object = false; int cap = 0, i = 2; for (; i < argc && std::string_view(argv[i]) != "--"; ++i) { @@ -290,8 +291,8 @@ int run(int argc, char** argv) { // compiler writes the bytes it had. if (option == "--object") { object = true; continue; } if ((option != "--sem" && option != "--cap" && option != "--output" && option != "--depfile" - && option != "--bmi") || i + 1 >= argc) { - std::println(libc::stderr(), "error: __compiler-cap: expected --sem [--cap ] [--object] [--bmi ] -- ..."); + && option != "--bmi" && option != "--relocate") || i + 1 >= argc) { + std::println(libc::stderr(), "error: __compiler-cap: expected --sem [--cap ] [--object] [--bmi ] [--relocate ] -- ..."); return 2; } const std::string_view value = argv[++i]; @@ -299,6 +300,7 @@ int run(int argc, char** argv) { else if (option == "--output") output = value; else if (option == "--depfile") depfile = value; else if (option == "--bmi") bmi = value; + else if (option == "--relocate") relocate = value; else { const auto [end, ec] = std::from_chars(value.data(), value.data() + value.size(), cap); if (ec != std::errc{} || end != value.data() + value.size()) return 2; @@ -308,7 +310,7 @@ int run(int argc, char** argv) { if (semaphore.empty() || cap < 0 || i + 1 >= argc) return 2; std::vector command; for (++i; i < argc; ++i) command.emplace_back(argv[i]); - return mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, object); + return mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, object, relocate); } ReportUnnarrowablePaths reportUnnarrowable_; @@ -1458,6 +1460,7 @@ int run(int argc, char** argv) { .option(cl::Option("command-file").takes_value().value_name("PATH").help("file holding the compiler command line")) .option(cl::Option("dep-from").takes_value().value_name("PATH").help("scanner depfile to adopt")) .option(cl::Option("dep-to").takes_value().value_name("PATH").help("where ninja expects this edge's depfile")) + .option(cl::Option("relocate").takes_value().value_name("NAMING").help("write below an attempt of its own and publish (design 2026-11-19 §10.6)")) .action(cmd_bmi_compile)) .subcommand(cl::App("bmi-supervise") .description("(internal) Run a compiler to completion and record its status") @@ -1466,6 +1469,7 @@ int run(int argc, char** argv) { .option(cl::Option("cap").takes_value().value_name("N")) .option(cl::Option("object").takes_value().value_name("PATH")) .option(cl::Option("command-file").takes_value().value_name("PATH")) + .option(cl::Option("relocation").takes_value().value_name("DIR")) .action(cmd_bmi_supervise)) .subcommand(cl::App("bmi-await") .description("(internal) Join a detached compiler and replay its diagnostics") diff --git a/src/cli/cmd_build.cppm b/src/cli/cmd_build.cppm index f08cae4df..957dec064 100644 --- a/src/cli/cmd_build.cppm +++ b/src/cli/cmd_build.cppm @@ -2540,6 +2540,8 @@ export int cmd_bmi_compile(const mcpplibs::cmdline::ParsedArgs& parsed) { req.bmi = file_path("bmi"); req.object = file_path("object"); req.bmiTarget = opt_value(parsed, "bmi"); + req.objectTarget = opt_value(parsed, "object"); + req.relocate = opt_value(parsed, "relocate"); req.slot = file_path("slot"); req.self = std::filesystem::path{opt_value(parsed, "self")}; req.semaphore = file_path("sem"); @@ -2582,7 +2584,9 @@ export int cmd_bmi_supervise(const mcpplibs::cmdline::ParsedArgs& parsed) { std::println(libc::stderr(), "error: bmi-supervise needs --slot"); return 2; } - return mcpp::build::schedule::detach::supervise(slot, semaphore, cap, object, command); + const auto relocation = opt_value(parsed, "relocation"); + return mcpp::build::schedule::detach::supervise(slot, semaphore, cap, object, command, + relocation.empty() ? std::filesystem::path{} : std::filesystem::path{relocation}); } // Phase 2: join the detached compiler before anything reads its object. diff --git a/src/toolchain/clang.cpp b/src/toolchain/clang.cpp index 129e7f659..58a7ef58f 100644 --- a/src/toolchain/clang.cpp +++ b/src/toolchain/clang.cpp @@ -139,6 +139,9 @@ void enrich_toolchain(Toolchain& tc, const mcpp::run::Env& env) { tc.stdlibVersion = tc.version.empty() ? "unknown" : tc.version; tc.linkRuntimeDirs = mcpp::toolchain::discover_link_runtime_dirs( tc.binaryPath, tc.targetTriple); + // Not asked of a driver for the MSVC ABI, whose objects record their path + // and are written in place whatever it takes (`output_relocation`). + tc.takesDumpdir = !msvTarget && mcpp::toolchain::driver_takes_dumpdir(tc.binaryPath, env); if (auto p = find_libcxx_std_module_source(tc.binaryPath, env)) { tc.set_std_modules(*p, find_libcxx_std_compat_source(tc.binaryPath, env) diff --git a/src/toolchain/probe.cppm b/src/toolchain/probe.cppm index e850b8b33..ad2b55b0a 100644 --- a/src/toolchain/probe.cppm +++ b/src/toolchain/probe.cppm @@ -51,6 +51,14 @@ bool compresses_debug_sections(const std::filesystem::path& compiler, const mcpp // name (design 2026-11-19 §6.2). bool linker_reports_inputs(const std::vector& argv, bool direct, const mcpp::run::Env& env = {}); +// Whether a clang driver takes `-dumpdir`, from which it names a compile's +// auxiliary outputs (`.dwo`, `-ftime-trace`): a compile then writes its object +// below an attempt of its own and its auxiliary outputs keep the names they +// have in place (design 2026-11-19 §10.6). LLVM took the word in 17; Apple +// clang 16 and emcc 6.0.9 take it under numbers of their own, so the driver is +// asked by one compile of an empty unit, and the answer is kept under the +// driver's identity. +bool driver_takes_dumpdir(const std::filesystem::path& compiler, const mcpp::run::Env& env = {}); std::string extract_version(std::string_view s); std::string first_line_of(std::string_view s); @@ -298,6 +306,41 @@ bool linker_reports_inputs(const std::vector& argv, bool direct, return answered[key] = yes; } +bool driver_takes_dumpdir(const std::filesystem::path& compiler, const mcpp::run::Env& env) { + static std::mutex mutex; + static std::map answered; + const auto key = probe_cache_key({compiler.string(), "-dumpdir"}, env, true); + std::lock_guard lock(mutex); + if (auto it = answered.find(key); it != answered.end()) return it->second; + const auto work = mcpp::home::root() / "cache" / "probe" / "dumpdir"; + const auto answer = work / (mcpp::toolchain::hash_string(key) + ".answer"); + if (std::ifstream in(answer); in) { + std::string v; + in >> v; + if (v == "yes" || v == "no") return answered[key] = (v == "yes"); + } + std::error_code ec; + std::filesystem::create_directories(work, ec); + const auto src = work / "empty.c"; + if (!std::filesystem::exists(src, ec)) std::ofstream(src, std::ios::binary).flush(); + const auto stem = work / std::format("probe-{:x}", std::random_device{}()); + const auto object = stem.string() + ".o"; + auto r = mcpp::run::execute({compiler.string(), "-dumpdir", stem.string() + ".", "-c", "-x", "c", + src.string(), "-o", object}, + {.env = env, .host_tool = true, .streams = mcpp::run::Streams::merged, + .report_spawn_failure = false}); + // A word a driver knows and does not use is said in a warning, which + // names it: such a driver is not taken as naming its outputs from it. + const bool yes = r.exit_code == 0 && r.output.find("dumpdir") == std::string::npos; + std::filesystem::remove(object, ec); + mcpp::log::verbose("toolchain", std::format("'{}' takes -dumpdir: {}{}", compiler.string(), + yes ? "yes" : "no", yes ? std::string{} : " (" + first_line_of(r.output) + ")")); + // An answer the driver did not give (it did not start) is not kept. + if (mcpp::effects::writes_global_cache() && (yes || !r.output.empty())) + std::ofstream(answer) << (yes ? "yes\n" : "no\n"); + return answered[key] = yes; +} + std::expected run_capture(const std::vector& argv, const mcpp::run::Env& env, bool withErrors) { const auto key = probe_cache_key(argv, env, withErrors); diff --git a/tests/e2e/667_an_android_pack_stages_its_closure.sh b/tests/e2e/667_an_android_pack_stages_its_closure.sh index ad51959cc..143260bdd 100755 --- a/tests/e2e/667_an_android_pack_stages_its_closure.sh +++ b/tests/e2e/667_an_android_pack_stages_its_closure.sh @@ -10,9 +10,12 @@ # Every ELF shared library carries a SONAME equal to its file name (A4), which # bionic enforces from API level 23. # -# The negative directions: a library the link used and that is gone by the -# time of the pack makes `dir` and `tar` refuse naming it, and a dispatched -# format receives `closure = not-walked` with the name as `unresolved`. +# The negative directions. A library the link used and that is gone is an input +# of the link that is gone (SPEC-004 §8.1, mcpp#931): the pack's build links +# again and the link names it. A library the link found through a flag the +# pack does not read is needed and found in none of the pack's directories: +# `dir` and `tar` refuse naming it, and a dispatched format receives +# `closure = not-walked` with the name as `unresolved`. set -e TMP=$(mktemp -d) @@ -97,18 +100,32 @@ grep -qxF "needs${TAB}libext.so${TAB}lib/libext.so" "$manifest" \ || fail "the manifest does not state libext.so as staged" "$manifest" echo " ok: a library from [runtime] link_library_dirs is staged" -# The build is up to date, so the pack does not relink; the file is simply gone. rm ext/lib/libext.so +if "$MCPP" pack --target x86_64-linux-android --pkg-format dir > gone.log 2>&1; then + fail "the pack succeeded with libext.so gone" gone.log +fi +grep -q -- "-lext" gone.log || fail "the link did not name the library that is gone" gone.log +echo " ok: a library the link read and that is gone is an input of the link, which names it" + +# The same library, found by the link through `-L` in `[build] ldflags`. +mkdir -p ext/flag +"$NDKCC" --target=x86_64-linux-android24 -shared -fPIC -Wl,-soname,libext.so \ + -o ext/flag/libext.so ext/ext.c +sed -i 's|^link_library_dirs = \["ext/lib"\]$||' mcpp.toml +cat >> mcpp.toml < "gone-$fmt.log" 2>&1; then - fail "--format $fmt succeeded with libext.so gone" "gone-$fmt.log" + fail "--format $fmt succeeded with libext.so outside its directories" "gone-$fmt.log" fi grep -q "libext.so: found in none of" "gone-$fmt.log" \ || fail "--format $fmt did not name libext.so" "gone-$fmt.log" grep -q "incomplete" "gone-$fmt.log" \ || fail "--format $fmt did not say the closure is incomplete" "gone-$fmt.log" done -echo " ok: dir and tar refuse, naming the library that is gone" +echo " ok: dir and tar refuse, naming the library none of their directories holds" # ── 3. a dispatched format receives the incomplete closure ──────────────── cat > dist.sh <<'EOF' diff --git a/tests/e2e/build/basics.cpp b/tests/e2e/build/basics.cpp index 16c0a4d68..93e12a9e5 100644 --- a/tests/e2e/build/basics.cpp +++ b/tests/e2e/build/basics.cpp @@ -897,11 +897,12 @@ a = { path = "../a" } expect(s::count_lines(s::text(r), "compat.zlib+p") >= 2u) << "the refusal does not name both variants\n" << s::text(r); }; "952 a build interrupted in another process space continues"_test[s::gcc][s::elf] = [] { - // mcpp#952, design 2026-11-19 §10.3 C and D. A build killed inside a pid namespace of its - // own leaves admitted compiler units this process space can never observe. The next build - // continues, settles them as abandoned and says so once (`compiler-run/unobservable`, whose - // text names the process space); - // `mcpp clean` then leaves no generation of the configuration behind. + // mcpp#952, design 2026-11-19 §10.3 C and D, §10.6. A build killed inside a pid namespace of + // its own leaves admitted compiler units this process space can never observe. The next + // build continues. GCC's compiles wrote below attempts of their own, so they are settled + // without being observed and nothing is reported; a compiler that writes in place would be + // settled as abandoned and reported once (`compiler-run/unobservable`). `mcpp clean` then + // leaves no generation of the configuration behind. const auto unshare = s::on_path("unshare"); const auto timeout = s::on_path("timeout"); if (!unshare || !timeout || !e2e::succeeded(s::run(*unshare, {"-Urpf", "--mount-proc", "true"}))) @@ -937,12 +938,122 @@ a = { path = "../a" } if (!hit) skip("952 -- no kill landed while a compiler was admitted"); auto r = hit->mcpp({"build"}); expect(fatal(e2e::succeeded(r))) << "the build after the interrupted one was refused: " << s::text(r); - expect(s::count_lines(s::text(r), "cannot be observed from this process space") == 1u) - << "the abandoned units were not reported once: " << s::text(r); + expect(s::count_lines(s::text(r), "cannot be observed from this process space") == 0u) + << "a relocated attempt was observed: " << s::text(r); r = hit->mcpp({"clean"}); expect(fatal(e2e::succeeded(r))) << "mcpp clean failed: " << s::text(r); expect(s::find_files(hit->dir() / ".mcpp", "owner").empty()) << "a compiler-run generation survived mcpp clean"; }; + "952 a compiler that outlives its build writes only its own attempt"_test[s::gcc][s::elf] = [] { + // Design 2026-11-19 §10.6 (N1). A build is cut while two compiles run -- a module + // interface, compiled under the GCC schedule that publishes its BMI early, and a plain + // unit -- and its compilers, units of their own, keep running. Writing in place, the + // next build waited for them and was refused (`compiler-run-occupied`); writing below + // their attempts, the next build continues at once, and what the orphans write after it + // reaches nothing it published. + const auto bash = s::on_path("bash"), setsid = s::on_path("setsid"), pgrep = s::on_path("pgrep"); + if (!bash || !setsid || !pgrep || !s::on_path("pkill")) + skip("952 -- bash, setsid, pgrep and pkill are needed to cut a build and keep its compilers"); + // About 15 s of constant evaluation each on GCC 16.1. + auto spin = [](std::string_view name) { + return std::format("constexpr unsigned long long {}(unsigned long long x) {{\n" + " for (unsigned k = 0; k < 3000; ++k)\n" + " for (unsigned j = 0; j < 3000; ++j) x = x * 6364136223846793005ull + 1442695040888963407ull;\n" + " return x;\n}}\n", name); + }; + s::project p({{"mcpp.toml", "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[build]\n" + "cxxflags = [\"-fconstexpr-ops-limit=4294967296\"]\n\n" + "[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}}); + p.write("src/slow.cppm", "export module slow;\n" + spin("spin_m") + + "export unsigned long long slow_value() { constexpr auto v = spin_m(1); return v % 1000; }\n"); + p.write("src/plain.cpp", spin("spin_p") + "unsigned long long plain_value() { constexpr auto v = spin_p(2); return v % 1000; }\n"); + p.write("src/main.cpp", "import std;\nimport slow;\nunsigned long long plain_value();\n" + "int main() { std::println(\"{}\", slow_value() + plain_value()); }\n"); + // A file the builds published: below the build directory and not below an attempt. + auto published = [&](std::string_view name) { + std::vector out; + for (const auto& f : s::find_files(p.at("target"), name)) + if (!f.generic_string().contains("/.mcpp-attempts/")) out.push_back(f); + return out; + }; + auto app = [&] { + for (const auto& f : published("app")) + if (f.parent_path().filename() == "bin") return output_of(f); + return std::string{}; + }; + // The compiles of the two units (the auxiliary name `-dumpbase` is the compiles', not the + // scans'), of this project. + const auto running = std::format( + "n=0; for c in $(pgrep -f 'cc1plus .*-dumpbase (slow[.]m|plain)'); do " + "grep -q '{0}/' /proc/$c/cmdline 2>/dev/null && n=$((n+1)); done; echo $n", p.dir().string()); + // A helper whose build ended ends its compiler's unit; one killed outright cannot, and its + // compiler, the unit's, runs on. The helpers first, by their own ids, then the build's group. + const auto script = std::format( + "cd '{0}' && {1} bash -c 'echo $$ > build.pid; exec \"$0\" build' '{2}' > cut.log 2>&1 &\n" + "for i in $(seq 1 1200); do [ \"$({3})\" -ge 2 ] && break; sleep 0.1; done\n" + "sleep 1\n" + "pkill -KILL -f '^{0}/target/[^ ]*/mcpp (__compiler-cap|bmi-supervise|bmi-compile)'\n" + "kill -KILL -- -$(cat build.pid); wait; {3}\n", + p.dir().string(), setsid->string(), s::mcpp_path(), running); + auto cut = s::run(*bash, {"-c", script}); + if (std::atoi(cut.out.c_str()) < 2) skip("952 -- the compilers had ended when the build was cut"); + auto r = p.mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << "the build after the cut one was refused: " << s::text(r); + expect(!e2e::contains(s::text(r), "compiler-run-occupied")) << s::text(r); + expect(!e2e::contains(s::text(r), "cannot be observed")) << "a relocated attempt was observed: " << s::text(r); + const auto object = published(R"(plain\.o)"); + const auto bmi = published(R"(slow\.gcm)"); + expect(fatal(object.size() == 1u && bmi.size() == 1u)) << "the published object and BMI were not found"; + const auto objectBytes = s::read_file(object.front()), bmiBytes = s::read_file(bmi.front()); + const auto value = app(); + expect(fatal(!value.empty())) << "the program does not run"; + // The orphans end on their own; what they write after the build lands below their attempts, + // which the build removed. + for (int i = 0; i < 1800 && std::atoi(s::run(*bash, {"-c", running}).out.c_str()) > 0; ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + expect(eq(s::read_file(object.front()), objectBytes)) << "an orphan rewrote the published object"; + expect(eq(s::read_file(bmi.front()), bmiBytes)) << "an orphan rewrote the published BMI"; + r = p.mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << s::text(r); + expect(eq(app(), value)); + std::size_t attempts = 0; + for (const auto& f : s::find_files(p.at("target"), ".*")) + attempts += f.generic_string().contains("/.mcpp-attempts/"); + expect(eq(attempts, 0u)) << "an attempt's files outlived the builds"; + }; + "952 a clang compile names its auxiliary outputs after the published object"_test[s::llvm][s::elf] = [] { + // Design 2026-11-19 §10.6, P3. A clang driver that takes `-dumpdir`, which detection asks + // of it, compiles below an attempt as GCC does. Told the object's final name, it writes a + // split-DWARF file beside the published object, `..dwo`: none + // is left below an attempt, and a second build does nothing. + const s::options home{.where = s::home::inherited}; + s::project p({{"mcpp.toml", std::format("[package]\nname = \"aux\"\nversion = \"0.1.0\"\n" + "[toolchain]\ndefault = \"llvm@{}\"\n" + "[build]\ncxxflags = [\"-gsplit-dwarf\"]\n" + "[targets.aux]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n", + s::llvm_version())}, + {"src/helper.cpp", "int helper() { return 42; }\n"}, + {"src/main.cpp", "int helper();\nint main() { return helper() == 42 ? 0 : 1; }\n"}}); + auto r = p.mcpp({"build"}, home); + expect(fatal(e2e::succeeded(r))) << s::text(r); + std::string ninja; + for (const auto& f : s::find_files(p.at("target"), R"(build\.ninja)")) ninja += s::read_file(f); + expect(fatal(e2e::contains(ninja, "--relocate dumpdir"))) << "the driver's compiles were not relocated"; + const auto dwo = s::find_files(p.at("target"), R"(.*\.dwo)"); + expect(fatal(!dwo.empty())) << "no split-DWARF file was written"; + for (const auto& f : dwo) { + expect(!f.generic_string().contains("/.mcpp-attempts/")) << "a .dwo is below an attempt: " << f; + const auto name = f.filename().string(); + bool beside = false; + for (const auto& o : s::find_files(f.parent_path(), R"(.*\.o)")) + beside |= o.parent_path() == f.parent_path() && name.starts_with(o.stem().string() + "."); + expect(beside) << f << " is not named after an object beside it"; + } + r = p.mcpp({"build", "--verbose"}, home); + expect(fatal(e2e::succeeded(r))) << s::text(r); + expect(eq(s::count_matching(s::text(r), R"(^\[[0-9]+/)"), 0u)) << "the second build compiled again:\n" + << s::text(r); + }; }}; } // namespace diff --git a/tests/e2e/modules/programs.cpp b/tests/e2e/modules/programs.cpp index a4affed44..e8c8feaeb 100644 --- a/tests/e2e/modules/programs.cpp +++ b/tests/e2e/modules/programs.cpp @@ -103,7 +103,7 @@ main = "main.cpp" std::size_t nq = 0, nf = 0; for (const auto& f : s::find_files(a.at("app/target"), R"(boost\..*)")) { const auto g = f.generic_string(); - if (std::regex_search(g, std::regex{R"(\.cache/.*updater/boost\.)"})) ++nq; + if (std::regex_search(g, std::regex{R"(\.cache/.*updater/[0-9a-f]{8}/boost\.)"})) ++nq; if (std::regex_search(g, std::regex{R"(\.cache/boost\.)"})) ++nf; } expect(fatal(nq == 1u && nf == 1u)) << "A: expected the updater's BMI below its directory and the app's at its name, found " << nq << " and " << nf; @@ -158,17 +158,17 @@ main = "main.cpp" expect(fatal(failed(r))) << "F: one file twice in one program was accepted\n" << s::text(r); expect(fatal(e2e::contains(s::text(r), "one file is reached as two packages"))) << "F: the refusal does not say that one file is reached as two packages\n" << s::text(r); + // G: the root's own modules lie at their names, where every compiler looks unaided. A GCC + // compile that writes its BMI below its attempt names it through a map (design 2026-11-19 + // §10.6), and that map names each module at its name too. r = a.mcpp_in("updater", {"build"}); expect(fatal(e2e::succeeded(r))) << "G: a plain build failed\n" << s::text(r); - bool modmap = false; - std::error_code ec; - for (auto it = fs::recursive_directory_iterator(a.at("updater/target"), ec); it != fs::recursive_directory_iterator(); it.increment(ec)) - modmap = modmap || (it->is_directory() && it->path().filename() == "modmap"); - expect(fatal(!modmap)) << "G: a plan of the root's own modules wrote a module map"; auto graphs = s::find_files(a.at("updater/target"), R"(build\.ninja)"); std::ranges::sort(graphs); expect(fatal(!graphs.empty())) << "G: no build.ninja"; - expect(fatal(s::no_line(s::read_file(graphs.front()), R"(module-map|fmodule-mapper|cache/[^ ]*/boost\.)"))) << "G: a plan of the root's own modules binds a module name"; + expect(fatal(s::no_line(s::read_file(graphs.front()), R"(cache/[^ ]*/boost\.)"))) << "G: a plan of the root's own modules binds a module name below a directory"; + for (const auto& map : s::find_files(a.at("updater/target"), R"(.*\.map)")) + expect(fatal(s::no_line(s::read_file(map), R"(^boost [^ ]*/)"))) << "G: " << map << " binds the root's module below a directory"; }; "857 a host module several members import is compiled once"_test = [] { diff --git a/tests/e2e/pack/modes.cpp b/tests/e2e/pack/modes.cpp index 857b72377..6d44751df 100644 --- a/tests/e2e/pack/modes.cpp +++ b/tests/e2e/pack/modes.cpp @@ -551,28 +551,42 @@ int main() { std::puts("1-2-3"); return fw_anchor() == 41 && named_anchor() == 1 expect(fatal(e2e::succeeded(run))) << "the staged program does not run\n" << s::text(run); expect(fatal(s::has_line(s::text(run), "^1-2-3$"))) << "the staged program did not print 1-2-3\n" << s::text(run); - s::project ext{tmp / "ext", {{"ext.cpp", "int ext_value() { return 1; }\n"}}}; + s::project ext{tmp / "ext", {{"ext.c", "int ext_value(void) { return 1; }\n"}}}; fs::create_directories(ext.at("lib")); - auto gxx = e2e::run({.argv = {"g++", "-shared", "-fPIC", "-Wl,-soname,libext.so", "-o", ext.at("lib/libext.so").string(), ext.at("ext.cpp").string()}}); - expect(fatal(e2e::succeeded(gxx))) << "g++ failed\n" << s::text(gxx); + const auto cc = s::host_cc(); + expect(fatal(cc.has_value())) << "no host C compiler links a shared object"; + auto built = e2e::run({.argv = {cc->string(), "-shared", "-fPIC", "-Wl,-soname,libext.so", "-o", ext.at("lib/libext.so").string(), ext.at("ext.c").string()}}); + expect(fatal(e2e::succeeded(built))) << "the host C compiler failed\n" << s::text(built); const auto TMP = tmp.generic_string(); - s::project extapp{tmp / "extapp", {{"mcpp.toml", std::format(R"cpp([package] + auto extapp_manifest = [&](std::string_view runpath) { + return std::format(R"cpp([package] name = "extapp" version = "0.1.0" [toolchain] linux = "gcc@16.1.0" [build] -ldflags = ["-L{0}/ext/lib", "-lext", "-Wl,-rpath,{0}/ext/lib"] -)cpp", TMP)}, {"src/main.cpp", "int ext_value();\nint main() { return ext_value() == 1 ? 0 : 1; }\n"}}}; +ldflags = ["-L{0}/ext/lib", "-lext"{1}] +)cpp", TMP, runpath); + }; + s::project extapp{tmp / "extapp", {{"mcpp.toml", extapp_manifest(std::format(", \"-Wl,-rpath,{}/ext/lib\"", TMP))}, + {"src/main.cpp", "extern \"C\" int ext_value();\nint main() { return ext_value() == 1 ? 0 : 1; }\n"}}}; r = extapp.mcpp({"pack", "--pkg-format", "dir"}); expect(fatal(e2e::succeeded(r))) << "the pack with a prebuilt library failed\n" << s::text(r); const auto etree = dist_dir(extapp, "extapp-0.1.0-"); expect(fatal(etree.has_value())) << "no staged tree\n" << s::text(r); const fs::path emanifest = etree->string() + ".stage-manifest"; expect(fatal(has_exact_line(emanifest, "needs\tlibext.so\tlib/libext.so"))) << "libext.so is not stated as bundled\n" << s::read_file(emanifest); - fs::remove(ext.at("lib/libext.so")); + // A library the link read and that is gone is an input of the link that is gone (SPEC-004 + // §8.1, mcpp#931): the pack's build links again, and the link names it. + fs::rename(ext.at("lib/libext.so"), ext.at("lib/libext.so.away")); r = extapp.mcpp({"pack", "--pkg-format", "dir"}); expect(fatal(failed(r))) << "--format dir succeeded with libext.so gone\n" << s::text(r); + expect(e2e::contains(s::text(r), "-lext")) << "the link does not name the library that is gone\n" << s::text(r); + fs::rename(ext.at("lib/libext.so.away"), ext.at("lib/libext.so")); + // A library the link finds and no run path names: needed, and found by no loader. + extapp.write("mcpp.toml", extapp_manifest("")); + r = extapp.mcpp({"pack", "--pkg-format", "dir"}); + expect(fatal(failed(r))) << "--format dir succeeded with libext.so out of the loader's reach\n" << s::text(r); expect(e2e::contains(s::text(r), "libext.so: the loader finds no file for it")) << "the refusal does not name libext.so\n" << s::text(r); }; "1043 several programs and a dotted name are packed as stated"_test[s::elf][s::unix_shell] = [] { diff --git a/tests/e2e/run/runners.cpp b/tests/e2e/run/runners.cpp index ffb178c3b..d1f737199 100644 --- a/tests/e2e/run/runners.cpp +++ b/tests/e2e/run/runners.cpp @@ -15,10 +15,16 @@ namespace { bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } // `printf '\xff\xff' | dd of= bs=1 seek=18 conv=notrunc`: the ELF's e_machine spoiled. +// The program's time is kept: a link edge records the files it read with the time of its output +// (SPEC-004 §8.1), and an output newer than that record is one the build makes again. void spoil_machine(const fs::path& file) { - std::fstream f(file, std::ios::in | std::ios::out | std::ios::binary); - f.seekp(18); - f.write("\xff\xff", 2); + const auto time = fs::last_write_time(file); + { + std::fstream f(file, std::ios::in | std::ios::out | std::ios::binary); + f.seekp(18); + f.write("\xff\xff", 2); + } + fs::last_write_time(file, time); } // `od -An -tx1 -j18 -N2`: the two bytes at offset 18 are 0xff 0xff. diff --git a/tests/e2e/workspace/graph.cpp b/tests/e2e/workspace/graph.cpp index 1fae1aec4..506bb9e27 100644 --- a/tests/e2e/workspace/graph.cpp +++ b/tests/e2e/workspace/graph.cpp @@ -105,7 +105,7 @@ std::map, std::vector> units(st // A word with the package instance it names (`obj///`, and the module map whose // name hashes the instances it lists) replaced: what remains when only the instance differs. std::string without_instance(const std::string& word) { - static const std::regex instance{R"(/[0-9a-f]{8}/)"}, map{R"((modmap/[^/ ]+)-[0-9a-f]{8}\.)"}; + static const std::regex instance{R"([/\\][0-9a-f]{8}[/\\])"}, map{R"((modmap[/\\][^/\\ ]+)-[0-9a-f]{8}\.)"}; return std::regex_replace(std::regex_replace(word, instance, "//"), map, "$1-."); } diff --git a/tests/unit/test_compile_relocation.cpp b/tests/unit/test_compile_relocation.cpp new file mode 100644 index 000000000..bc4cbf206 --- /dev/null +++ b/tests/unit/test_compile_relocation.cpp @@ -0,0 +1,161 @@ +import std; +import mcpplibs.mtest; +import mcpp.build.schedule.relocation; + +using namespace mcpplibs::mtest; + +namespace rl = mcpp::build::schedule::relocation; +namespace fs = std::filesystem; + +namespace { +// A build directory of its own, entered for the case: the executor runs where +// ninja does, and spells the attempt's paths relative to it. +struct BuildDir { + fs::path root, before; + BuildDir() { + root = fs::temp_directory_path() / std::format("mcpp-relocation-{:08x}", std::random_device{}()); + fs::create_directories(root); + before = fs::current_path(); + fs::current_path(root); + } + ~BuildDir() { + std::error_code ec; + fs::current_path(before, ec); + fs::remove_all(root, ec); + } + static void write(const fs::path& p, std::string_view text) { + fs::create_directories(p.parent_path().empty() ? fs::path(".") : p.parent_path()); + std::ofstream(p, std::ios::binary | std::ios::trunc) << text; + } + static std::string read(const fs::path& p) { + std::ifstream in(p, std::ios::binary); + return {std::istreambuf_iterator(in), {}}; + } +}; +} // namespace + +UTest words_naming_an_output_are_written_below_the_attempt{"Relocation/WordsNamingAnOutputAreWrittenBelowTheAttempt", [] { + BuildDir d; + std::vector argv{"clang++", "-c", "src/x.cppm", "-o", "obj/p/x.m.o", "-MMD", "-MF", "obj/p/x.m.o.d.raw", + "@obj/p/x.m.o.rsp", "-fmodule-output=pcm.cache/p/x.pcm", "-Iobj/p/x.m.o.inc"}; + auto plan = rl::relocate(argv, {"obj/p/x.m.o", "obj/p/x.m.o.d", "pcm.cache/p/x.pcm", true}, "a1", "dumpdir"); + expect(fatal(bool(plan) && plan->relocated())); + const std::vector want{"clang++", "-c", "src/x.cppm", "-o", ".mcpp-attempts/a1/out/obj/p/x.m.o", "-MMD", "-MF", + ".mcpp-attempts/a1/out/obj/p/x.m.o.d.raw", "@obj/p/x.m.o.rsp", + "-fmodule-output=.mcpp-attempts/a1/out/pcm.cache/p/x.pcm", "-Iobj/p/x.m.o.inc", + "-dumpdir", "obj/p/x.m."}; + expect(eq(argv, want)); + expect(eq(plan->outputs.size(), 3u)); + // Kinds decide publication: an object and a BMI equal to the published one are kept. + expect(plan->outputs[0].kind == rl::Output::Kind::object); + expect(plan->outputs[1].kind == rl::Output::Kind::depfile); + expect(plan->outputs[2].kind == rl::Output::Kind::bmi); + expect(fs::is_directory(".mcpp-attempts/a1/out/obj/p")) << "the directory of the attempt's -o was not made"; +}}; + +UTest a_gcc_module_map_names_the_attempts_bmi{"Relocation/AGccModuleMapNamesTheAttemptsBmi", [] { + BuildDir d; + BuildDir::write("modmap/p-0011aabb.map", "$root gcm.cache\nx p/d1/x.gcm\ny y.gcm\n"); + std::vector argv{"g++", "-fmodule-mapper=modmap/p-0011aabb.map", "-c", "x.cppm", "-o", "obj/p/d1/x.m.o"}; + auto plan = rl::relocate(argv, {"obj/p/d1/x.m.o", "", "gcm.cache/p/d1/x.gcm", true}, "a2", "dumpbase"); + expect(fatal(bool(plan) && plan->relocated())); + expect(eq(argv[1], std::string("-fmodule-mapper=.mcpp-attempts/a2/module.map"))); + // Its own module below the attempt, relative to the repository; its imports where they are. + expect(eq(BuildDir::read(".mcpp-attempts/a2/module.map"), + std::string("$root gcm.cache\nx ../.mcpp-attempts/a2/out/gcm.cache/p/d1/x.gcm\ny y.gcm\n"))); + expect(eq(BuildDir::read("modmap/p-0011aabb.map"), std::string("$root gcm.cache\nx p/d1/x.gcm\ny y.gcm\n"))); + // The auxiliary outputs keep the names GCC derives from the final object. + const std::vector tail(argv.end() - 4, argv.end()); + expect(eq(tail, std::vector{"-dumpdir", "obj/p/d1/", "-dumpbase", "x.m"})); + expect(eq(rl::written_bmi(*plan).generic_string(), std::string(".mcpp-attempts/a2/out/gcm.cache/p/d1/x.gcm"))); +}}; + +UTest a_compile_that_cannot_be_relocated_stays_in_place{"Relocation/ACompileThatCannotBeRelocatedStaysInPlace", [] { + BuildDir d; + // An output outside the build directory, and a map that does not name the BMI: every + // output below the attempt or none, so the command is as it was. + const auto outside = (fs::temp_directory_path() / "elsewhere/x.o").string(); + std::vector argv{"g++", "-c", "x.cpp", "-o", outside}; + const auto before = argv; + auto plan = rl::relocate(argv, {outside, "", "", true}, "a3", "dumpbase"); + expect(fatal(bool(plan))); + expect(!plan->relocated()); + expect(eq(argv, before)); + BuildDir::write("modmap/q.map", "$root gcm.cache\nother other.gcm\n"); + std::vector mapped{"g++", "-fmodule-mapper=modmap/q.map", "-c", "x.cppm", "-o", "obj/x.o"}; + const auto beforeMapped = mapped; + plan = rl::relocate(mapped, {"obj/x.o", "", "gcm.cache/x.gcm", true}, "a4", "dumpbase"); + expect(fatal(bool(plan))); + expect(!plan->relocated()); + expect(eq(mapped, beforeMapped)); + expect(!fs::exists(".mcpp-attempts/a4")) << "a compile in place left an attempt directory"; +}}; + +UTest publication_keeps_an_equal_output_and_replaces_another{"Relocation/PublicationKeepsAnEqualOutputAndReplacesAnother", [] { + BuildDir d; + std::vector argv{"g++", "-c", "x.cpp", "-o", "obj/x.o", "-MMD", "-MF", "obj/x.o.d"}; + auto plan = rl::relocate(argv, {"obj/x.o", "obj/x.o.d", "", true}, "a5", ""); + expect(fatal(bool(plan) && plan->relocated())); + BuildDir::write("obj/x.o", "object bytes"); + const auto old = fs::last_write_time("obj/x.o") - std::chrono::hours(1); + fs::last_write_time("obj/x.o", old); + BuildDir::write(".mcpp-attempts/a5/out/obj/x.o", "object bytes"); + BuildDir::write(".mcpp-attempts/a5/out/obj/x.o.d", "obj/x.o: x.cpp\n"); + BuildDir::write(".mcpp-attempts/a5/out/obj/x.json", "{}"); // named after the object by the compiler + expect(fatal(bool(rl::publish(*plan)))); + // The equal object stays as it was, its time unchanged: `restat` ends the rebuild there. + expect(eq(BuildDir::read("obj/x.o"), std::string("object bytes"))); + expect(fs::last_write_time("obj/x.o") == old); + expect(eq(BuildDir::read("obj/x.o.d"), std::string("obj/x.o: x.cpp\n"))); + expect(eq(BuildDir::read("obj/x.json"), std::string("{}"))); + BuildDir::write(".mcpp-attempts/a5/out/obj/x.o", "other bytes"); + expect(fatal(bool(rl::publish(*plan)))); + expect(eq(BuildDir::read("obj/x.o"), std::string("other bytes"))); + rl::discard(plan->directory); + expect(!fs::exists(".mcpp-attempts/a5")); +}}; + +UTest a_shell_command_is_rewritten_word_by_word{"Relocation/AShellCommandIsRewrittenWordByWord", [] { + BuildDir d; + std::string command = "g++ -I\"/a b/obj/x.o\" -c x.cpp -o obj/x.o"; + auto plan = rl::relocate(command, {"obj/x.o", "", "", true}, "a6", "dumpbase"); + expect(fatal(bool(plan) && plan->relocated())); + expect(eq(command, std::string("g++ -I\"/a b/obj/x.o\" -c x.cpp -o .mcpp-attempts/a6/out/obj/x.o -dumpdir obj/ -dumpbase x"))); +}}; + +UTest a_saved_plan_reads_back{"Relocation/ASavedPlanReadsBack", [] { + BuildDir d; + std::vector argv{"g++", "-c", "x.cpp", "-o", "obj/x.o", "-MF", "obj/x.o.d"}; + auto plan = rl::relocate(argv, {"obj/x.o", "obj/x.o.d", "", true}, "a7", ""); + expect(fatal(bool(plan) && bool(rl::save(*plan)))); + auto loaded = rl::load(plan->directory); + expect(fatal(bool(loaded))); + expect(eq(loaded->outputs.size(), plan->outputs.size())); + for (std::size_t i = 0; i < plan->outputs.size() && i < loaded->outputs.size(); ++i) { + expect(loaded->outputs[i].kind == plan->outputs[i].kind); + expect(eq(loaded->outputs[i].published, plan->outputs[i].published)); + expect(eq(loaded->outputs[i].written, plan->outputs[i].written)); + } +}}; + +UTest a_published_dependency_file_names_what_is_published{"Relocation/APublishedDependencyFileNamesWhatIsPublished", [] { + BuildDir d; + // GCC lists its module mapper among a unit's prerequisites, and names the attempt's object as + // the target: published as they were, every later build found the attempt's map missing and + // compiled the unit again. + BuildDir::write("modmap/p.map", "$root gcm.cache\nx x.gcm\n"); + std::vector argv{"g++", "-fmodule-mapper=modmap/p.map", "-c", "x.cppm", "-o", "obj/x.m.o", + "-MMD", "-MF", "obj/x.m.o.d.raw"}; + auto plan = rl::relocate(argv, {"obj/x.m.o", "obj/x.m.o.d", "gcm.cache/x.gcm", true}, "a8", "dumpbase"); + expect(fatal(bool(plan) && plan->relocated())); + BuildDir::write(".mcpp-attempts/a8/out/obj/x.m.o.d.raw", + ".mcpp-attempts/a8/out/obj/x.m.o .mcpp-attempts/a8/out/gcm.cache/x.gcm: x.cppm \\\n" + " .mcpp-attempts/a8/module.map\n"); + BuildDir::write(".mcpp-attempts/a8/out/obj/x.m.o", "o"); + BuildDir::write(".mcpp-attempts/a8/out/gcm.cache/x.gcm", "b"); + expect(fatal(bool(rl::save(*plan)))); + auto loaded = rl::load(plan->directory); + expect(fatal(bool(loaded) && bool(rl::publish(*loaded)))); + expect(eq(BuildDir::read("obj/x.m.o.d.raw"), std::string("obj/x.m.o gcm.cache/x.gcm: x.cppm \\\n modmap/p.map\n"))); + expect(eq(BuildDir::read("gcm.cache/x.gcm"), std::string("b"))); +}}; diff --git a/tests/unit/test_compiler_transaction.cpp b/tests/unit/test_compiler_transaction.cpp index 414dd9347..1c301ba80 100644 --- a/tests/unit/test_compiler_transaction.cpp +++ b/tests/unit/test_compiler_transaction.cpp @@ -391,3 +391,42 @@ UTest failed_close_cannot_publish_or_acknowledge_a_record{"CompilerTransaction/F for(const auto& entry:fs::directory_iterator(t.root)) expect(!(entry.path().filename().string().starts_with(".mcpp-record-"))); }}; +// Design 2026-11-19 §10.6: an admitted attempt whose unit no observation answers (`unknown' +// here) holds recovery back while it writes in place, and is settled at once when it writes +// below an attempt of its own: what it did not publish is discarded, and its outputs are +// withdrawn so that their edges run again. +UTest a_relocated_attempt_is_settled_without_observation{"CompilerTransaction/ARelocatedAttemptIsSettledWithoutObservation", [] { + Transaction t;auto a=t.prepare();expect(fatal(bool(a.prepare_backup())));t.admitted(a); + std::ofstream(t.object)<<"previous object"; + expect(fatal(bool(tx::recover(a))));expect(!*tx::recover(a)) << "an unobserved in-place attempt was settled"; + const auto own=t.root/".mcpp-attempts/a1"; + fs::create_directories(own/"out");std::ofstream(own/"out/unit.o")<<"unpublished"; + expect(fatal(bool(a.relocate(own)))); + const auto loaded=tx::Attempt::load(a.directory);expect(fatal(bool(loaded))); + expect(bool(tx::validate(*loaded))); + auto settled=tx::recover(*loaded); + expect(fatal(bool(settled)));expect(*settled); + expect(!fs::exists(own)) << "the attempt's unpublished files remain"; + expect(!fs::exists(t.object)) << "an abandoned attempt's object was kept"; + expect(!fs::exists(t.reservation)); +}}; +// Design 2026-11-19 §10.3 C: an attempt that writes in place, whose unit was made in another +// process space (an `L2` reference names its space by number only, which openkal-linux 0.27 +// answers as another space), cannot be observed from here. It holds recovery back, counted +// as undecided, unless the caller states that the generation's locks are free; then it is +// settled as abandoned and counted as such. +UTest an_unobservable_unit_is_settled_when_the_caller_holds_the_locks{"CompilerTransaction/AnUnobservableUnitIsSettledWhenTheCallerHoldsTheLocks", [] { + if(!fs::exists("/proc/sys/kernel/random/boot_id"))skip("the reference of another process space is spelled for Linux"); + std::string boot=Transaction::contents("/proc/sys/kernel/random/boot_id"); + while(!boot.empty()&&(boot.back()=='\n'||boot.back()=='\r'))boot.pop_back(); + Transaction t;auto a=t.prepare();expect(fatal(bool(a.prepare_backup())));t.admitted(a); + const auto reference=std::format("L2;4242;1;{};pid:[1]",boot); + expect(fatal(bool(tx::publish(a.directory/"unit","mcpp-unit-v1",{reference,"1","1",tx::spelling(t.generation/"actors/helper.lock")})))); + tx::Recovery met; + auto held=tx::recover(a,nullptr,&met,false); + expect(fatal(bool(held)));expect(!*held);expect(eq(met.unknown, 1u));expect(eq(met.unobservable, 0u)); + met=tx::Recovery{}; + auto settled=tx::recover(a,nullptr,&met,true); + expect(fatal(bool(settled)));expect(*settled);expect(eq(met.unobservable, 1u)); + expect(eq(Transaction::contents(t.bmi), "verified old BMI")); +}}; diff --git a/tests/unit/test_module_address.cpp b/tests/unit/test_module_address.cpp index ec81cb261..8678615f6 100644 --- a/tests/unit/test_module_address.cpp +++ b/tests/unit/test_module_address.cpp @@ -200,12 +200,16 @@ UTest member_bmis_are_qualified_and_the_map_is_the_units_reach{"ModuleAddress/Me auto gcc = plan_core(t, mcpp::toolchain::CompilerId::GCC, true, &err); expect(fatal(bool(gcc))) << err; auto const& iface = gcc->core.at("core.cppm"); - expect(eq(iface.bmiFile, "core/corelib.gcm")); + // Below the package's instance directory, `/` (mcpp#953), + // one directory for every module of the instance. + const std::regex instance{R"(core/[0-9a-f]{8}/corelib\.gcm)"}; + expect(fatal(std::regex_match(iface.bmiFile, instance))) << iface.bmiFile; + const auto dir = std::filesystem::path(iface.bmiFile).parent_path().generic_string(); // GCC's map is rooted at the BMI directory, as its default mapper is, so // a BMI records its imports at the same paths whichever mapper made it. expect(iface.mapContent.starts_with("$root gcm.cache\n")) << iface.mapContent; - expect(ne(iface.mapContent.find("corelib core/corelib.gcm\n"), std::string::npos)); - expect(ne(iface.mapContent.find("m core/m.gcm\n"), std::string::npos)); + expect(ne(iface.mapContent.find("corelib " + dir + "/corelib.gcm\n"), std::string::npos)) << iface.mapContent; + expect(ne(iface.mapContent.find("m " + dir + "/m.gcm\n"), std::string::npos)) << iface.mapContent; expect(ne(iface.mapContent.find("std std.gcm\n"), std::string::npos)); expect(eq(iface.mapContent.find("tool"), std::string::npos)) << "the map names a module the unit does not reach:\n" << iface.mapContent; @@ -216,12 +220,13 @@ UTest member_bmis_are_qualified_and_the_map_is_the_units_reach{"ModuleAddress/Me auto clang = plan_core(t, mcpp::toolchain::CompilerId::Clang, true, &err); expect(fatal(bool(clang))) << err; auto const& ci = clang->core.at("core.cppm"); - expect(eq(ci.bmiFile, "core/corelib.pcm")); + expect(fatal(std::regex_match(ci.bmiFile, std::regex{R"(core/[0-9a-f]{8}/corelib\.pcm)"}))) << ci.bmiFile; + const auto cdir = std::filesystem::path(ci.bmiFile).parent_path().generic_string(); // clang reads the imported module's BMI from the argument file; its own // module is written through -fmodule-output and is not listed there. expect(fatal(eq(ci.arguments.size(), 1u))); expect(ci.arguments.front().starts_with("-fmodule-file=m=")); - expect(ci.arguments.front().ends_with("pcm.cache/core/m.pcm")) << ci.arguments.front(); + expect(ci.arguments.front().ends_with("pcm.cache/" + cdir + "/m.pcm")) << ci.arguments.front(); expect(fatal(!ci.flags.empty())); expect(ci.flags.back().find(".modmap") != std::string::npos) << ci.flags.back(); expect(eq(ci.flags.back().front(), '@')); @@ -229,7 +234,10 @@ UTest member_bmis_are_qualified_and_the_map_is_the_units_reach{"ModuleAddress/Me // The root package's modules stay at their names, and a unit that reaches only // those and `std` is told nothing: a project without dependency modules is laid -// out, and every command spelled, as before. +// out, and every command spelled, as before. A GCC unit that provides a module +// is given a map all the same: a relocated compile names its own BMI through a +// copy of the map (design 2026-11-19 §10.6), and GCC's mapper answers only the +// names it lists, so the map lists each at its name. UTest root_modules_stay_flat_and_need_no_map{"ModuleAddress/RootModulesStayFlatAndNeedNoMap", [] { Tmp t; const auto proj = t.path / "proj"; @@ -254,12 +262,19 @@ UTest root_modules_stay_flat_and_need_no_map{"ModuleAddress/RootModulesStayFlatA auto plan = make_plan(root, toolchain(id), {}, graph, topo, packages, proj, proj / "target" / "t", {}, {}, {}); if (auto&& failed = expect(fatal(bool(plan))); !failed) failed << plan.error(); - expect(plan->moduleScopes.empty()); + const bool gcc = id == mcpp::toolchain::CompilerId::GCC; for (auto const& cu : plan->compileUnits) { - expect(cu.moduleScope.empty()) << cu.source; + expect(eq(cu.moduleScope.empty(), !(gcc && !cu.providesModule.empty()))) << cu.source; if (!cu.providesModule.empty()) expect(eq(cu.bmiFile.find('/'), std::string::npos)) << cu.bmiFile; } + if (!gcc) { + expect(plan->moduleScopes.empty()); + continue; + } + expect(fatal(eq(plan->moduleScopes.size(), 1u))); + expect(eq(plan->moduleScopes.begin()->second.content, + std::string("$root gcm.cache\nm m.gcm\nstd std.gcm\nstd.compat std.compat.gcm\n"))); } }}; diff --git a/tests/unit/test_ninja_backend.cpp b/tests/unit/test_ninja_backend.cpp index a680eb100..bd894e5b0 100644 --- a/tests/unit/test_ninja_backend.cpp +++ b/tests/unit/test_ninja_backend.cpp @@ -159,14 +159,21 @@ UTest ordinary_compilers_share_the_detached_schedule_bound{"NinjaBackend/Ordinar return start == std::string::npos ? std::string{} : text.substr(start, text.find("\n\n", start) - start); }; const std::string executor = "__compiler-cap --sem .mcpp-sched --output $out --depfile $out.d"; - for (const auto name : {"asm_object", "asm_object_raw", "nasm_object", "cxx_scan"}) - expect(ne(rule(name).find(executor + " -- "), std::string::npos)) << name; + // GCC 16 writes its outputs below an attempt of the compile's own, which the + // executor publishes (design 2026-11-19 §10.6): the driver's compiles, its + // assembler and its scans, a scan with no auxiliary outputs to name. NASM, + // not measured, writes in place. + for (const auto name : {"asm_object", "asm_object_raw"}) + expect(ne(rule(name).find(executor + " --relocate dumpbase -- "), std::string::npos)) << name; + expect(ne(rule("cxx_scan").find(executor + " --relocate none -- "), std::string::npos)); + expect(ne(rule("nasm_object").find(executor + " -- "), std::string::npos)); // A C or C++ compile names its output as an object the executor keeps when // the bytes are the ones it had; the module rule names its BMI as well // (SPEC-018 §1). No rule keeps a file by a shell fragment of its command. for (const auto name : {"cxx_object", "c_object"}) - expect(ne(rule(name).find(executor + " --object -- "), std::string::npos)) << name; - expect(ne(rule("cxx_module").find(executor + " --object --bmi \"$bmi_out\" -- "), std::string::npos)); + expect(ne(rule(name).find(executor + " --object --relocate dumpbase -- "), std::string::npos)) << name; + expect(ne(rule("cxx_module").find(executor + " --object --bmi \"$bmi_out\" --relocate dumpbase -- "), + std::string::npos)); expect(eq(text.find(".bak"), std::string::npos)); expect(eq(text.find("bmi-equal"), std::string::npos)); // The generation carries the cap: the graph names no number of compilers, @@ -1497,6 +1504,50 @@ UTest object_only_package_builds_own_objects_alongside_cache_fills{"NinjaBackend expect(eq(count_occurrences(defaults, " obj/dep.o"), 1u)) << defaults; }}; +// A cache fill is its package's every artifact. Under the two-phase shape a +// module's BMI is an edge its object does not wait for, so the fill asks for +// the BMI by name: a module no unit of the build imported was never compiled to +// a BMI, and the fill was refused for it (mcpp's own clang build, `openkal.time`). +UTest a_cache_fill_asks_for_every_bmi_of_its_package{"NinjaBackend/ACacheFillAsksForEveryBmiOfItsPackage", [] { + auto plan = minimal_plan(); + plan.toolchain.compiler = mcpp::toolchain::CompilerId::Clang; + plan.toolchain.version = "23.1.3"; + plan.toolchain.binaryPath = "/usr/bin/clang++"; + plan.scanDepsPath = "/usr/bin/clang-scan-deps"; + plan.scheduleTag = "two-phase"; + plan.compileUnits.push_back({ + .source = "/store/dep/src/time.cppm", + .kind = mcpp::SourceKind::ModuleInterface, + .object = "obj/dep/0011aabb/src/time.m.o", + .packageName = "dep", + .providesModule = "dep.time", + .bmiFile = "dep/0011aabb/dep.time.pcm", + .fillsCache = true, + }); + plan.compileUnits.push_back({ + .source = "src/main.cpp", + .kind = mcpp::SourceKind::Cxx, + .object = "obj/main.o", + .packageName = "objc_rule_test", + }); + plan.linkUnits.push_back({ + .targetName = "app", + .kind = mcpp::build::LinkUnit::Binary, + .objects = {"obj/main.o"}, + .output = "bin/app", + .entryMain = "src/main.cpp", + }); + + auto ninja = emit_ninja_string(plan); + expect(fatal(ne(ninja.find("build pcm.cache/dep/0011aabb/dep.time.pcm : cxx_precompile"), std::string::npos))) + << "the fixture is not the two-phase shape\n" << ninja; + auto at = ninja.rfind("\ndefault "); + expect(fatal(ne(at, std::string::npos))) << ninja; + auto defaults = ninja.substr(at, ninja.find('\n', at + 1) - at); + expect(ne(defaults.find(" obj/dep/0011aabb/src/time.m.o"), std::string::npos)) << defaults; + expect(ne(defaults.find(" pcm.cache/dep/0011aabb/dep.time.pcm"), std::string::npos)) << defaults; +}}; + // A cached unit whose cachedObject was never filled in is a bug upstream, not a // reason to emit a stage edge with an empty input — it must fall back to being // compiled rather than producing an unbuildable graph. From 174e06461ba26848335871017bde20d453eb17a6 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 18:43:46 +0900 Subject: [PATCH 12/30] A project that names its own link dependency file keeps it, and the build says what the link's record lacks (mcpp#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GNU ld and ld.lld write one dependency file, the last named, and the engine's option followed the project's [build] ldflags: a project that asked for --dependency-file got no file, with nothing said (found by the design record's reproduction 931d). When the project's link words name one, the engine adds no option of its own and reports build/link-report: the link's record then holds only the files its words name. SPEC-004 §8.1 rule 2, docs/30. --- CHANGELOG.md | 5 ++++- docs/30-build-mcpp.md | 5 ++++- docs/specs/manifest-semantics.md | 6 ++++-- docs/zh/30-build-mcpp.md | 4 +++- src/build/flags.cppm | 20 ++++++++++++++++++-- tests/e2e/build/link.cpp | 18 ++++++++++++++++++ 6 files changed, 51 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51a35695b..d1db2ebd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,7 +121,10 @@ compile edge records the compiler's report, so a library found through `-L`/`-l` relinks the program when it changes, and one that is gone links again and is named by the linker. As for a compile, a program changed outside the build is linked again. Whether a linker accepts - the option is measured once and kept in the probe cache. + the option is measured once and kept in the probe cache. A project whose `[build] ldflags` + name `--dependency-file` keeps that file, the linker writing only one; mcpp adds no option of + its own, and the build reports that the link's record holds only the files its words name + (`build/link-report`). - Every member of a workspace whose root states `[workspace.target..abi]` was warned that it declares `[target..abi]`, which only the root decides: a build of about 60 members for wasm32-emscripten printed the warning 262 times (mcpp#954). A diff --git a/docs/30-build-mcpp.md b/docs/30-build-mcpp.md index ac0a46817..2ff08bb04 100644 --- a/docs/30-build-mcpp.md +++ b/docs/30-build-mcpp.md @@ -801,7 +801,10 @@ sources (SPEC-004 §8.1): records the files the linker read, as a compile edge records the headers the compiler read: libraries found through `-L`/`-l` and files a linker script pulls in with `INPUT` are covered. Whether the linker accepts the option is - measured once per linker and kept in the probe cache. + measured once per linker and kept in the probe cache. When a project's + `[build] ldflags` name `--dependency-file`, the linker writes that file, as + it writes only one, and mcpp adds no option of its own: those links have + rule 1 only, and the build reports it (`build/link-report`). A relative path and a file that neither exists when the build is planned nor is declared by an action are not inputs, unless the linker reports them. diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index fb334ca86..dc27c8d05 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -572,8 +572,10 @@ mcpp 2026.9.26.2,#703)。** 文件(ninja `depfile`、`deps = gcc`)。链接器是否接受由一次实测回答:经编译器驱动时链接一个 空的共享对象,直接运行链接器时询问 `--version`;所用的词是链接行中决定链接器的词(`--target`、 `-fuse-ld`、`--ld-path`、`-B`、驱动配置、`-flavor`),答案按这些词与它们点名的文件保存。 - 报告覆盖按 `-l` 搜索找到的库与链接脚本中 `INPUT` 引入的文件。其余链接器(ld64、wasm-ld、 - lld-link、link.exe)只有第 1 条。 + 报告覆盖按 `-l` 搜索找到的库与链接脚本中 `INPUT` 引入的文件。工程的 `[build] ldflags` 自己写出 + `--dependency-file` 时,链接器只写一个文件(GNU ld 与 ld.lld 取最后写出的那个),链接保留工程的 + 文件,引擎**禁止**再加入自己的选项;这些链接只有第 1 条,构建**必须**以 `build/link-report` + 报告降级。其余链接器(ld64、wasm-ld、lld-link、link.exe)只有第 1 条。 改写一个输入使携带它的链接重新运行;没有输入变化的构建不运行链接。 diff --git a/docs/zh/30-build-mcpp.md b/docs/zh/30-build-mcpp.md index 1b1cc3d35..f5387d7fa 100644 --- a/docs/zh/30-build-mcpp.md +++ b/docs/zh/30-build-mcpp.md @@ -671,7 +671,9 @@ stamp 不会喂给任何编译或链接边,只喂给下文说的那条 order-o 2. **链接器的报告。** 目标为 ELF、且链接器接受 `--dependency-file=<文件>`(GNU ld 2.35 起、gold、 ld.lld)时,链接边记录链接器读取的文件,与编译边记录编译器读取的头文件相同:经 `-L`、`-l` 找到的库与链接脚本以 `INPUT` 引入的文件都在其中。链接器是否接受该选项按链接器实测一次, - 答案保存在探测缓存中。 + 答案保存在探测缓存中。工程的 `[build] ldflags` 写出 `--dependency-file` 时,链接器只写一个 + 文件,写出的是工程的文件,mcpp 不再加入自己的选项;这些链接只有第 1 条,构建报告这一点 + (`build/link-report`)。 相对路径,以及规划时既不存在、也不由动作声明的文件,不是输入,除非链接器报告了它。 diff --git a/src/build/flags.cppm b/src/build/flags.cppm index 3c52a92d3..ef18926cf 100644 --- a/src/build/flags.cppm +++ b/src/build/flags.cppm @@ -2863,9 +2863,25 @@ CompileFlags compute_flags_impl(const BuildPlan& plan, bool packageLinkFlags) { const bool direct = !f.ldDriver.empty(); std::vector argv { direct ? f.ldDriver.string() : f.cxxBinary.string() }; for (auto const& w : link_selection_words(f.ld)) argv.push_back(w); + // A link that names a dependency file of its own keeps it: GNU ld and + // ld.lld write one file, the last named, and the engine's option would + // follow the project's words. The engine then asks for no report, and + // the link's record holds only the files its words name. + const bool own = std::ranges::any_of(linkWords, [](const std::string& w) { + return w.contains("--dependency-file"); + }); if (mcpp::toolchain::linker_reports_inputs(argv, direct, - mcpp::toolchain::compiler_env(plan.toolchain))) - f.linkReport = direct ? " --dependency-file=$out.d" : " -Wl,--dependency-file=$out.d"; + mcpp::toolchain::compiler_env(plan.toolchain))) { + if (!own) + f.linkReport = direct ? " --dependency-file=$out.d" : " -Wl,--dependency-file=$out.d"; + else + mcpp::diag::degraded("build/link-report", + "[build] ldflags names its own --dependency-file, so the link records no report " + "of what the linker read", + "a library the linker finds by searching (-L, -l) relinks the program only when a " + "word of the link names it", + "remove --dependency-file from [build] ldflags to have mcpp record the link's inputs"); + } } return f; diff --git a/tests/e2e/build/link.cpp b/tests/e2e/build/link.cpp index 3e006375d..a89ff98e2 100644 --- a/tests/e2e/build/link.cpp +++ b/tests/e2e/build/link.cpp @@ -503,6 +503,24 @@ int main() { std::println("val {}", val()); } r = app.mcpp({"run"}); expect(fatal(e2e::succeeded(r))) << "the second run\n" << s::text(r); expect(fatal(e2e::contains(s::text(r), "val 42"))) << "a replaced library found through -L/-l did not relink the program\n" << s::text(r); + // A link that names its own dependency file keeps it, the linker writing the last one + // named, and the build says what its record then lacks. + app.write("mcpp.toml", std::format(R"cpp([package] +name = "app" +version = "0.1.0" + +[build] +ldflags = ["-L{0}/prebuilt", "-lval", "-Wl,--dependency-file={0}/own.d"] + +[targets.app] +kind = "bin" +main = "src/main.cpp" +)cpp", work_s)); + r = app.mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << "the build with the link's own dependency file\n" << s::text(r); + expect(fs::is_regular_file(work / "own.d")) << "the link's own dependency file was not written\n" << s::text(r); + expect(e2e::contains(s::text(r), "names its own --dependency-file")) + << "the build did not report the link's own dependency file\n" << s::text(r); }; "1062 link flags on a compiler link are named not dropped"_test[s::elf][s::llvm] = [] { // [platform.link] flags a compiler-driven link does not use are named with the remedy From 039d350d59857d90051b50beb8cde7425877a5e6 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 19:26:44 +0900 Subject: [PATCH 13/30] A published output survives a transient hold on Windows, and a query asks the driver without creating a home On a Windows runner a GCC BMI published from a cross build was refused once among some 350 ("being used by another process"): a scanner opens a compile's output the moment it is written, and Windows refuses a rename while the file is held. Publication retries such a refusal for up to 30 s (mcpp::home::replace_file_within). The -dumpdir probe ran at detection, which a query (self env) reaches too, and created the home's probe cache: a query now asks in a temporary directory and keeps no answer. The mtest form of e2e 667 follows the link record as the script did (mcpp#931), 669 accepts the runtime-closure refusal where the program's loader is private, and the response-file check of LinkWordInputs names what it read. --- src/build/schedule/relocation.cppm | 11 ++++++++--- src/home/files.cpp | 7 ++++++- src/home/files.cppm | 4 ++++ src/toolchain/probe.cppm | 14 ++++++++++---- tests/e2e/android/rows.cpp | 21 +++++++++++++++++---- tests/e2e/pack/modes.cpp | 8 ++++++-- tests/unit/test_build_flags.cpp | 16 ++++++++++++++-- 7 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/build/schedule/relocation.cppm b/src/build/schedule/relocation.cppm index d98bffd0e..84c4eabd8 100644 --- a/src/build/schedule/relocation.cppm +++ b/src/build/schedule/relocation.cppm @@ -32,6 +32,7 @@ export module mcpp.build.schedule.relocation; import std; import mcpp.buildopt.keep; +import mcpp.home.files; export namespace mcpp::build::schedule::relocation { @@ -295,12 +296,16 @@ std::string published_text(const Plan& plan, std::string text) { } // Moves `from` over `to`: one rename, so a reader sees the old file or the -// new one and never a part of either. +// new one and never a part of either. Windows refuses the rename while another +// program holds the file, and a virus scanner opens a compile's output the +// moment it is written: a GCC BMI published from a cross build on a Windows +// runner was refused once among some 350 ("being used by another process"). +// The refusal is retried for up to 30 s, longer than a scan of the largest +// output takes. std::expected replace(const fs::path& from, const fs::path& to) { std::error_code ec; fs::create_directories(to.parent_path(), ec); - fs::rename(from, to, ec); - if (!ec) return {}; + if (mcpp::home::replace_file_within(from, to, ec, std::chrono::seconds{30})) return {}; return std::unexpected(std::format("cannot publish {} as {}: {}", from.generic_string(), to.generic_string(), ec.message())); } diff --git a/src/home/files.cpp b/src/home/files.cpp index af4f14c31..1af3e8306 100644 --- a/src/home/files.cpp +++ b/src/home/files.cpp @@ -124,7 +124,12 @@ bool replace_file(const std::filesystem::path& source, const std::filesystem::pa // the time: programs started at once on a loaded Windows runner each read // config.toml, and a writer that tried a dozen times in two seconds found // it held at each (unit Config/ConfigValueIsReplacedWhole, 2026-11-14). - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{2}; + return replace_file_within(source, destination, ec, std::chrono::seconds{2}); +} + +bool replace_file_within(const std::filesystem::path& source, const std::filesystem::path& destination, + std::error_code& ec, std::chrono::milliseconds patience) { + const auto deadline = std::chrono::steady_clock::now() + patience; auto delay = std::chrono::milliseconds{1}; for (;;) { const auto r = mcpplibs::os::fs::rename(source, destination, true); diff --git a/src/home/files.cppm b/src/home/files.cppm index b3b50e2fc..689f493f5 100644 --- a/src/home/files.cppm +++ b/src/home/files.cppm @@ -93,6 +93,10 @@ private: // seconds. bool replace_file(const std::filesystem::path& source, const std::filesystem::path& destination, std::error_code& ec); +// The same, retried for up to `patience`: a compile's output, which a virus +// scanner opens the moment it is written (design 2026-11-19 §10.6). +bool replace_file_within(const std::filesystem::path& source, const std::filesystem::path& destination, + std::error_code& ec, std::chrono::milliseconds patience); // `text` written beside `file` under a name of this writer's own and renamed over // it (replace_file): a process reading the file while this one writes it sees the diff --git a/src/toolchain/probe.cppm b/src/toolchain/probe.cppm index ad2b55b0a..21ac57e3b 100644 --- a/src/toolchain/probe.cppm +++ b/src/toolchain/probe.cppm @@ -312,13 +312,19 @@ bool driver_takes_dumpdir(const std::filesystem::path& compiler, const mcpp::run const auto key = probe_cache_key({compiler.string(), "-dumpdir"}, env, true); std::lock_guard lock(mutex); if (auto it = answered.find(key); it != answered.end()) return it->second; - const auto work = mcpp::home::root() / "cache" / "probe" / "dumpdir"; - const auto answer = work / (mcpp::toolchain::hash_string(key) + ".answer"); + const auto kept = mcpp::home::root() / "cache" / "probe" / "dumpdir"; + const auto answer = kept / (mcpp::toolchain::hash_string(key) + ".answer"); if (std::ifstream in(answer); in) { std::string v; in >> v; if (v == "yes" || v == "no") return answered[key] = (v == "yes"); } + // Asked, by a query too (`self env`, `toolchain list`), in a directory of + // its own; only a run that writes the global cache keeps the answer, and a + // cold home is not created for a query. + const bool keep = mcpp::effects::writes_global_cache(); + const auto work = keep ? kept + : std::filesystem::temp_directory_path() / std::format("mcpp-dumpdir-{:x}", std::random_device{}()); std::error_code ec; std::filesystem::create_directories(work, ec); const auto src = work / "empty.c"; @@ -333,11 +339,11 @@ bool driver_takes_dumpdir(const std::filesystem::path& compiler, const mcpp::run // names it: such a driver is not taken as naming its outputs from it. const bool yes = r.exit_code == 0 && r.output.find("dumpdir") == std::string::npos; std::filesystem::remove(object, ec); + if (!keep) std::filesystem::remove_all(work, ec); mcpp::log::verbose("toolchain", std::format("'{}' takes -dumpdir: {}{}", compiler.string(), yes ? "yes" : "no", yes ? std::string{} : " (" + first_line_of(r.output) + ")")); // An answer the driver did not give (it did not start) is not kept. - if (mcpp::effects::writes_global_cache() && (yes || !r.output.empty())) - std::ofstream(answer) << (yes ? "yes\n" : "no\n"); + if (keep && (yes || !r.output.empty())) std::ofstream(answer) << (yes ? "yes\n" : "no\n"); return answered[key] = yes; } diff --git a/tests/e2e/android/rows.cpp b/tests/e2e/android/rows.cpp index 37de06a80..86c7bb79d 100644 --- a/tests/e2e/android/rows.cpp +++ b/tests/e2e/android/rows.cpp @@ -290,9 +290,12 @@ int main() { // An app with a shared path dependency: lib/ holds the object, libfw.so and // libc++_shared.so, the stubs are the platform's (the stage manifest says so); the // libraries carry their file names as SONAME; a prebuilt library from [runtime] is - // staged, and when it is gone dir and tar refuse naming it; a dispatched format receives - // `closure = not-walked`, the unresolved name and the members; two triples stage a - // closure each. + // staged. A library the link read and that is gone is an input of the link that is gone + // (SPEC-004 §8.1, mcpp#931): the pack's build links again, and the link names it. A + // library the link found through a flag the pack does not read is needed and found in + // none of the pack's directories: dir and tar refuse naming it, and a dispatched format + // receives `closure = not-walked`, the unresolved name and the members; two triples + // stage a closure each. const auto tmp = s::case_dir(); s::project fw(tmp / "fw", {{"mcpp.toml", "[package]\nnamespace = \"demo\"\nname = \"fw\"\nversion = \"0.1.0\"\n\n[targets.fw]\nkind = \"lib\"\n"}, {"src/fw.cppm", "export module fw;\nexport int fw_anchor() { return 41; }\n"}}); @@ -325,9 +328,19 @@ int main() { expect(fatal(fs::is_regular_file(tree / "lib/libext.so"))) << "the prebuilt libext.so is not staged"; expect(fatal(lines_contain(s::read_file(stage_manifest), "needs\tlibext.so\tlib/libext.so"))) << "the manifest does not state libext.so as staged: " << s::read_file(stage_manifest); fs::remove(app.at("ext/lib/libext.so")); + r = app.mcpp({"pack", "--target", "x86_64-linux-android", "--pkg-format", "dir"}); + expect(fatal(failed(r))) << "the pack succeeded with libext.so gone"; + expect(fatal(e2e::contains(s::text(r), "-lext"))) << "the link did not name the library that is gone: " << s::text(r); + // The same library, found by the link through `-L` in `[build] ldflags`. + fs::create_directories(app.at("ext/flag")); + const auto flagged = s::run(cc, {"--target=x86_64-linux-android24", "-shared", "-fPIC", "-Wl,-soname,libext.so", "-o", app.at("ext/flag/libext.so").string(), app.at("ext/ext.c").string()}, app.dir()); + expect(fatal(e2e::succeeded(flagged))) << "the NDK's clang did not build libext.so: " << s::text(flagged); + app.write("mcpp.toml", manifest + std::format("\n[target.'cfg(env = \"android\")'.runtime]\nlibraries = [\"ext\"]\n" + "\n[target.'cfg(env = \"android\")'.build]\nldflags = [\"-L{}\"]\n", + app.at("ext/flag").generic_string())); for (std::string_view fmt : {"dir", "tar"}) { r = app.mcpp({"pack", "--target", "x86_64-linux-android", "--pkg-format", std::string(fmt)}); - expect(fatal(failed(r))) << "--pkg-format " << fmt << " succeeded with libext.so gone"; + expect(fatal(failed(r))) << "--pkg-format " << fmt << " succeeded with libext.so outside its directories"; expect(fatal(e2e::contains(s::text(r), "libext.so: found in none of"))) << "--pkg-format " << fmt << " did not name libext.so: " << s::text(r); expect(fatal(e2e::contains(s::text(r), "incomplete"))) << "--pkg-format " << fmt << " did not say the closure is incomplete: " << s::text(r); } diff --git a/tests/e2e/pack/modes.cpp b/tests/e2e/pack/modes.cpp index 6d44751df..00ee4d939 100644 --- a/tests/e2e/pack/modes.cpp +++ b/tests/e2e/pack/modes.cpp @@ -583,11 +583,15 @@ ldflags = ["-L{0}/ext/lib", "-lext"{1}] expect(fatal(failed(r))) << "--format dir succeeded with libext.so gone\n" << s::text(r); expect(e2e::contains(s::text(r), "-lext")) << "the link does not name the library that is gone\n" << s::text(r); fs::rename(ext.at("lib/libext.so.away"), ext.at("lib/libext.so")); - // A library the link finds and no run path names: needed, and found by no loader. + // A library the link finds and no run path names: needed, and found by no loader. The + // pack refuses it; where the program's interpreter is a private loader (a toolchain + // payload's), the build refuses it first, by its runtime closure. Each names it. extapp.write("mcpp.toml", extapp_manifest("")); r = extapp.mcpp({"pack", "--pkg-format", "dir"}); expect(fatal(failed(r))) << "--format dir succeeded with libext.so out of the loader's reach\n" << s::text(r); - expect(e2e::contains(s::text(r), "libext.so: the loader finds no file for it")) << "the refusal does not name libext.so\n" << s::text(r); + expect(e2e::contains(s::text(r), "libext.so: the loader finds no file for it") + || e2e::contains(s::text(r), "libext.so not found on the search path this artifact will actually use")) + << "the refusal does not name libext.so\n" << s::text(r); }; "1043 several programs and a dotted name are packed as stated"_test[s::elf][s::unix_shell] = [] { // Several programs, none named after the package: the first by name is packed and the diff --git a/tests/unit/test_build_flags.cpp b/tests/unit/test_build_flags.cpp index 6d71c6f0b..c6df10ebe 100644 --- a/tests/unit/test_build_flags.cpp +++ b/tests/unit/test_build_flags.cpp @@ -353,14 +353,26 @@ UTest every_place_a_word_carries_a_path_is_a_link_input{"LinkWordInputs/EveryPla const auto a = dir.path / "a.a", m = dir.path / "v.map", js = dir.path / "lib.js", rsp = dir.path / "libs.rsp"; touch(a); touch(m); touch(js); - std::ofstream(rsp) << a.string() << "\n"; + { + std::ofstream out(rsp, std::ios::binary); + out << a.string() << "\n"; + expect(fatal(bool(out.flush()))) << "the response file was not written: " << rsp; + } const std::set none; using mcpp::build::link_word_inputs; + // The paths a word gave, for a failure to name (one failed once on a CI + // runner and not again with the same seed). + auto named = [](const std::vector& paths) { + std::string out; + for (const auto& p : paths) out += "\n " + p.string(); + return out.empty() ? std::string("\n (none)") : out; + }; expect(link_word_inputs(a.string(), false, none) == std::vector{a}); expect(link_word_inputs("--js-library=" + js.string(), false, none) == std::vector{js}); expect(link_word_inputs("-Wl,--version-script=" + m.string(), false, none) == std::vector{m}); expect(link_word_inputs("-Wl,-force_load," + a.string(), false, none) == std::vector{a}); - expect(link_word_inputs("@" + rsp.string(), false, none) == std::vector{rsp, a}); + const auto viaResponseFile = link_word_inputs("@" + rsp.string(), false, none); + expect(viaResponseFile == std::vector{rsp, a}) << "@" << rsp.string() << " gave:" << named(viaResponseFile); }}; UTest a_directory_or_a_relative_path_is_no_link_input{"LinkWordInputs/ADirectoryOrARelativePathIsNone", [] { From 814346802a75b7bceb9c1d760e1d60e30605fe0a Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 20:01:14 +0900 Subject: [PATCH 14/30] The default platform is openkal-platform 0.31.0, and the tests run under mtest 0.10.0 openkal-platform 0.31.0 states the openkal 0.26 family: a start bounded by the environment alone, an observation of a unit that answers whether this process space can observe it, and a runner that names the call each task of an image waits in when it is asked to end. mcpp's own tests and the CI runners use mtest 0.10.0, which records a timed-out program's processes before it ends them. The design record states the measurements of the replayed issues and two findings of the implementation: a transient hold on a published file on Windows, and a probe that made a home. --- ...11-19-ecosystem-rule-convergence-design.md | 11 +++++++++- .github/versions.env | 4 ++-- CHANGELOG.md | 7 +++++++ bench/mcpp.toml | 2 +- docs/05-dependencies.md | 4 ++-- docs/50-machine-output.md | 2 +- docs/zh/05-dependencies.md | 4 ++-- docs/zh/50-machine-output.md | 2 +- mcpp.toml | 2 +- modules/buildmcpp/mcpp.toml | 2 +- modules/buildopt/mcpp.toml | 2 +- modules/graph/mcpp.toml | 2 +- modules/manifest/mcpp.toml | 2 +- modules/toolchain-model/data/builtin.toml | 6 +++--- modules/toolchain-model/mcpp.toml | 2 +- modules/versioning/mcpp.toml | 2 +- tests/canary/cxx/mcpp.toml | 2 +- tests/fixtures/target-rows.snapshot | 20 +++++++++---------- tests/support/mcpp.toml | 2 +- 19 files changed, 48 insertions(+), 32 deletions(-) diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md index 9ac831c18..36a32ce2c 100644 --- a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md @@ -769,7 +769,8 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 | 10.6(N1) | mcpp 检查点 10b;SPEC-011 §5.1 第 8 条 | | 11.1(#953 D5) | 检查点 9;SPEC-004 §9 第 17 条 | | 11.2(#953) | xim-pkgindex#1196:llvm 23.1.3 修订 4,FreeBSD 基座修订 3 | -| 12(超时) | mtest 0.10.0(mcpp-community/mtest#25),openkal-image 0.7.0(okrun 的结束请求记录) | +| 12(超时) | mtest 0.10.0(mcpp-community/mtest#25),openkal-image 0.7.0(okrun 的结束请求记录);mcpp 的 CI 运行器与开发依赖为 mtest 0.10.0(xim-pkgindex#1199),缺省平台为 openkal-platform 0.31.0(检查点 11) | +| 级联 | mcpp.plugins 0.26.1(mcpplibs.os 0.4.5)→ openkal-platform 0.31.0(openkal-svr4 0.3.0、ibcs-us 4.3-2)→ mcpp 2026.11.19.1 → mcpp.plugins 0.27.0(`steps`);xlings 的 Windows 工具链为 llvm 23.1.3(openxlings/xlings#722:MSVC 14.52 的 STL 拒绝 Clang 22 之前的版本) | ### 19.2 测量 @@ -808,6 +809,10 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 - 11.1 的两种交替、10.6 的 P1 至 P3:见 19.2。 - 规则 B:ld.lld 经 mcpp 的链接边运行(e2e 931 两例与 emscripten 一例);mold 未核对。 - mtest 0.9.0 的超时直接结束程序;0.10.0 先记录进程、请求结束,宽限之后再结束。 +- 复现脚本以本轮的引擎运行(Linux x86_64):960、956、952、952s(4 次中 0 次被拒)、931、954、958 不再复现。 + 955 的边通道按设计不变:经框架到达的应用中 `dep_bin` 仍回答空串,构建以 `build-program/undeclared-tool` 报告, + 写出应写的声明。931d 显示工程在 `[build] ldflags` 中写出的 `--dependency-file` 被引擎追加的同名选项覆盖而不报告 + (链接器只写最后一个);此后工程写出时引擎不再加入自己的选项,并以 `build/link-report` 报告降级(SPEC-004 §8.1)。 ### 19.4 实施中发现、单列的事项 @@ -821,3 +826,7 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 而 openkal-platform 的构建依赖正是它。发布顺序因此为:mcpp.plugins 0.26.1(仅 mcpplibs.os 0.4.5,引擎下界不变)→ openkal-platform 0.31.0 → mcpp 2026.11.19.1(缺省平台 0.31.0)→ mcpp.plugins 0.27.0(`steps`,引擎下界为该 mcpp)。 后者的引擎下界使两者不能合为一次发布。 +- Windows 上执行器以改名发布 BMI 时偶遇 `being used by another process`(CI 中约 350 个 BMI 出现一次,其他进程短暂打开了 + 刚写出的文件)。改名对暂时的拒绝(`again`、`permission`)在 30 s 内重试(`replace_file_within`),其后的拒绝照旧报告。 +- `-dumpdir` 的探测在不写全局缓存的命令(`self env` 等)中创建了冷 home(macOS 的 e2e 测得):这类命令在临时目录中探测, + 回答不保存,home 不变(SPEC-011 §6)。 diff --git a/.github/versions.env b/.github/versions.env index cea9579db..c571ac8c9 100644 --- a/.github/versions.env +++ b/.github/versions.env @@ -1,5 +1,5 @@ # The engines CI pins, read by the composite actions (.github/actions/mtest). # mtest: an xlings package (xim:mtest) whose sha256 the index states; the release # bundle's sha256 checks the fallback download. -MTEST_VERSION=0.8.0 -MTEST_SHA256=06555eb8dbdebdf978dc2f074a3efe8521bf71bb51844976216ec21d7fb4be94 +MTEST_VERSION=0.10.0 +MTEST_SHA256=c7187e6211ea15cff23f622c449247508f6f661f0d3b45dcd031f91104b8f499 diff --git a/CHANGELOG.md b/CHANGELOG.md index d1db2ebd7..1539a9c99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,13 @@ ### Changed +- `mcpplibs.openkal-platform@0.31.0` is the default platform of the rows openkal serves + (0.29.2 in 2026.11.18.1): the openkal 0.26 family, in which a start is bounded by the + environment alone and one refused for its size is `E2BIG`, and an observation of a unit + answers whether this process space can observe it (mcpp#958, mcpp#952). The runner of an ok + image, asked to end, names the openkal call each task of the image waits in before it acts. + From 0.30.0 the platform serves `i686-svr4-musl`, programs of UNIX System V Release 4 on i386 + (`preview`); in 0.31.0 they start other programs. - A package instance has a directory of its own: `obj///` and `///`, the digest covering the package's include directories and flags (its features' defines among them) and the digests of the packages whose modules it diff --git a/bench/mcpp.toml b/bench/mcpp.toml index 0b9e79da3..20021a9bb 100644 --- a/bench/mcpp.toml +++ b/bench/mcpp.toml @@ -21,4 +21,4 @@ windows = "llvm@23.1.3" # small fixture. `mcpp build` ignores [dev-dependencies], so the harness itself # still takes nothing beyond `import std;`. [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" diff --git a/docs/05-dependencies.md b/docs/05-dependencies.md index 65d501e50..b0cd68e24 100644 --- a/docs/05-dependencies.md +++ b/docs/05-dependencies.md @@ -67,8 +67,8 @@ not deprecated: ```toml [dev-dependencies] -mcpplibs.mtest = "0.9.0" -"mcpplibs.mtest.gtest" = "0.9.0" +mcpplibs.mtest = "0.10.0" +"mcpplibs.mtest.gtest" = "0.10.0" ``` ```toml diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index cca86097c..83877c219 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -553,7 +553,7 @@ installation, which is located on the machine rather than installed by mcpp. A target row is `{target, note, toolchain, pin, status, default}`, and `status` is one of `installed` / `available` / `via dependency graph` / `planned`. A target row also carries `platform` (2026.10.27.1+): the platform package the -target table selects for it (`mcpplibs.openkal-platform@0.29.2` from 2026.11.18.1, `@0.29.1` from 2026.11.17.2, `@0.28.0` from 2026.11.15.1, `@0.27.1` from 2026.11.12.1, `@0.26.0` in 2026.11.10.2, `@0.25.0` in 2026.11.10.1, `@0.24.4` from 2026.11.8.1, `@0.24.3` in 2026.11.7.2; the series `^0.24` in 2026.11.7.1), or `null`. +target table selects for it (`mcpplibs.openkal-platform@0.31.0` from 2026.11.19.1, `@0.29.2` in 2026.11.18.1, `@0.29.1` from 2026.11.17.2, `@0.28.0` from 2026.11.15.1, `@0.27.1` from 2026.11.12.1, `@0.26.0` in 2026.11.10.2, `@0.25.0` in 2026.11.10.1, `@0.24.4` from 2026.11.8.1, `@0.24.3` in 2026.11.7.2; the series `^0.24` in 2026.11.7.1), or `null`. **`toolchain` and `pin` are not the same field twice.** `toolchain` is what the row is associated with — the installed payload on an installed row, the diff --git a/docs/zh/05-dependencies.md b/docs/zh/05-dependencies.md index 5c5a9a7da..619ae0d9a 100644 --- a/docs/zh/05-dependencies.md +++ b/docs/zh/05-dependencies.md @@ -62,8 +62,8 @@ mcpp 把它读作这些点所拼写的选择器,警告该拼法已弃用(自 ```toml [dev-dependencies] -mcpplibs.mtest = "0.9.0" -"mcpplibs.mtest.gtest" = "0.9.0" +mcpplibs.mtest = "0.10.0" +"mcpplibs.mtest.gtest" = "0.10.0" ``` ```toml diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index 7335ec324..a4d7936f0 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -488,7 +488,7 @@ mcpp toolchain list --format json 的形状是 `{target, note, toolchain, pin, status, default}`,`status` 取 `installed` / `available` / `via dependency graph` / `planned` 之一。 目标行另有 `platform`(2026.10.27.1+):目标表为该目标选择的平台包 -(2026.11.18.1 起为 `mcpplibs.openkal-platform@0.29.2`,2026.11.17.2 为 `@0.29.1`,2026.11.15.1 起为 `@0.28.0`,2026.11.12.1 为 `@0.27.1`,2026.11.10.2 为 `@0.26.0`,2026.11.10.1 为 `@0.25.0`,2026.11.8.1 起为 `@0.24.4`,2026.11.7.2 为 `@0.24.3`;2026.11.7.1 为系列 `^0.24`),或 `null`。 +(2026.11.19.1 起为 `mcpplibs.openkal-platform@0.31.0`,2026.11.18.1 为 `@0.29.2`,2026.11.17.2 为 `@0.29.1`,2026.11.15.1 起为 `@0.28.0`,2026.11.12.1 为 `@0.27.1`,2026.11.10.2 为 `@0.26.0`,2026.11.10.1 为 `@0.25.0`,2026.11.8.1 起为 `@0.24.4`,2026.11.7.2 为 `@0.24.3`;2026.11.7.1 为系列 `^0.24`),或 `null`。 **`toolchain` 与 `pin` 不是把同一个字段写了两遍。** `toolchain` 表示这一行 关联到什么 —— 已装的行是装好的载荷,词表行是它自己的约定。`pin` 只承载目标 diff --git a/mcpp.toml b/mcpp.toml index 2145b065b..aa667f325 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -180,7 +180,7 @@ members = [ # `mcpp build` ignores [dev-dependencies]; only `mcpp test` resolves them. [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" # The support of the end-to-end cases (tests/support, design 2026-11-12 §5.9). mcpp.test-support = { path = "tests/support" } diff --git a/modules/buildmcpp/mcpp.toml b/modules/buildmcpp/mcpp.toml index 32fba3a26..e02b2515f 100644 --- a/modules/buildmcpp/mcpp.toml +++ b/modules/buildmcpp/mcpp.toml @@ -41,4 +41,4 @@ mcpp.source-kind = { path = "../source-kind" } mcpp.toolchain-model = { path = "../toolchain-model" } [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" diff --git a/modules/buildopt/mcpp.toml b/modules/buildopt/mcpp.toml index 17b429cdd..083c09659 100644 --- a/modules/buildopt/mcpp.toml +++ b/modules/buildopt/mcpp.toml @@ -21,4 +21,4 @@ mcpp.manifest = { path = "../manifest" } mcpp.toolchain-model = { path = "../toolchain-model" } [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" diff --git a/modules/graph/mcpp.toml b/modules/graph/mcpp.toml index 0e5838f2a..dd56ac94d 100644 --- a/modules/graph/mcpp.toml +++ b/modules/graph/mcpp.toml @@ -17,4 +17,4 @@ description = "Algorithms over index-addressed directed graphs: topological orde license = "Apache-2.0" [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" diff --git a/modules/manifest/mcpp.toml b/modules/manifest/mcpp.toml index 1421be750..dbe50edb5 100644 --- a/modules/manifest/mcpp.toml +++ b/modules/manifest/mcpp.toml @@ -30,4 +30,4 @@ mcpplibs.os = "0.4.5" # state its contract in isolation; the root `tests/unit/` states the facts two # layers must agree on. [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" diff --git a/modules/toolchain-model/data/builtin.toml b/modules/toolchain-model/data/builtin.toml index 02c976428..d12c65969 100644 --- a/modules/toolchain-model/data/builtin.toml +++ b/modules/toolchain-model/data/builtin.toml @@ -684,7 +684,7 @@ hosts = ["*"] toolchain = "llvm@23.1.3" pairs = ["llvm"] linkage = "static" -platform = "mcpplibs.openkal-platform@0.29.2" +platform = "mcpplibs.openkal-platform@0.31.0" [[platform.provider]] name = "openkal" @@ -694,7 +694,7 @@ hosts = ["*"] toolchain = "llvm@23.1.3" pairs = ["llvm"] linkage = "static" -platform = "mcpplibs.openkal-platform@0.29.2" +platform = "mcpplibs.openkal-platform@0.31.0" # The same arrangement on Windows, whose own interfaces a resource script # compiles against: their headers are the MinGW-w64 base's, which states them @@ -707,7 +707,7 @@ hosts = ["*"] toolchain = "llvm@23.1.3" pairs = ["llvm"] linkage = "static" -platform = "mcpplibs.openkal-platform@0.29.2" +platform = "mcpplibs.openkal-platform@0.31.0" system-headers = "xim:mingw-w64-sysroot@20261104-2" # ── Systems ────────────────────────────────────────────────────────────────── diff --git a/modules/toolchain-model/mcpp.toml b/modules/toolchain-model/mcpp.toml index a6cdc8862..7ef2bbef9 100644 --- a/modules/toolchain-model/mcpp.toml +++ b/modules/toolchain-model/mcpp.toml @@ -29,7 +29,7 @@ mcpplibs.os = "0.4.5" # state its contract in isolation; the root `tests/unit/` states the facts two # layers must agree on. [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" # build.mcpp compiles data/builtin.toml (the built-in target rows and providers) # into a module of static tables, with mcpp's own TOML parser. diff --git a/modules/versioning/mcpp.toml b/modules/versioning/mcpp.toml index a579ee2ab..7c5d19a47 100644 --- a/modules/versioning/mcpp.toml +++ b/modules/versioning/mcpp.toml @@ -25,4 +25,4 @@ license = "Apache-2.0" # state its contract in isolation; the root `tests/unit/` states the same facts # as the engine consumes them, and the pair is the cross-check. [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" diff --git a/tests/canary/cxx/mcpp.toml b/tests/canary/cxx/mcpp.toml index 71d0fbcc9..a70b33875 100644 --- a/tests/canary/cxx/mcpp.toml +++ b/tests/canary/cxx/mcpp.toml @@ -9,4 +9,4 @@ kind = "bin" main = "src/main.cpp" [dev-dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" diff --git a/tests/fixtures/target-rows.snapshot b/tests/fixtures/target-rows.snapshot index b7166edcd..adea97bdc 100644 --- a/tests/fixtures/target-rows.snapshot +++ b/tests/fixtures/target-rows.snapshot @@ -113,7 +113,7 @@ row aarch64-windows-gnu descriptor-keys windows sysroot xim:mingw-w64-sysroot@20261104-2 row x86_64-windows-musl - table tier=preview note=PE pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=preview note=PE pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str x86_64-windows-musl llvm x86_64-w64-windows-gnu llvm@15.0 x86_64-w64-windows-gnu @@ -170,7 +170,7 @@ row aarch64-macos descriptor-keys macosx sysroot row aarch64-macos-musl - table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str aarch64-macos-musl llvm arm64-apple-macos14.0 llvm@15.0 arm64-apple-macos15.0 @@ -208,7 +208,7 @@ row riscv64-linux-musl descriptor-keys linux sysroot row i686-linux-musl - table tier=planned note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=planned note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str i686-linux-musl llvm i686-unknown-linux-musl llvm@15.0 i686-unknown-linux-musl @@ -246,7 +246,7 @@ row i686-linux-gnu descriptor-keys linux sysroot row i686-openkal - table tier=planned note=ok pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=planned note=ok pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str i686-openkal llvm i686-unknown-none-elf llvm@15.0 i686-unknown-none-elf @@ -284,7 +284,7 @@ row x86_64-macos descriptor-keys macosx sysroot row x86_64-macos-musl - table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str x86_64-macos-musl llvm x86_64-apple-macos14.0 llvm@15.0 x86_64-apple-macos15.0 @@ -341,7 +341,7 @@ row aarch64-freebsd descriptor-keys freebsd sysroot row x86_64-freebsd-musl - table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str x86_64-freebsd-musl llvm x86_64-unknown-freebsd14.0 llvm@15.0 x86_64-unknown-freebsd14.0 @@ -360,7 +360,7 @@ row x86_64-freebsd-musl descriptor-keys freebsd sysroot row aarch64-freebsd-musl - table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=preview note= pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str aarch64-freebsd-musl llvm aarch64-unknown-freebsd14.0 llvm@15.0 aarch64-unknown-freebsd14.0 @@ -817,7 +817,7 @@ row wasm32-emscripten descriptor-keys emscripten linux sysroot row x86_64-openkal - table tier=verified note=ok pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=verified note=ok pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str x86_64-openkal llvm x86_64-unknown-none-elf llvm@15.0 x86_64-unknown-none-elf @@ -836,7 +836,7 @@ row x86_64-openkal descriptor-keys openkal linux sysroot row aarch64-openkal - table tier=verified note=ok pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.29.2 + table tier=verified note=ok pin=llvm@23.1.3 sysroot= static=1 platform=mcpplibs.openkal-platform@0.31.0 str aarch64-openkal llvm aarch64-unknown-none-elf llvm@15.0 aarch64-unknown-none-elf @@ -971,7 +971,7 @@ set linux-musl -> x86_64-linux-musl aarch64-linux-musl | set windows -> x86_64-windows-gnu aarch64-windows-gnu | set windows-msvc -> x86_64-windows-msvc | set macos -> aarch64-macos | -set openkal -> x86_64-openkal aarch64-openkal | mcpplibs.openkal-platform@0.29.2 +set openkal -> x86_64-openkal aarch64-openkal | mcpplibs.openkal-platform@0.31.0 set none-elf -> riscv64-none-elf riscv32-none-elf aarch64-none-elf x86_64-none-elf i686-none-elf | set none-eabihf -> thumbv7em-none-eabihf thumbv8m.main-none-eabihf armv7a-none-eabihf | set android -> aarch64-linux-android x86_64-linux-android | diff --git a/tests/support/mcpp.toml b/tests/support/mcpp.toml index 6a7a2c8bc..1bf503f01 100644 --- a/tests/support/mcpp.toml +++ b/tests/support/mcpp.toml @@ -13,5 +13,5 @@ license = "Apache-2.0" kind = "lib" [dependencies] -mcpplibs.mtest = "0.9.0" +mcpplibs.mtest = "0.10.0" nlohmann.json = "3.12.0" From 6cffb87ae57df825d8d38e26cdb9f961f06391e7 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 20:40:23 +0900 Subject: [PATCH 15/30] cl.exe's record names what a compile read, and the text of every program reaches mcpp as UTF-8 (mcpp#970) A compile under cl.exe writes its /sourceDependencies record, and the executor turns it into the Make rule every compile edge is read by (deps = gcc): the headers of INCLUDE and EXTERNAL_INCLUDE are left out, a path under the build directory is spelled as the build directory and the case of each component on disk, and no line of the compiler's text is read for a dependency. VSLANG is no longer set. The text a program writes is decoded where it is read, by the dialect's text encoding: UTF-8 for GNU, and for MSVC the output code page of the attached console, else the system's ANSI code page (mcpplibs.os 0.4.6, mcpplibs.os.text). Every JSON document mcpp writes replaces a sequence that is not UTF-8 instead of failing. An MSVC diagnostic is read by its structure, and C1083 names its header in any language. Design 2026-11-19 section 20; SPEC-011 section 5.1 items 11 and 12; SPEC-012 section 5. --- ...11-19-ecosystem-rule-convergence-design.md | 135 +++++++++++++++++- CHANGELOG.md | 22 +++ docs/20-toolchains.md | 14 ++ docs/50-machine-output.md | 9 ++ docs/specs/README.md | 4 +- docs/specs/concurrency.md | 12 +- docs/specs/human-output.md | 19 ++- docs/zh/20-toolchains.md | 8 ++ docs/zh/50-machine-output.md | 5 + mcpp.toml | 2 +- modules/invocation/mcpp.toml | 2 +- modules/manifest/mcpp.toml | 2 +- modules/toolchain-model/mcpp.toml | 2 +- modules/toolchain-model/src/dialect.cppm | 26 +++- src/build/depfile.cppm | 99 +++++++++++++ src/build/execute.cppm | 9 +- src/build/ninja_backend.cppm | 117 ++++++++++----- src/build/result.cppm | 27 ++-- src/build/schedule/detach_codegen.cppm | 129 +++++++++++++++-- src/cli.cppm | 20 ++- src/cli/cmd_build.cppm | 37 ++++- src/run.cppm | 9 +- src/run/run.cpp | 16 ++- src/toolchain/msvc.cppm | 9 +- src/toolchain/stdmod.cppm | 12 +- src/wire.cppm | 14 +- tests/e2e/windows/msvc.cpp | 63 +++++++- tests/unit/test_build_result.cpp | 52 +++++++ tests/unit/test_depfile.cpp | 73 ++++++++++ tests/unit/test_ninja_backend.cpp | 56 +++++++- tests/unit/test_wire.cpp | 15 ++ 31 files changed, 907 insertions(+), 112 deletions(-) create mode 100644 tests/unit/test_build_result.cpp diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md index 36a32ce2c..c4ff7451a 100644 --- a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md @@ -5,8 +5,9 @@ status: active # 生态规则收敛方案:2026.11.18.1 之后十个 issue 的五类成因、七项决定与实施范围 -**日期:** 2026-11-19 · **版本:** v4(实施中。用户决定:全部在本轮实施,每个仓库一个拉取请求,提交作为检查点, -从成因解决而不留变通;N1 至 N3 按建议。第 15 节为七项决定的依据,第 16 节为两轮自评,第 17 节为已定事项,第 19 节为实施的状态与测量) · **基线:** mcpp 2026.11.18.1(`3b886163`)、openkal 0.25.3、openkal-linux 0.26.3(`363ccbb`)、 +**日期:** 2026-11-19 · **版本:** v5(实施中。用户决定:全部在本轮实施,每个仓库一个拉取请求,提交作为检查点, +从成因解决而不留变通;N1 至 N3 按建议;mcpp#970 并入本轮,随 2026.11.19.1 发布。第 15 节为七项决定的依据,第 16 节为两轮自评, +第 17 节为已定事项,第 19 节为实施的状态与测量,第 20 节为 #970 的方案) · **基线:** mcpp 2026.11.18.1(`3b886163`)、openkal 0.25.3、openkal-linux 0.26.3(`363ccbb`)、 openkal-macos 0.23.5(`9dbc36f`,索引中为 0.23.4)、openkal-platform 0.29.2(引擎默认)与 0.30.1、mcpplibs.os 0.4.4、 mcpp.plugins 0.26.0、mcpp-index `3ee5f08`、xlings `4f5df092` · **前文:** [第二部分之后的补充方案](2026-11-17-ninja-cascade-publish-supplement-design.md)、 @@ -34,6 +35,7 @@ mcpp.plugins 0.26.0、mcpp-index `3ee5f08`、xlings `4f5df092` · **前文:** | #953 · 特性并集不同的 43 条边 | 纳入 | 同一输出路径对应两种配置 | 11 | | #953 · Windows LLVM 载荷的 lld 不带 zlib | 纳入 | 同一工具链载荷在不同宿主上能力不同 | 11 | | #953 · e2e 1033 与 986 的间歇失败 | 只纳入证据的留存 | 第一次出现时没有留下可供诊断的状态 | 12 | +| #970 | 纳入,范围大于报告 | 编译的依赖取自写给人看的本地化文本;程序的文本跨进程时不带编码;写出 JSON 的函数在外来字节上抛出 | 20 | 已部分实现、本文只补缺口的三处:#956 的目标图请求自 2026.11.7.1 起已由版本组约束(SPEC-013 §7.5);#931 的整词绝对路径与 `@file` 自 2026.10.31.1 与 2026.11.2.2 起已是链接输入;#959 所需的“复制一个文件”自 2026.9.13.1 起由 `${mcpp.self} stage` 提供。 @@ -517,6 +519,7 @@ okrun 收到结束请求时输出映像中每个任务正在等待的 openkal | 3 | openkal-musl、openkal-llvm-runtime,各一个 | 精确钉住的级联与版本 | | 3 | openkal-image(`okld`、`okrun`、工具) | ① okrun 收到结束请求时输出每个任务正在等待的调用(12) ② 级联与版本 | | 4 | mcpplibs.os | ① 启动失败带两个向量的条数与字节数 ② 不可观察的区分、性质的查询 ③ 成员的范围、引擎下界与版本 | +| 4′ | mcpplibs.os(0.4.6,第二个拉取请求) | ① `mcpplibs.os.text`:程序文本的编码与到 UTF-8 的转换(20.4 T2) ② 版本 | | 4 | mtest | ① 超时先请求结束、宽限、记录进程树(12) ② 版本 | | 5 | openkal-platform | ① 版本组点名新的成员 ② 版本 | | 6 | mcpp | 见下 | @@ -539,6 +542,7 @@ mcpp 的拉取请求的检查点: | 10 | 编译运行状态:恢复、清理、准入的条件、返回路径 | 10.3 C–G | | 10b | 产物的事务化发布:先测量 P1 至 P3,再实现(N1 采纳时) | 10.6 | | 11 | 新的 mcpplibs.os:启动失败的消息、不可观察的区分、性质 | 9.2 之 4,10.3 B、H | +| 11b | 编译器的文本与依赖记录(#970),mcpplibs.os 0.4.6 | 20 | | 12 | 发布:版本、CHANGELOG、默认平台、依赖改为版本依赖(`mcpp.toml` 与 `version.cppm` 在这一个提交中) | — | 检查点 2 至 10b 不依赖新的 openkal,先于上游的发布完成;11 与 12 在第 1 至 5 项发布之后。 @@ -567,6 +571,8 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 | 规则在事后遍历 `obj/` 并按路径排除;以 shell 的 `printf` 写出含绝对路径的文件 | 规则包 | 8.2、8.3 | 可改用记号 | | FreeBSD 基座修订 2 解压其链接输入 | xim-pkgindex | 11.2 | 修订 3 | | e2e 1116 在 Windows 宿主上跳过 | mcpp | 11.2 | 不再跳过 | +| 构建环境中的 `VSLANG=1033`(假定英文资源存在,并覆盖用户的设置) | mcpp | 20 | 删除 | +| 安装英文语言包并设 `VSLANG=1033` 以取得头文件依赖(#970 的报告) | 工程 | 20 | 不再需要 | 工程与规则包一侧的各项由其仓库在升级后删除并以自己的 CI 核对,不属于 14.2 的拉取请求。 @@ -770,6 +776,7 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 | 11.1(#953 D5) | 检查点 9;SPEC-004 §9 第 17 条 | | 11.2(#953) | xim-pkgindex#1196:llvm 23.1.3 修订 4,FreeBSD 基座修订 3 | | 12(超时) | mtest 0.10.0(mcpp-community/mtest#25),openkal-image 0.7.0(okrun 的结束请求记录);mcpp 的 CI 运行器与开发依赖为 mtest 0.10.0(xim-pkgindex#1199),缺省平台为 openkal-platform 0.31.0(检查点 11) | +| 20(#970) | mcpplibs.os 0.4.6(`mcpplibs.os.text`),mcpp 检查点 11b;SPEC-011 §5.1 第 11、12 条,SPEC-012 §2,docs/50 §9,docs/20 | | 级联 | mcpp.plugins 0.26.1(mcpplibs.os 0.4.5)→ openkal-platform 0.31.0(openkal-svr4 0.3.0、ibcs-us 4.3-2)→ mcpp 2026.11.19.1 → mcpp.plugins 0.27.0(`steps`);xlings 的 Windows 工具链为 llvm 23.1.3(openxlings/xlings#722:MSVC 14.52 的 STL 拒绝 Clang 22 之前的版本) | ### 19.2 测量 @@ -830,3 +837,127 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 刚写出的文件)。改名对暂时的拒绝(`again`、`permission`)在 30 s 内重试(`replace_file_within`),其后的拒绝照旧报告。 - `-dumpdir` 的探测在不写全局缓存的命令(`self env` 等)中创建了冷 home(macOS 的 e2e 测得):这类命令在临时目录中探测, 回答不保存,home 不变(SPEC-011 §6)。 + +## 20. 编译器的文本与依赖记录(#970) + +**来源。** mcpp#970(Windows 11,`msvc@system`,VS 2022 BuildTools,MSVC 14.44.35207,只装中文资源)报告两项: +头文件改动后不重编,`ninja -t deps` 对每个目标文件为 `#deps 0`;编译失败时 mcpp 以 +`internal: unhandled exception: [json.exception.type_error.316] invalid UTF-8 byte` 结束。用户决定本轮从成因解决, +不留变通,随 2026.11.19.1 发布。 + +### 20.1 现状 + +- **依赖。** 方言的 `ninjaDepsMode = "msvc"` 使 cl.exe 的编译规则带 `/showIncludes` 与 `deps = msvc`,不写 + `msvc_deps_prefix`(`ninja_backend.cppm`);ninja 以缺省的英文前缀 `Note: including file:` 识别依赖行。 + `build_env_for_cl` 在构建环境中写 `VSLANG=1033`,注释称它保证英文前缀(`msvc.cppm`)。 +- **文本。** 自 2026.11.7.1 起每个编译边经执行器 `__compiler-cap` 运行,执行器以 `Streams::inherit` 启动编译器;链接边经 + `link-output`,归档边(lib.exe)由 ninja 直接运行。ninja 把各步骤的字节原样交给 mcpp。mcpp 的文本是 UTF-8:mcpp.exe + 声明 UTF-8 活动代码页,openkal-windows 写控制台时把 UTF-8 转为 UTF-16。失败步骤的输出经 `parse_compiler_diagnostics` + 写入 `result.json` 与事件流,`doc.dump(2)` 在不是 UTF-8 的字节上抛出 `type_error.316`。`execute.cppm` 中测试记录的两处 + 写出已使用替换策略。 +- **读诊断的另外两处。** `missing_header_in` 以英文句子 `Cannot open include file: '` 识别 C1083; + `parse_compiler_diagnostics` 的 MSVC 分支要求严重性词后跟代码,不带代码的 `note:` 行落入 GNU 分支,路径记为 + `k.cpp(1)`,行号为 0。 + +### 20.2 实测 + +windows-2022,探针 mcpp#971(运行 44508574496),以 `CreateProcessW` 启动 cl.exe 与 link.exe,两个输出流写入一个管道。 + +| 项 | 结果 | +|---|---| +| 附着于控制台,输出代码页 437、1252、936、65001 | cl 与 link 按该代码页写文本:437 中 `é` 为 0x82、汉字为 `?`;1252 中 `é` 为 0xE9;936 为 GBK;65001 为 UTF-8 | +| `DETACHED_PROCESS`(无控制台) | 两者按系统的 ANSI 代码页写(runner 为 1252) | +| `CREATE_NO_WINDOW` | 得到一个新的控制台,代码页为系统的 OEM 代码页(437) | +| 代码页不能表示的字符 | cl 写出时已成为 `?`:437 下的标识符 `未定义_é` 写作 `???_é` | +| `/sourceDependencies <文件>` | JSON(`"Version": "1.2"`),UTF-8,非 ASCII 字符原样;列出全部头文件,含工具链与 SDK 的;路径整体小写(源目录写作 `Inc Dir\Héader.h`,记录为 `c:\p970\中文é\inc dir\héader.h`);编译失败时也写出;模块接口单元同样(`ProvidedModule`)。只给该选项时 cl 的输出只有源文件名一行 | +| 加装 zh-cn 语言包(6.4 分钟),`VSLANG=2052` | 前缀为 `注意: 包含文件: `(其后两个空格)。诊断为 `b.cpp(2): error C2065: “未定义_é”: 未声明的标识符`、`k.cpp(1): note: 可能是“void h(long)”`、`c.cpp(1): fatal error C1083: 无法打开包括文件: “缺失.h”: No such file or directory`、`LINK : fatal error LNK1181: 无法打开输入文件“缺失.obj”`:位置、`fatal error`、`error`、`note` 与代码不随语言变化,正文与引号随语言变化。记录与英文版相同 | +| runner 的步骤 | 附着于一个没有窗口、代码页为 65001 的控制台 | + +### 20.3 成因 + +1. **依赖取自写给人看的文本。** 前缀是界面字符串,随安装的语言资源与 `VSLANG` 变化;路径以控制台的代码页编码,代码页 + 不能表示的字符成为 `?`,记录下的依赖指向不存在的文件。`VSLANG=1033` 是对前缀的变通:它假定英文资源存在,并覆盖 + 用户自己的设置。 +2. **文本跨进程时不带编码。** cl.exe 的文本按它所附着的控制台的代码页编码,mcpp 的文本是 UTF-8,两者之间的管道只有字节。 + 编码只在运行该程序的进程中可知:ninja 的输出流中同时有 ninja 自己的 UTF-8 行(取自构建文件)与各步骤的字节,宿主工具链 + 与目标工具链的步骤编码也可以不同,所以在 mcpp 读取 ninja 处统一解码是错误的。 +3. **写出 JSON 的函数在外来字节上抛出。** JSON 只承载 UTF-8(RFC 8259 §8.1);一个程序写出的字节不应使 mcpp 以内部错误 + 结束。 + +识别 C1083 的英文句子是第 1 类在读诊断上的一个实例。 + +### 20.4 方案 + +**T1 依赖取自编译器为程序写出的记录。** 一次编译读取的文件取自编译器为程序写出的记录,不取自写给人看的文本。方言以数据 +说明记录的形式(`inputsRecord`):GNU 驱动写 Make 规则(`-MMD -MF $out.d`,`make`),cl.exe 写 `/sourceDependencies` +的 JSON 记录(`source-dependencies`)。执行器把 JSON 记录写成同形的依赖文件,ninja 对每个编译边以 `deps = gcc` 读取。 +- 位于工具链环境的 `INCLUDE` 与 `EXTERNAL_INCLUDE` 目录之下的头文件不写入,与 `-MMD` 不列系统头文件相同;工具链与 SDK + 的改变由工具链的身份与构建环境反映。 +- 位于构建目录(执行器的工作目录,即 ninja 的 `-C`)之下的路径,写作该目录的写法加上其下各级名字在磁盘上的写法:记录把 + 路径改为小写,而生成的头文件是图中另一条边的输出,节点名须与之相同,否则生成器重新运行之后包含它的单元要到下一次构建 + 才重编。其余路径按记录写出,Windows 的文件系统以不区分大小写的名字打开它们。 +- 编译成功后写出依赖文件,失败时不写(ninja 不读取失败的边的依赖);记录在两种情况下都删除。 +- 依赖因此与语言、控制台的代码页无关。`/showIncludes` 与 `deps = msvc` 不再使用,`VSLANG` 不再设置:编译器的语言是 + 用户环境的(`VSLANG` 或 Visual Studio 的界面语言),要英文诊断的用户自己设 `VSLANG=1033`。 + +**T2 文本在读入它的进程中按产生者的编码解码。** mcpp 内部的文本是 UTF-8。一个程序的文本在读入它的 mcpp 进程中解码, +编码由该程序所属工具链的方言以数据说明(`textEncoding`):GNU 驱动为 `utf-8`;cl.exe、link.exe、lib.exe 与 rc.exe 为 +`console`,即所附着控制台的输出代码页,没有控制台时为系统的 ANSI 代码页(20.2)。 +- 执行器(`__compiler-cap`、`link-output`)接受 `--text console`:以一个管道读取程序合并的两个输出流,逐行解码,写到 + 自己的标准输出。没有该词时照旧继承两个流,GNU 工具链的命令与行为不变。逐行解码不会切开字符:Windows 代码页的多字节 + 序列与 UTF-8 的序列都不含换行字节。 +- 方言的文本为 `console` 时,归档边经 `link-output --text console` 运行;不带发布选项时 `link-output` 只运行命令。 +- mcpp 自己运行并读取的同一工具链的程序(std 模块的构建)以同一规则解码(`mcpp::run::Options`)。 +- `console` 到代码页的解析与代码页到 UTF-8 的转换属于系统层:mcpplibs.os 0.4.6 的 `mcpplibs.os.text`,在每个系统上 + 定义,Windows 以外 `console` 即 UTF-8。执行器、ninja 与编译器附着于同一个控制台(或都没有),执行器解析一次,所见即 + 编译器所见;引擎不点名系统。 +- 解码不改变内容:源文件名一行、诊断及其顺序保持编译器写出的样子。代码页不能表示的字符在 cl 写出时已成为 `?`,解码 + 不能恢复;依赖不受影响(T1)。终端为 UTF-8 控制台(`chcp 65001`)时诊断保留全部字符。 + +**T3 承载程序文本的文档总是 UTF-8。** `result.json`、`target/.last-result.json`、事件流与协议的信封承载程序写出的文本 +(诊断、失败的输出),由一个函数写出(`mcpp::wire::text`),把不是 UTF-8 的序列写作 U+FFFD(Unicode 第 3.9 节,最大子部分 +的替换)。T2 之下 MSVC 的文本不再经过它;它留给声明为 UTF-8 而实际不是的字节,例如 GCC 引用一个 Latin-1 源文件的行。 +终端与日志中的输出保持原样。 + +**T4 诊断按各语言不变的结构读取。** MSVC 的诊断为 `<路径>(<行>[,<列>]): <严重性>[ <代码>]: <正文>`,严重性为 +`fatal error`、`error`、`warning`、`note`,中文版亦然(20.2)。不带代码的 `note:` 行按这一形式读取。缺失的头文件以代码 +`C1083` 与其操作数的位置识别:操作数是最后两个 `: ` 之间的一段,去掉首尾的一对引号,引号的字形随语言变化。GNU 驱动的 +诊断不在本节:其严重性词随区域设置翻译,mcpp 不设置区域,行为不变。 + +### 20.5 多视角复核 + +- **架构。** 事实(记录的形式、文本的编码)在方言的数据中;机制(转换、解码)在执行器与系统层;`check_system_names` + 的计数不增加。 +- **稳定性。** 依赖与语言、代码页无关;程序写出的字节不能使 mcpp 以内部错误结束;依赖取自工具为程序写出的记录,与 + #931 的规则 B(链接器的报告)是同一原则。 +- **简洁。** 每个编译边的依赖为同一形式(`deps = gcc`);删除 `VSLANG` 的变通与 ninja 前缀的依赖。 +- **用户体验。** 任何语言的 Visual Studio 在头文件改动后重编;终端中的诊断可读;`result.json` 的诊断为 UTF-8,`note` + 的路径与行号正确;缺失头文件的建议在各语言中给出。 +- **兼容与无感升级。** cl.exe 的编译命令改变一次,各单元在升级后的第一次构建中重编一次;快速路径的环境键随 `VSLANG` + 的删除改变,自动重新生成。GNU 工具链的命令不变。用户自己的 `VSLANG` 从此生效。 +- **跨平台。** `mcpplibs.os.text` 在每个系统上定义;Windows 以外执行器不加 `--text`。 +- **测试覆盖。** 见 20.7;此前没有用例断言 MSVC 的构建记录了头文件依赖,#970 因此未被发现。 + +### 20.6 不采纳的做法 + +- **探测前缀并写 `msvc_deps_prefix`(报告的建议,CMake 的 `CMAKE_CL_SHOWINCLUDES_PREFIX`)。** 前缀的字节随控制台的 + 代码页变化(同一前缀在 936 与 65001 下字节不同),按工具链缓存的前缀在另一个终端中失效;代码页不能表示的路径成为 `?`, + 依赖仍然错误。 +- **设置控制台的代码页(`SetConsoleOutputCP(65001)`)。** 控制台由同一会话的所有进程共有,并行的执行器相互竞争,mcpp + 被强制结束时用户的终端停留在被改的代码页。 +- **`VS_UNICODE_OUTPUT`。** MSBuild 内部的接口,没有公开的说明。 +- **在 mcpp 读取 ninja 输出处解码。** 见 20.3 第 2 条。 + +### 20.7 验证 + +- **单元。** JSON 记录到依赖文件的转换:工具链目录的排除、构建目录之下的写法、空格与 `#`、`$` 的转义;MSVC 诊断的解析, + 英文与中文的样本(`error`、`fatal error`、不带代码的 `note`、`LNK`);C1083 的识别,英文与中文的样本;`mcpp::wire::text` + 的替换。样本取自 20.2 的字节。 +- **mcpplibs.os 0.4.6。** 936、1252、437 与 65001 的样本及无效序列的转换;附着于控制台时 `console` 解析为该控制台的 + 代码页。 +- **e2e(Windows,MSVC,970)。** 一个非 ASCII 目录中、头文件目录带空格的工程:编译规则带记录而不读 `/showIncludes`; + 头文件改为 `static_assert(false)` 后构建失败,改回后成功;在一个代码页为 936 的新控制台(`start /wait`)中构建失败时, + mcpp 不以内部错误结束,终端与 `result.json` 中的诊断含正确的汉字。 +- **规则的形状(单元)。** cl.exe 的编译规则带 `/sourceDependencies $out.json`、执行器的两个词与 `deps = gcc`,链接与归档经 + `link-output --text console`;GNU 工具链的命令不带这两个词。 +- **语言包。** CI 不安装(6.4 分钟);20.2 的测量由单元测试的样本覆盖。 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1539a9c99..1d7ef909a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,9 +56,31 @@ given a map too, which names each module at its name. The files named after the object (`.dwo`, `.gcno`) keep their names through `-dumpdir` and `-dumpbase`; clang's are named `..`. +- mcpp does not set `VSLANG` for cl.exe: the compiler speaks the language of the user's + environment (`VSLANG`, else Visual Studio's), where it spoke English whenever the English + resources were installed. `VSLANG=1033` asks for English (mcpp#970). +- A build program that pins `mcpp.plugins` below 0.26 shares no host graph with + `openkal-platform@0.31.0`, whose build program links `mcpp.plugins` 0.26.1: such a pin is + refused as irreconcilable on the rows openkal serves, and moves to 0.26.1. ### Fixed +- A header edit rebuilt nothing on a Visual Studio without English resources: the files a + cl.exe compile read came from its `/showIncludes` lines, whose prefix is a string of the + installed language (`注意: 包含文件:`), and `VSLANG=1033` cannot make it English (mcpp#970). + They come from cl.exe's `/sourceDependencies` record, the same in every language and console + code page, which the executor writes as the compile's dependency file (SPEC-011 §5.1); a + header under the build directory keeps its spelling, so a generated header is the node its + generator writes. A record that cannot be read fails the compile with the reason. +- A cl.exe failure on a Chinese console ended mcpp with `internal: unhandled exception: + [json.exception.type_error.316] invalid UTF-8 byte`, and its diagnostics were unreadable: + cl.exe, link.exe, lib.exe and rc.exe write their text in the code page of the console they run + in, and mcpp read it as UTF-8 (mcpp#970). The executor reads their text in that page (with no + console, the system's ANSI code page) and writes it on as UTF-8 (SPEC-012 §5), and every + document that carries a program's text writes a byte that is not UTF-8 as U+FFFD. A cl.exe + `note` in `result.json` carries the location of its file, and the advice for a header a + dependency lost names it in every language (`C1083`). + - A build interrupted in another process space -- a container job that reuses the working directory, a rebuilt development container, an agent's sandbox session -- could never be continued, `mcpp clean` included: `previous compiler unit is occupied or unknown` (mcpp#952). diff --git a/docs/20-toolchains.md b/docs/20-toolchains.md index 55c12d209..c9b37b66d 100644 --- a/docs/20-toolchains.md +++ b/docs/20-toolchains.md @@ -705,6 +705,20 @@ environment from the VC tools + Windows SDK (no `vcvarsall` involved), stages `/interface /TP /ifcOutput`, scans with `/scanDependencies`, and links with `link.exe`/`lib.exe` through response files. +**The headers a compile read, and the compiler's text, in any language** +*(2026.11.19.1+)*. Each compile writes cl.exe's `/sourceDependencies` record, +which mcpp writes as the compile's dependency file, so an edit of a header +rebuilds the units that include it whatever language the installed Visual +Studio speaks and whatever code page the console uses. cl.exe, link.exe, +lib.exe and rc.exe write their text in the code page of the console they run +in (with no console, the system's ANSI code page), and mcpp reads it as such, +so their diagnostics reach the terminal and `result.json` as written. mcpp does +not set `VSLANG`: the compiler speaks the language of the user's environment +(`VSLANG`, else Visual Studio's), and `VSLANG=1033` asks for English where the +English resources are installed. A character the console's code page cannot +represent arrives as `?` from cl.exe itself; a console in UTF-8 +(`chcp 65001`) keeps every character. + **The Windows SDK follows the origin**, because the two origins answer different questions and so must the SDK: diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index 83877c219..b041f6751 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -1029,6 +1029,15 @@ terminal is therefore visible to an IDE that reads the record, and the reverse. | `artifacts` | `name`, `kind` (`bin`, `test`, `shared`, `static`), `path`, for a runnable product `launch`, and `entry` when the program's beginning or its entry symbol was stated | | `diagnostics` | the compilers' diagnostics of a failed build: `severity`, `message`, `path`, `range` | +**Text** *(2026.11.19.1+)*. The record, the event stream and every envelope are +UTF-8. A compiler's text is decoded where mcpp reads it, in the encoding its +toolchain states ([SPEC-012](specs/human-output.md) §5: cl.exe, link.exe, +lib.exe and rc.exe write in the code page of the console they run in), so a +`message` holds the compiler's words in the language it speaks. A byte a +program wrote that is not UTF-8 is written as U+FFFD; the terminal and the logs +keep the program's output as it was written. A cl.exe `note` carries the +location of its file, as its `error` and `warning` lines do. + `launch` is how `mcpp run` starts the product: `argv`, `cwd`, `env` and `via`. `env` holds only the variables mcpp sets or rewrites for the program (its loader's search path, the variables a runtime payload needs), never the diff --git a/docs/specs/README.md b/docs/specs/README.md index 489e71b79..f243f2b81 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -40,8 +40,8 @@ | [SPEC-008](library-interface.md) | 库的接口:公开模块、发布闭包与两种形态的一致 | 草案 v0.2 | 2026-10-11 | 第一阶段(只警告)mcpp >= 2026.9.28.3;v0.2 mcpp >= 2026.10.11.1 | | [SPEC-009](toolchain-maintenance.md) | 工具链的支持与维护:版本线、默认值、来源、移动与退役 | 草案 v0.6 | 2026-10-09 | 逐条标注;本版只有规范,多数条款未实现;§6.4 mcpp >= 2026.10.8.1 | | [SPEC-010](home-initialization.md) | home 选择、单跳入口、初始化、网络边界与单次运行的 `--home` | 草案 v0.2 | 2026-10-26 | mcpp >= 2026.10.19.1;网络能力逐条标注;§6 mcpp >= 2026.10.26.1、xlings >= 2026.10.26.1 | -| [SPEC-011](concurrency.md) | 并发运行:三段、构建目录锁、原子发布、`mcpp.lock` 的乐观写入、取消 | 评审中 v1.0 | 2026-10-26 | mcpp >= 2026.10.26.1 | -| [SPEC-012](human-output.md) | 面向人的输出:两条流、两种介质、行的种类、颜色 | 评审中 v1.0 | 2026-10-26 | mcpp >= 2026.10.26.1 | +| [SPEC-011](concurrency.md) | 并发运行:三段、构建目录锁、原子发布、`mcpp.lock` 的乐观写入、取消;编译执行器的恢复、发布、记录与文本 | 评审中 v1.2 | 2026-11-19 | mcpp >= 2026.10.26.1;§5.1 mcpp >= 2026.11.7.1;§5.1 第 6 条的不可观察单元与第 8 至 12 条 mcpp >= 2026.11.19.1 | +| [SPEC-012](human-output.md) | 面向人的输出:两条流、两种介质、行的种类、颜色、程序的文本 | 评审中 v1.1 | 2026-11-19 | mcpp >= 2026.10.26.1;§5 mcpp >= 2026.11.19.1 | | [SPEC-013](platforms.md) | 平台:`[platform]` 的七组键、版本组与 `hosted`,三种写法与层次,目标图与宿主图的选择,`cfg(platform)`,依赖图的补全,链接、命名与封装,运行器与载体,平台阶段与协议 17 | 评审中 v1.7 | 2026-11-10 | mcpp >= 2026.10.27.1;§10 的执行宿主与 §11 的载体架构 mcpp >= 2026.10.29.1;§7.5 版本组参与解析与 §14 平台系列 mcpp >= 2026.11.7.1;§14 内置平台为确切版本 mcpp >= 2026.11.7.2;v1.7 条款 mcpp >= 2026.11.10.1 | | [SPEC-014](packaging.md) | 打包与分发:链接、闭包、容器三个维度与构建腿,腿的合并,暂存树与 `mcpp-package.json`,平台的树、运行覆盖、单文件、polyglot、容器列表、签名、压缩、合并与分发描述 | 评审中 v2.2 | 2026-11-10 | mcpp >= 2026.10.31.1;逐条标注;v2.2 条款 mcpp >= 2026.11.10.1 | | [SPEC-015](target-sets.md) | 目标集合与腿:集合的写法与展开,命令对多条腿的语义,串行规划与一起构建,测试归档,机器输出 | 评审中 v1.0 | 2026-10-30 | mcpp >= 2026.10.30.1 | diff --git a/docs/specs/concurrency.md b/docs/specs/concurrency.md index 52afd6e72..d31b29428 100644 --- a/docs/specs/concurrency.md +++ b/docs/specs/concurrency.md @@ -7,7 +7,7 @@ | 状态 | 评审中 v1.2 | | 版本 | 1.2 | | 最后修改 | 2026-11-19 | -| 对应实现 | 基础保证: mcpp >= 2026.10.26.1; §5.1: mcpp >= 2026.11.7.1;§5.1 第 6 条的不可观察单元与第 8 至 10 条: mcpp >= 2026.11.19.1 | +| 对应实现 | 基础保证: mcpp >= 2026.10.26.1; §5.1: mcpp >= 2026.11.7.1;§5.1 第 6 条的不可观察单元与第 8 至 12 条: mcpp >= 2026.11.19.1 | | 相关设计文档 | `.agents/docs/2026-10-26-control-contract-runtime-and-ide-ecosystem-design.md` §3、§4、§5.4 | | 相关 issue | #839 | | 使用文档 | [50 — 机器可读输出](../zh/50-machine-output.md) §9、§11 | @@ -137,6 +137,14 @@ CI。在 2026.10.24.1 上,同一配置的两次冷构建相隔 0.1–0.5 s 启 状态与取消以外的任何原因结束时**必须**写出结束它的步骤与原因。环境不能声明执行单元时 (`units_observable()` 为假),整次构建的编译不声明单元,仍受同一编译上限约束,并经 `degraded`(`compiler-run/no-unit`)报告一次。 +11. 编译器为程序写出的、一次编译所读文件的记录不是 Make 规则时(工具链方言的 `inputsRecord`:cl.exe 的 + `/sourceDependencies`),执行器**必须**在编译成功后把它写为该边的依赖文件,ninja 以 `deps = gcc` 读取: + 工具链环境的 `INCLUDE` 与 `EXTERNAL_INCLUDE` 目录之下的头文件不写入;构建目录之下的路径写作构建目录的写法加上 + 磁盘上各级名字的写法;其余路径按记录写出。记录不能读出或依赖文件不能写出时该边**必须**失败,并写出原因; + 编译失败时不写依赖文件。记录在两种情况下都删除。依赖**禁止**取自编译器写给人看的文本(设计 2026-11-19 §20 T1)。 +12. 工具链方言说明其程序的文本不是 UTF-8 时(`textEncoding = "console"`:cl.exe、link.exe、lib.exe、rc.exe), + 运行该程序的执行器(`__compiler-cap`、`link-output`)**必须**以一个管道读取它的两个输出流,逐行解码为 UTF-8 + 后写到自己的标准输出;归档边因此也经 `link-output` 运行。解码**禁止**改变内容与顺序(SPEC-012 §5)。 ## 6. 全局一侧 @@ -152,4 +160,4 @@ CI。在 2026.10.24.1 上,同一配置的两次冷构建相隔 0.1–0.5 s 启 |---|---|---| | 1.0 | 2026-10-26 | 首版:三段、构建目录锁、原子发布、`mcpp.lock` 的乐观写入、取消(#839) | | 1.1 | 2026-11-06 | 分离编译的单元观察、共享名额、绝对输出事务与有限恢复边界;对齐取消请求的 OS 接收方式。 | -| 1.2 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#952,设计 2026-11-19 §10):§5.1 第 6 条不可观察单元的恢复与报告、拒绝写出个数与出路;第 8 条输出可重定向的工具链上编译写入尝试私有的路径并由执行器发布;第 9 条清理退役运行状态;第 10 条准入的条件、执行器的结束原因与不能声明单元的环境。 | +| 1.2 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#952,设计 2026-11-19 §10):§5.1 第 6 条不可观察单元的恢复与报告、拒绝写出个数与出路;第 8 条输出可重定向的工具链上编译写入尝试私有的路径并由执行器发布;第 9 条清理退役运行状态;第 10 条准入的条件、执行器的结束原因与不能声明单元的环境;第 11 条编译器的记录由执行器写为依赖文件,第 12 条程序的文本在执行器中解码为 UTF-8(mcpp#970,设计 2026-11-19 §20)。 | diff --git a/docs/specs/human-output.md b/docs/specs/human-output.md index 1480a9ec8..57ace367e 100644 --- a/docs/specs/human-output.md +++ b/docs/specs/human-output.md @@ -4,10 +4,10 @@ |---|---| | 规范编号 | SPEC-012 | | 标题 | 面向人的输出的形态:两条流、两种介质、行的种类、颜色 | -| 状态 | 评审中 v1.0 | -| 版本 | 1.0 | -| 最后修改 | 2026-10-26 | -| 对应实现 | mcpp >= 2026.10.26.1 | +| 状态 | 评审中 v1.1 | +| 版本 | 1.1 | +| 最后修改 | 2026-11-19 | +| 对应实现 | mcpp >= 2026.10.26.1;§5 mcpp >= 2026.11.19.1 | | 相关设计文档 | `.agents/docs/2026-09-29-build-progress-display-design.md`、`.agents/docs/2026-09-30-build-output-refinement-design.md`、`.agents/docs/2026-10-26-control-contract-runtime-and-ide-ecosystem-design.md` §5.7、§5.9 | | 使用文档 | [50 — 机器可读输出](../zh/50-machine-output.md) §3 | @@ -66,8 +66,17 @@ 颜色只修饰状态行的词、消息行的前缀与强调,**禁止**承载其他地方没有的信息。 -## 5. 变更记录 +## 5. 程序的文本 已实现 + +mcpp 的文本是 UTF-8。mcpp 运行的程序的文本**必须**在读入它的进程中,按该程序所属工具链的方言说明的编码解码为 +UTF-8,再写出或记录:GNU 驱动的文本为 UTF-8;cl.exe 一族(cl、link、lib、rc)的文本为所附着控制台的输出代码页, +没有控制台时为系统的 ANSI 代码页(设计 2026-11-19 §20 T2,在 windows-2022 上以 437、1252、936、65001 测量)。 +解码**禁止**改变内容与顺序。代码页不能表示的字符在程序写出时已成为 `?`,解码不能恢复它。承载程序文本的文档 +(docs/50 的结果记录、事件流与信封)中不是 UTF-8 的序列写作 U+FFFD;终端与日志中的输出保持程序写出的样子。 + +## 6. 变更记录 | 版本 | 日期 | 变更 | |---|---|---| | 1.0 | 2026-10-26 | 首版:两条流、两种介质、行的种类(`help:` 统一补救行)、颜色的优先级 | +| 1.1 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#970,设计 2026-11-19 §20):§5 程序的文本按其工具链的编码解码为 UTF-8,文档中不是 UTF-8 的序列写作 U+FFFD | diff --git a/docs/zh/20-toolchains.md b/docs/zh/20-toolchains.md index bc869d201..a28869141 100644 --- a/docs/zh/20-toolchains.md +++ b/docs/zh/20-toolchains.md @@ -625,6 +625,14 @@ staging 成 `.ifc` BMI,用 `/interface /TP /ifcOutput` 编译 `.cppm` 模块 单元,用 `/scanDependencies` 扫描依赖,并通过 response file 调用 `link.exe`/`lib.exe` 完成链接。 +**一次编译读取的头文件与编译器的文本,与语言无关** *(2026.11.19.1+)*。每次编译写出 cl.exe 的 +`/sourceDependencies` 记录,mcpp 把它写为该编译的依赖文件,所以无论安装的 Visual Studio 是哪种语言、 +控制台用哪个代码页,修改头文件都会重新编译包含它的单元。cl.exe、link.exe、lib.exe 与 rc.exe 按它们运行 +所在控制台的代码页写出文本(没有控制台时为系统的 ANSI 代码页),mcpp 按此读取,所以它们的诊断原样到达终端 +与 `result.json`。mcpp 不设置 `VSLANG`:编译器使用用户环境的语言(`VSLANG`,否则为 Visual Studio 的界面语言), +装有英文资源时 `VSLANG=1033` 得到英文诊断。控制台代码页不能表示的字符由 cl.exe 自己写成 `?`;UTF-8 控制台 +(`chcp 65001`)保留全部字符。 + **Windows SDK 跟随来源走**,因为两种来源回答的是不同的问题,SDK 的选定 也必须如此: diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index a4d7936f0..d75ca8562 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -917,6 +917,11 @@ mcpp build [target] [-p ] [--release] --message-format json | `artifacts` | `name`、`kind`(`bin`、`test`、`shared`、`static`)、`path`;可运行的产物另有 `launch`;陈述了程序起点或入口符号时另有 `entry` | | `diagnostics` | 失败构建的编译器诊断:`severity`、`message`、`path`、`range` | +**文本** *(2026.11.19.1+)*。结果记录、事件流与每个信封都是 UTF-8。编译器的文本在 mcpp 读入它的地方按其工具链 +说明的编码解码([SPEC-012](../specs/human-output.md) §5:cl.exe、link.exe、lib.exe 与 rc.exe 按它们运行所在控制台的 +代码页写出),所以 `message` 是编译器所用语言的原文。程序写出的不是 UTF-8 的字节写作 U+FFFD;终端与日志中的输出 +保持程序写出的样子。cl.exe 的 `note` 与它的 `error`、`warning` 一样带有所在文件的位置。 + `launch` 是 `mcpp run` 启动该产物的方式:`argv`、`cwd`、`env` 与 `via`。`env` 只含 mcpp 为程序设置或改写的变量(加载器的搜索路径、运行时载荷需要的变量),从不包含 调用方的其余变量,因此记录中不含调用方的机密。`via` 为 `null`、`"runner"`(由 diff --git a/mcpp.toml b/mcpp.toml index aa667f325..58ac88539 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -125,7 +125,7 @@ mcpplibs.mwrap.xz = "5.8.3.1" # The operating system is mcpplibs.os's (design 2026-10-26 §25): mcpp keeps no # layer of its own. -mcpplibs.os = "0.4.5" +mcpplibs.os = "0.4.6" # `modules/` holds what gets linked into this binary; `src/` is what has not # been separated yet, and shrinking it is the direction. Each entry is an diff --git a/modules/invocation/mcpp.toml b/modules/invocation/mcpp.toml index efbf6fa0b..496101574 100644 --- a/modules/invocation/mcpp.toml +++ b/modules/invocation/mcpp.toml @@ -13,4 +13,4 @@ description = "The environment of one invocation of mcpp: statements over the st license = "Apache-2.0" [dependencies] -mcpplibs.os = "0.4.5" +mcpplibs.os = "0.4.6" diff --git a/modules/manifest/mcpp.toml b/modules/manifest/mcpp.toml index dbe50edb5..9d473104a 100644 --- a/modules/manifest/mcpp.toml +++ b/modules/manifest/mcpp.toml @@ -24,7 +24,7 @@ mcpp.source-kind = { path = "../source-kind" } mcpp.toolchain-model = { path = "../toolchain-model" } mcpp.versioning = { path = "../versioning" } -mcpplibs.os = "0.4.5" +mcpplibs.os = "0.4.6" # Subsystem-level tests. They see only this package, which is what lets them # state its contract in isolation; the root `tests/unit/` states the facts two diff --git a/modules/toolchain-model/mcpp.toml b/modules/toolchain-model/mcpp.toml index 7ef2bbef9..18eba6295 100644 --- a/modules/toolchain-model/mcpp.toml +++ b/modules/toolchain-model/mcpp.toml @@ -23,7 +23,7 @@ license = "Apache-2.0" mcpp.invocation = { path = "../invocation" } mcpp.versioning = { path = "../versioning" } -mcpplibs.os = "0.4.5" +mcpplibs.os = "0.4.6" # Subsystem-level tests. They see only this package, which is what lets them # state its contract in isolation; the root `tests/unit/` states the facts two diff --git a/modules/toolchain-model/src/dialect.cppm b/modules/toolchain-model/src/dialect.cppm index 2510f0ae1..4c964e0fd 100644 --- a/modules/toolchain-model/src/dialect.cppm +++ b/modules/toolchain-model/src/dialect.cppm @@ -78,10 +78,22 @@ struct CommandDialect { // Artifact naming. std::string_view objExt; // ".o" | ".obj" - // Ninja integration. - // deps mode for header dependencies ("" = none today for gnu — module - // deps go through the P1689/dyndep pipeline instead). - std::string_view ninjaDepsMode; // "" | "msvc" + // THE FILES A COMPILE READ ARE TAKEN FROM A RECORD THE COMPILER WRITES FOR + // PROGRAMS (mcpp#970, design 2026-11-19 §20 T1), never from the text it + // writes for people. "make": the rule a GNU driver writes (-MMD -MF). + // "source-dependencies": cl.exe's JSON record (/sourceDependencies ), + // which the compiler executor writes as the same rule. Every compile edge + // reads a Make rule (`deps = gcc`). Until 2026.11.19.1 cl.exe's edges read + // the `/showIncludes` lines (`deps = msvc`), whose prefix is a string of + // the installed language and whose paths are text of the console's code + // page: a Visual Studio without English resources recorded no header. + std::string_view inputsRecord; // "make" | "source-dependencies" + // THE ENCODING OF THE TEXT THE DIALECT'S PROGRAMS WRITE (T2): "utf-8", or + // "console" -- the output code page of the console the program is attached + // to, else the system's ANSI code page (mcpplibs.os.text), measured with + // cl.exe and link.exe. A program whose text is not UTF-8 runs through an + // executor that decodes it where it is read. + std::string_view textEncoding; // "utf-8" | "console" // Whether link/archive command lines must go through a response file // (Windows 8191-char command-line limit; cl/link/lib accept @file). bool rspfileLink = false; @@ -270,7 +282,8 @@ constexpr CommandDialect kGnuDialect{ .staticRuntime = "-static", .outputExePrefix = "-o ", .objExt = ".o", - .ninjaDepsMode = "", + .inputsRecord = "make", + .textEncoding = "utf-8", .rspfileLink = false, .linkStyle = CommandDialect::LinkStyle::Driver, .archiveCmd = "$ar rcs $out $in", @@ -311,7 +324,8 @@ constexpr CommandDialect kMsvcDialect{ .dynamicRuntime = "/MD", .outputExePrefix = "/Fe:", .objExt = ".obj", - .ninjaDepsMode = "msvc", + .inputsRecord = "source-dependencies", + .textEncoding = "console", .rspfileLink = true, .linkStyle = CommandDialect::LinkStyle::SeparateLinker, .archiveCmd = "$ar /nologo /OUT:$out $in", diff --git a/src/build/depfile.cppm b/src/build/depfile.cppm index a1565bae2..46460657e 100644 --- a/src/build/depfile.cppm +++ b/src/build/depfile.cppm @@ -17,6 +17,7 @@ export module mcpp.build.depfile; import std; +import nlohmann.json; export namespace mcpp::build::depfile { @@ -41,4 +42,102 @@ inline std::string first_record(std::string_view raw) { return out; } +// THE RULE OF cl.exe'S RECORD (mcpp#970, design 2026-11-19 §20 T1). +// +// cl.exe states the files a compile read in the JSON record +// `/sourceDependencies ` writes, in UTF-8 and the same in every language +// and console code page (measured on windows-2022, probe mcpp#971). The +// executor writes it as the Make rule every compile edge reads: +// * a header under one of `toolchainDirs` (the INCLUDE and EXTERNAL_INCLUDE +// of the toolchain's environment) is left out, as -MMD leaves out a system +// header; +// * a header under `buildDir`, the directory the compile ran in, is written +// as `buildDir` and then each name below it as the file system spells it +// (`spelled`): the record spells every path in lower case, and a generated +// header is another edge's output, whose node keeps its case; +// * every other header is written as the record spells it, which the file +// system of a cl.exe host opens without regard to case. +// Paths are compared in the generic spelling (`/`), without ASCII case when +// `foldCase`. Not inline: an inline function of an imported module that +// iterates a string fails to compile in the importer under GCC 16.1. +struct SourceRecord { + std::string_view record; // what /sourceDependencies wrote + std::string_view target; // the edge's output, as the rule names it + std::string buildDir; // generic spelling + std::vector toolchainDirs; // generic spellings + bool foldCase = true; + // The entry of `dir` that `name` names, as the file system spells it. + std::function spelled; +}; + +namespace detail { + +std::string generic(std::string_view path) { + std::string out(path); + std::ranges::replace(out, '\\', '/'); + while (out.size() > 1 && out.back() == '/' && out[out.size() - 2] != ':') out.pop_back(); + return out; +} + +bool same_text(std::string_view a, std::string_view b, bool fold) { + if (a.size() != b.size()) return false; + for (std::size_t i = 0; i < a.size(); ++i) { + char x = a[i], y = b[i]; + if (fold && x >= 'A' && x <= 'Z') x = static_cast(x - 'A' + 'a'); + if (fold && y >= 'A' && y <= 'Z') y = static_cast(y - 'A' + 'a'); + if (x != y) return false; + } + return true; +} + +// The part of `path` below `dir`, when `path` is beneath it. +std::optional below(std::string_view path, std::string_view dir, bool fold) { + if (dir.empty() || path.size() <= dir.size() + 1) return std::nullopt; + if (!same_text(path.substr(0, dir.size()), dir, fold) || path[dir.size()] != '/') return std::nullopt; + return path.substr(dir.size() + 1); +} + +// A name in a Make rule: a space and `#` escaped, `$` doubled. +std::string escaped(std::string_view name) { + std::string out; + for (const char c : name) { + if (c == ' ' || c == '#') out += '\\'; + if (c == '$') out += '$'; + out += c; + } + return out; +} + +} // namespace detail + +std::expected rule_from_source_dependencies(const SourceRecord& in) { + const auto doc = nlohmann::json::parse(in.record, nullptr, false); + if (doc.is_discarded() || !doc.is_object()) + return std::unexpected(std::string("the record is not a JSON object")); + const auto data = doc.find("Data"); + if (data == doc.end() || !data->is_object() || !data->contains("Includes") || !(*data)["Includes"].is_array()) + return std::unexpected(std::string("the record has no Data.Includes list")); + const std::string buildDir = detail::generic(in.buildDir); + std::vector system; + for (auto const& d : in.toolchainDirs) + if (!d.empty()) system.push_back(detail::generic(d)); + std::string rule = detail::escaped(in.target) + ":"; + for (auto const& entry : (*data)["Includes"]) { + if (!entry.is_string()) return std::unexpected(std::string("an entry of Data.Includes is not a path")); + const std::string path = detail::generic(entry.get()); + if (std::ranges::any_of(system, [&](const std::string& d) { return detail::below(path, d, in.foldCase).has_value(); })) + continue; + std::string named = path; + if (const auto rest = detail::below(path, buildDir, in.foldCase); rest && in.spelled) { + named = buildDir; + for (auto part : std::views::split(*rest, '/')) { + const std::string name(std::string_view(part.begin(), part.end())); + named += '/' + in.spelled(named, name); + } + } + rule += " \\\n " + detail::escaped(named); + } + return rule + "\n"; +} + } // namespace mcpp::build::depfile diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 3359fae86..5df045be7 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -2579,7 +2579,7 @@ std::optional run_ninja_fast(const std::string& ninjaProgram, std::vector> childEnv; if (runtimeEnvKey == "@env") { - // Multi-var encoding (MSVC INCLUDE/LIB/PATH/VSLANG + optional runtime + // Multi-var encoding (MSVC INCLUDE/LIB/PATH + optional runtime // pair): \x1f-separated k=v records in the single value slot. std::string_view rest = runtimeEnvValue; while (!rest.empty()) { @@ -3427,7 +3427,7 @@ bool& launch_stream_flag() { export void set_launch_stream(bool on) { launch_stream_flag() = on; } void emit_launch(const mcpp::wire::Envelope& env) { - if (launch_stream_flag()) mcpp::ui::terminal::write_protocol_stdout(mcpp::wire::to_json(env).dump() + "\n"); + if (launch_stream_flag()) mcpp::ui::terminal::write_protocol_stdout(mcpp::wire::text(mcpp::wire::to_json(env)) + "\n"); else mcpp::wire::emit(env); } @@ -4746,7 +4746,7 @@ static void emit_test_json(const std::string& memberName, const TestResult& r) { } if (!r.guide.empty()) record["guide"] = r.guide; if (r.launch) record["launch"] = mcpp::build::result::to_json(*r.launch); - std::println("{}", record.dump(-1, ' ', false, nlohmann::ordered_json::error_handler_t::replace)); + std::println("{}", mcpp::wire::text(record)); std::fflush(libc::stdout()); } @@ -5679,8 +5679,7 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, if (it.key() != "id") body[it.key()] = *it; } std::scoped_lock lock(reportMutex); - std::println("{}", nlohmann::ordered_json{{"case", std::move(body)}}.dump( - -1, ' ', false, nlohmann::ordered_json::error_handler_t::replace)); + std::println("{}", mcpp::wire::text(nlohmann::ordered_json{{"case", std::move(body)}})); std::fflush(libc::stdout()); } }; diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 48c26ec90..44af318fa 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -1158,13 +1158,26 @@ std::string missing_header_in(std::string_view line) { return std::string(line.substr(open + 1, at - open - 1)); return {}; } - constexpr std::string_view kCl = "Cannot open include file: '"; - if (auto at = line.find(kCl); at != std::string_view::npos) { - auto from = at + kCl.size(); - auto close = line.find('\'', from); - if (close != std::string_view::npos) - return std::string(line.substr(from, close - from)); - return {}; + // cl.exe, in every language: `(): fatal error C1083: : + // : `. The words, the quote marks and the + // reason are the installed language's (`'x.h'`, `“x.h”`); the location, + // the code and the place of the name are not (design 2026-11-19 §20 T4). + // A missing source file is C1083 too, reported with no line of a file. + constexpr std::string_view kC1083 = "): fatal error C1083: "; + if (auto at = line.find(kC1083); at != std::string_view::npos) { + const auto rest = line.substr(at + kC1083.size()); + const auto first = rest.find(": "); + const auto last = rest.rfind(": "); + if (first == std::string_view::npos || last <= first + 2) return {}; + auto quoted = rest.substr(first + 2, last - first - 2); + // One mark at each end, a character of UTF-8 each. + std::size_t lead = 1; + if (static_cast(quoted[0]) >= 0x80) + while (lead < quoted.size() && (static_cast(quoted[lead]) & 0xC0) == 0x80) ++lead; + std::size_t tail = quoted.size() - 1; + while (tail > lead && (static_cast(quoted[tail]) & 0xC0) == 0x80) --tail; + if (tail <= lead) return {}; + return std::string(quoted.substr(lead, tail - lead)); } constexpr std::string_view kGcc = "fatal error: "; constexpr std::string_view kEnoent = ": No such file or directory"; @@ -1644,6 +1657,17 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, const std::string compilerPrefix = plan.scheduleCompilerCap > 0 ? std::string("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d -- ") : std::string{}; + // WHAT THE EXECUTOR READS FOR THE COMPILER (design 2026-11-19 §20): the + // record of the files a compile read, when the compiler writes one that is + // not a Make rule (cl.exe's `/sourceDependencies`, T1), and the encoding of + // the text it writes, when that is not UTF-8 (`console`, T2). A GNU + // toolchain's commands carry neither and are unchanged. + const auto& readingDialect = mcpp::toolchain::dialect_for(plan.toolchain); + const std::string recordWord = + readingDialect.inputsRecord == std::string_view("source-dependencies") + ? std::string(" --source-dependencies $out.json") : std::string{}; + const std::string textWord = readingDialect.textEncoding == std::string_view("utf-8") + ? std::string{} : std::format(" --text {}", readingDialect.textEncoding); // A COMPILE WRITES PATHS OF ITS OWN ATTEMPT where the toolchain's outputs // are relocatable, and the executor publishes them (design 2026-11-19 // §10.6): a compiler that outlives its build cannot rewrite a published @@ -1654,11 +1678,12 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, ? std::format(" --relocate {}", relocation.auxNaming.empty() ? std::string_view("none") : relocation.auxNaming) : std::string{}; const std::string driverPrefix = plan.scheduleCompilerCap > 0 - ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{} -- ", relocateWord) + ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{}{}{} -- ", + relocateWord, recordWord, textWord) : std::string{}; const std::string scanPrefix = plan.scheduleCompilerCap > 0 - ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{} -- ", - relocation.relocatable ? " --relocate none" : "") + ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{}{} -- ", + relocation.relocatable ? " --relocate none" : "", textWord) : std::string{}; // The same executor for an edge that writes a BMI: it holds the previous // BMI while the compiler runs and puts it back when the new one is @@ -1674,12 +1699,13 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, auto bmi_keeping_prefix = [&](std::string_view bmi) { return plan.scheduleCompilerCap > 0 ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{}" - " --bmi \"{}\"{} -- ", bmi == "$out" ? "" : " --object", bmi, relocateWord) + " --bmi \"{}\"{}{}{} -- ", bmi == "$out" ? "" : " --object", bmi, relocateWord, + recordWord, textWord) : std::string{}; }; const std::string objectPrefix = plan.scheduleCompilerCap > 0 - ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d --object{} -- ", - relocateWord) + ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d --object{}{}{} -- ", + relocateWord, recordWord, textWord) : std::string{}; // The other split shape. Clang publishes no BMI early — it writes the BMI // at the end of the compile — so the GCC trick of releasing the file @@ -2101,17 +2127,21 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // Command spellings come from the toolchain's CommandDialect (gnu vs // msvc); the rule *structure* is shared across compilers. - // msvc: /showIncludes feeds ninja's deps=msvc header tracking; the - // stable-English prefix is guaranteed by VSLANG=1033 in envOverrides. - const bool msvcDeps = dial.ninjaDepsMode == std::string_view("msvc"); + // THE FILES A COMPILE READ COME FROM THE COMPILER'S RECORD FOR PROGRAMS + // (mcpp#970, design 2026-11-19 §20 T1). cl.exe writes its JSON record + // (`/sourceDependencies $out.json`), which the executor writes as the Make + // rule every compile edge reads. Until 2026.11.19.1 these edges read the + // `/showIncludes` lines through ninja's `deps = msvc`: their prefix is a + // string of the installed language and their paths text of the console's + // code page, so a Visual Studio with only Chinese resources recorded no + // header at all, and `VSLANG=1033` could not make it English. + const bool msvcDialect = dial.id == std::string_view("msvc"); + const bool sourceRecord = dial.inputsRecord == std::string_view("source-dependencies"); const std::string compile_tail = std::format( "{}{} $in {}$out", - msvcDeps ? "/showIncludes " : "", dial.compileOnly, dial.outputObjPrefix); - auto append_deps = [&] { - if (msvcDeps) append(" deps = msvc\n"); - }; + sourceRecord ? "/sourceDependencies $out.json " : "", dial.compileOnly, dial.outputObjPrefix); // mcpp#235: cxx_module/cxx_object had NO depfile at all on non-MSVC — - // only the msvcDeps branch tracked header deps (via /showIncludes). So + // only cl.exe's edges tracked header deps (then via /showIncludes). So // editing a file #include'd inside a module's purview (or a plain // header pulled in by a .cpp) never invalidated the compile edge: the // P1689 scan already emits a `.dep`/`.d`-shaped list of textual @@ -2146,7 +2176,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // (see the existing "Windows: skip BMI restat optimization" branch // below), so a non-MSVC Windows toolchain keeps the pre-#235 // behavior (no depfile) rather than depend on an unavailable filter — - // msvcDeps (cl.exe) is unaffected either way (deps=msvc, no -MMD). + // cl.exe was unaffected either way (it had no -MMD). // // #257: these are TWO decisions, and 0.0.97 conflated them. Emitting a // depfile at all is the minimum correctness contract for textual include @@ -2165,7 +2195,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // // WHETHER A UNIT EMITS A GNU DEPFILE IS A PROPERTY OF THE COMPILER, NOT // OF THE HOST (the 2026-09-28 design, WS2). Until 2026.9.28.2 this read - // `!msvcDeps && !is_windows`, so every clang++ build on Windows -- the + // `!(cl.exe) && !is_windows`, so every clang++ build on Windows -- the // default Windows row since #718 -- got no `-MMD`, and a header edit did // not rebuild the objects and BMIs that include it. Every GNU-dialect // compiler writes one now, on every host; only HOW GCC's filtered form @@ -2175,8 +2205,8 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // * Windows: no shell to chain a filter in, so `mcpp depfile-filter` // runs the compile itself and keeps the first record of the raw // depfile (the same rule as the awk program). - // cl.exe keeps deps=msvc via /showIncludes, the equivalent mechanism. - const bool gnuDepfile = !msvcDeps; + // cl.exe's record reaches the same Make rule through the executor (T1). + const bool gnuDepfile = dial.inputsRecord == std::string_view("make"); const bool needsGnuModuleFilter = gnuDepfile && plan.toolchain.compiler == mcpp::toolchain::CompilerId::GCC; const std::string mmd_flag = @@ -2192,13 +2222,11 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, (needsGnuModuleFilter && mcpplibs::os::is_windows) ? "$mcpp depfile-filter --raw $out.d.raw --out $out.d -- " : ""; + // Both records arrive as the Make rule `$out.d`: the GNU driver writes it, + // and the executor writes cl.exe's. auto append_cxx_deps = [&] { - if (gnuDepfile) { - append(" deps = gcc\n"); - append(" depfile = $out.d\n"); - } else { - append_deps(); - } + append(" deps = gcc\n"); + append(" depfile = $out.d\n"); }; // C and GAS units include headers too, and had no depfile on ANY // toolchain — the second half of the same asymmetry #257 reports. They @@ -2236,7 +2264,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // makes the response-file shape reachable from a non-Windows test host, // the same over-approximation the link rules use (`separateLinker || // is_windows`). - const bool useCompileRsp = mcpplibs::os::is_windows || msvcDeps; + const bool useCompileRsp = mcpplibs::os::is_windows || msvcDialect; // THE BYTE ORDER MARK, FOR THE TOOLS THAT NEED ONE TO READ UTF-8 (#693). // // build.ninja is UTF-8, and Ninja copies `rspfile_content` into the @@ -2257,7 +2285,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, if (!useCompileRsp) return; append(" rspfile = $out.rsp\n"); append(std::format(" rspfile_content ={}{}\n", - msvcDeps ? " " + std::string(kUtf8ByteOrderMark) : "", payload)); + msvcDialect ? " " + std::string(kUtf8ByteOrderMark) : "", payload)); }; // Tell the driver, every time, that this TU is a module interface. @@ -2491,10 +2519,15 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // script pulls in is declared instead, from a scan of the .rc plus // [resources].extra-inputs, and lands on this edge as implicit inputs. append("rule rc_object\n"); + // The cl.exe toolchain's rc.exe writes its text as cl.exe does (T2). + const std::string rcPrefix = plan.scheduleCompilerCap > 0 + ? std::format("$mcpp __compiler-cap --sem .mcpp-sched --output $out --depfile $out.d{} -- ", + plan.rcStyle == "msvc" ? textWord : std::string{}) + : std::string{}; if (plan.rcStyle == "msvc") - append(std::format(" command = {}$rc /nologo $rcflags /fo $out $in\n", compilerPrefix)); + append(std::format(" command = {}$rc /nologo $rcflags /fo $out $in\n", rcPrefix)); else - append(std::format(" command = {}$rc -O coff $rcflags -o $out $in\n", compilerPrefix)); + append(std::format(" command = {}$rc -O coff $rcflags -o $out $in\n", rcPrefix)); append(" description = RC $out\n\n"); } @@ -2570,11 +2603,14 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, if (auto const* row = mcpp::toolchain::target::find_row(t->str())) for (auto c : row->companions) companions += (companions.empty() ? "" : ",") + std::string(c); const std::string companionOption = companions.empty() ? std::string() : " --companions " + companions; + // link.exe and lib.exe write their text as cl.exe does: `link-output` + // decodes it where it reads it (design 2026-11-19 §20 T2). auto publish_link = [&](std::string cmd) { if constexpr (mcpplibs::os::is_windows) { - return "$mcpp link-output --output $out --aside" + companionOption + " -- " + cmd; + return "$mcpp link-output --output $out --aside" + companionOption + textWord + " -- " + cmd; } else { - if (!companions.empty()) return "$mcpp link-output --output $out" + companionOption + " -- " + cmd; + if (!companions.empty() || !textWord.empty()) + return "$mcpp link-output --output $out" + companionOption + textWord + " -- " + cmd; if (inPlaceTarget) return cmd; if (cmd.find("-o $out") == std::string::npos) return cmd; // Through a temporary name, `/.` (cmd_link_output): one @@ -2606,6 +2642,9 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // The permit wraps the publication: the attempt's output is the // artifact in place when the command it holds exits. if (desc != "AR") cmd = linkPrefix + publish_link(std::move(cmd)); + // An archive holds no permit; a lib.exe whose text is not UTF-8 + // still runs through `link-output`, which only decodes it. + else if (!textWord.empty()) cmd = "$mcpp link-output --output $out" + textWord + " -- " + cmd; append(std::format("rule {}\n", name)); if (useRsp) { if (auto pos = cmd.find("$in"); pos != std::string::npos) @@ -2803,7 +2842,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // the same TU (it wraps it), so it carries the same unbounded // include payload through the same response file (#261). const std::string scanPayload = " $local_includes"; - if (msvcDeps) { + if (msvcDialect) { // MSVC: compiler-integrated P1689 via /scanDependencies (scan // only — no codegen); /TP because our module units are .cppm. append(std::format(" command = {}$cxx_driver{} $cxxflags $unit_cxxflags " @@ -5100,7 +5139,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan r.compileCommands = cdb ? cdb->commandCount : 0; r.ninjaProgram = ninjaProgram; if (!plan.toolchain.envOverrides.empty()) { - // Toolchain-declared env (MSVC INCLUDE/LIB/PATH/VSLANG). Encode all + // Toolchain-declared env (MSVC INCLUDE/LIB/PATH). Encode all // pairs (plus any runtime-dirs pair) into the fast-path cache's // single env slot: "@env" key + \x1f-separated k=v records — the // fast path must re-create this exact environment for ninja. diff --git a/src/build/result.cppm b/src/build/result.cppm index 1fabb1438..df0a55eaf 100644 --- a/src/build/result.cppm +++ b/src/build/result.cppm @@ -32,6 +32,7 @@ import mcpp.version; // MCPP_VERSION import mcpp.run; import mcpp.ui.terminal; import mcpp.invocation; +import mcpp.wire; // text: a document that carries what a program wrote export namespace mcpp::build::result { @@ -274,7 +275,7 @@ std::string render(const Record& r) { {"mcpp", {{"version", std::string(mcpp::MCPP_VERSION)}}}, {"data", to_json(r)}, }; - return doc.dump(2) + "\n"; + return mcpp::wire::text(doc, 2) + "\n"; } std::string render_last(const Record& r) { nlohmann::json last{ @@ -286,7 +287,7 @@ std::string render_last(const Record& r) { {"finishedAt", r.finishedAt == 0 ? nlohmann::json(nullptr) : nlohmann::json(r.finishedAt)}}}, }; - return last.dump(2) + "\n"; + return mcpp::wire::text(last, 2) + "\n"; } } // namespace @@ -358,7 +359,10 @@ std::vector parse_compiler_diagnostics(std::string_view text) { } return std::nullopt; }; - // MSVC: ([,]): : + // MSVC: ([,]): [ ]: . The + // location, the severity and the code are the same in every language of + // cl.exe and the message is the installed language's (measured in English + // and Chinese, design 2026-11-19 §20 T4); a `note` carries no code. const auto msvc = [&](std::string_view line) -> std::optional { auto paren = line.find("): "); if (paren == std::string_view::npos) return std::nullopt; @@ -367,12 +371,17 @@ std::vector parse_compiler_diagnostics(std::string_view text) { auto rest = line.substr(paren + 3); std::size_t len = 0; auto severity = severity_at(rest, len); - if (severity.empty() || rest.substr(len).substr(0, 1) != " ") return std::nullopt; - auto codeEnd = rest.find(": ", len); - if (codeEnd == std::string_view::npos) return std::nullopt; + if (severity.empty()) return std::nullopt; + std::size_t messageAt = len + 2; + if (rest.substr(len).substr(0, 2) != ": ") { + if (rest.substr(len).substr(0, 1) != " ") return std::nullopt; + auto codeEnd = rest.find(": ", len); + if (codeEnd == std::string_view::npos) return std::nullopt; + messageAt = codeEnd + 2; + } Diagnostic d; d.severity = std::move(severity); - d.message = std::string(rest.substr(codeEnd + 2)); + d.message = std::string(rest.substr(messageAt)); d.path = std::string(line.substr(0, open)); auto nums = line.substr(open + 1, paren - open - 1); auto comma = nums.find(','); @@ -444,7 +453,7 @@ void write_last_of_legs(const std::filesystem::path& projectRoot, {"finishedAt", finishedAt == 0 ? nlohmann::json(nullptr) : nlohmann::json(finishedAt)}, {"legs", std::move(legs)}}}, }; - write_atomically(projectRoot / "target" / ".last-result.json", last.dump(2) + "\n"); + write_atomically(projectRoot / "target" / ".last-result.json", mcpp::wire::text(last, 2) + "\n"); } void emit(std::string_view event, nlohmann::json fields) { @@ -452,7 +461,7 @@ void emit(std::string_view event, nlohmann::json fields) { nlohmann::json line{{"schemaVersion", 1}, {"kind", "mcpp.event"}, {"event", std::string(event)}}; for (auto it = fields.begin(); it != fields.end(); ++it) line[it.key()] = it.value(); std::lock_guard lock(g_emit); - mcpp::ui::terminal::write_protocol_stdout(line.dump() + "\n"); + mcpp::ui::terminal::write_protocol_stdout(mcpp::wire::text(line) + "\n"); } } // namespace mcpp::build::result diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm index 79e32c2cd..983cb24a9 100644 --- a/src/build/schedule/detach_codegen.cppm +++ b/src/build/schedule/detach_codegen.cppm @@ -71,6 +71,11 @@ export module mcpp.build.schedule.detach_codegen; import std; import mcpplibs.os.fs; import mcpplibs.os.process; +import mcpplibs.os.env; +import mcpplibs.os.platform; +import mcpplibs.os.stdio; +import mcpplibs.os.text; +import mcpp.build.depfile; import mcpp.build.command_line; import mcpp.run; import mcpp.build.stage; @@ -194,12 +199,23 @@ int await_unit(const std::filesystem::path& slot, const std::filesystem::path& o // its own and this executor publishes them (design 2026-11-19 §10.6); the // value is the toolchain's auxiliary naming (`OutputRelocation::auxNaming`), // or `none`. +// +// `reading` is what the executor reads for the compiler (design 2026-11-19 +// §20): the record of the files the compile read when the compiler writes one +// that is not the Make rule `depfile` (cl.exe's `/sourceDependencies`, written +// as `depfile` once the compile succeeds), and the encoding of the compiler's +// text when it is not UTF-8, in which case its two streams are read through +// one pipe and written on as UTF-8. +struct Reading { + std::filesystem::path sourceDependencies; + mcpplibs::os::text::encoding text = mcpplibs::os::text::encoding::utf8; +}; int run_capped(const std::vector& argv, const std::filesystem::path& semaphore, int maxCompilers, const std::filesystem::path& output = {}, const std::filesystem::path& depfile = {}, const std::filesystem::path& bmi = {}, bool object = false, - std::string_view relocate = {}); + std::string_view relocate = {}, const Reading& reading = {}); } // namespace mcpp::build::schedule::detach @@ -1340,10 +1356,80 @@ struct Previous { ~Previous() { settle(false); } }; +namespace { + +// THE NAME OF AN ENTRY AS THE FILE SYSTEM SPELLS IT, for a name the record +// spelled in lower case (§20 T1): the entry whose name is the same without +// ASCII case, else the one the system opens for that name. +std::string on_disk_name(const std::string& dir, const std::string& name) { + namespace fs = std::filesystem; + const auto directory = mcpplibs::os::fs::from_utf8(dir); + std::error_code ec; + for (fs::directory_iterator it(directory, ec), end; !ec && it != end; it.increment(ec)) { + std::string entry = mcpplibs::os::fs::utf8(it->path().filename()); + if (mcpp::build::depfile::detail::same_text(entry, name, true)) return entry; + } + const auto wanted = directory / mcpplibs::os::fs::from_utf8(name); + ec.clear(); + for (fs::directory_iterator it(directory, ec), end; !ec && it != end; it.increment(ec)) { + std::error_code same; + if (fs::equivalent(it->path(), wanted, same)) return mcpplibs::os::fs::utf8(it->path().filename()); + } + return name; +} + +// The compiler's record of the files it read, written as the Make rule ninja +// reads (§20 T1), and then removed. A failed compile leaves no rule: ninja +// reads none for a failed edge. +std::expected write_inputs_rule(const std::filesystem::path& record, + const std::filesystem::path& depfile, + const std::filesystem::path& output, bool compiled) { + std::error_code ec; + if (!compiled) { + std::filesystem::remove(record, ec); + return {}; + } + std::ifstream in(record, std::ios::binary); + if (!in) return std::unexpected(std::format("the compiler wrote no record of the files it read ({})", + mcpp::home::spelling(record))); + const std::string text{std::istreambuf_iterator(in), std::istreambuf_iterator()}; + in.close(); + // The rule names the edge's output as the edge does: the text ninja gave. + const std::string target = mcpplibs::os::fs::generic_utf8(output); + mcpp::build::depfile::SourceRecord source{.record = text, .target = target}; + source.buildDir = mcpplibs::os::fs::generic_utf8(std::filesystem::current_path(ec)); + for (const char* variable : {"INCLUDE", "EXTERNAL_INCLUDE"}) + if (auto value = mcpp::invocation::get(variable)) + for (auto const& dir : mcpplibs::os::env::split_list(*value)) + source.toolchainDirs.push_back(mcpplibs::os::fs::generic_utf8(dir)); + source.foldCase = mcpplibs::os::conventions().case_insensitive_names; + source.spelled = on_disk_name; + auto rule = mcpp::build::depfile::rule_from_source_dependencies(source); + if (!rule) return std::unexpected(std::format("{}: {}", mcpp::home::spelling(record), rule.error())); + { + std::ofstream out(depfile, std::ios::binary | std::ios::trunc); + out << *rule; + out.close(); + if (!out) return std::unexpected(std::format("{} cannot be written", mcpp::home::spelling(depfile))); + } + std::filesystem::remove(record, ec); + return {}; +} + +// The compiler's text, decoded where it is read (§20 T2): a line on. +void write_line(std::string_view line) { + std::string out(line); + out += '\n'; + (void)mcpplibs::os::stdio::write_output(out); +} + +} // namespace + int run_capped(const std::vector& argv, const std::filesystem::path& semaphore, int maxCompilers, const std::filesystem::path& output, const std::filesystem::path& depfile, - const std::filesystem::path& bmi, bool object, std::string_view relocate) { + const std::filesystem::path& bmi, bool object, std::string_view relocate, + const Reading& reading) { if (argv.empty()) { std::println(std::cerr, "mcpp: __compiler-cap: no compiler command was given"); return 2; } auto actor = std::make_shared(); if (!actor->ready) return 130; @@ -1352,6 +1438,18 @@ int run_capped(const std::vector& argv, // helper that ended with 1 or 2 and nothing on its error stream left a // failed build with no cause (Termux, 2026-11-18). const auto out_name=output.empty() ? std::string("a compile") : mcpp::home::spelling(output); + const bool decoding = reading.text != mcpplibs::os::text::encoding::utf8; + // The compiler's record of the files it read, as the rule ninja reads + // (§20 T1); the status the edge ends with. + auto read_inputs = [&](int compiled) { + if (reading.sourceDependencies.empty()) return compiled; + auto ruled = write_inputs_rule(reading.sourceDependencies, depfile, output, + compiled == 0 && actor->owner.active()); + if (ruled) return compiled; + std::println(std::cerr, "mcpp: writing the files the compile read as its dependency file for {}: {}", + out_name, ruled.error()); + return 1; + }; auto ended=[&](int status,std::string_view step,std::string_view why) { std::println(std::cerr,"mcpp: {} for {}: {}",step,out_name,why);return status; }; @@ -1405,17 +1503,21 @@ int run_capped(const std::vector& argv, if(!transactions::publish(attempt.directory/"admitted","mcpp-admitted-v1",{})) return ended(2,"recording the compiler's admission","its record cannot be written"); const auto result=mcpp::run::execute(command,{ - .input=mcpp::run::Input::inherit,.streams=mcpp::run::Streams::inherit, - .should_stop=[&] {return !actor->owner.active();},.declared_unit=unit.get()}); + .input=mcpp::run::Input::inherit,.streams=decoding?mcpp::run::Streams::merged:mcpp::run::Streams::inherit, + .on_line=decoding?std::function(write_line):nullptr, + .should_stop=[&] {return !actor->owner.active();},.declared_unit=unit.get(),.text=reading.text}); if(auto retired=retire_owned(*unit,actor->owner);!retired) return ended(1,"retiring the compiler unit",retired.error()); - if(auto settled=settle(result.exit_code==0&&actor->owner.active());!settled) + // A compile whose record cannot be written as its rule failed: a rule + // that is not there would leave the files it read untracked. + const int status=read_inputs(result.exit_code); + if(auto settled=settle(status==0&&actor->owner.active());!settled) return ended(1,"publishing the compile's outputs",settled.error()); - if(!attempt.retired(result.exit_code)||!attempt.result(result.exit_code)) + if(!attempt.retired(status)||!attempt.result(status)) return ended(1,"certifying the compiler's result","its record cannot be written"); if(!transactions::release_reservation(permit->reservation,attempt.directory)) return ended(1,"releasing the compiler slot","its reservation cannot be released"); - return result.exit_code; + return status; } // An interrupted compile in place leaves no output a later build would // take for complete; a relocated one leaves its own directory only. @@ -1429,15 +1531,18 @@ int run_capped(const std::vector& argv, if (!actor->owner.active()) return 130; const auto result = mcpp::run::execute(command, { .input = mcpp::run::Input::inherit, - .streams = mcpp::run::Streams::inherit, - .should_stop = [&] { return !actor->owner.active(); }}); - if (!actor->owner.active() || result.exit_code != 0) { + .streams = decoding ? mcpp::run::Streams::merged : mcpp::run::Streams::inherit, + .on_line = decoding ? std::function(write_line) : nullptr, + .should_stop = [&] { return !actor->owner.active(); }, + .text = reading.text}); + const int status = read_inputs(result.exit_code); + if (!actor->owner.active() || status != 0) { std::error_code ec; if (!withdrawn.empty()) std::filesystem::remove_all(withdrawn, ec); - return actor->owner.active() ? result.exit_code : 130; + return actor->owner.active() ? status : 130; } if (auto settled = settle(true); !settled) return ended(1, "publishing the compile's outputs", settled.error()); - return result.exit_code; + return status; } } // namespace mcpp::build::schedule::detach diff --git a/src/cli.cppm b/src/cli.cppm index 38ff30dfa..95ce62904 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -31,6 +31,7 @@ import mcpp.wire; import mcpp.cli.cmd_sbom; import mcpplibs.os.platform; import mcpplibs.os.fs; +import mcpplibs.os.text; import mcpp.invocation; import mcpp.effects; import mcpp.runtime.search; @@ -283,6 +284,7 @@ int run(int argc, char** argv) { std::filesystem::path depfile; std::filesystem::path bmi; std::string relocate; + mcpp::build::schedule::detach::Reading reading; bool object = false; int cap = 0, i = 2; for (; i < argc && std::string_view(argv[i]) != "--"; ++i) { @@ -291,8 +293,9 @@ int run(int argc, char** argv) { // compiler writes the bytes it had. if (option == "--object") { object = true; continue; } if ((option != "--sem" && option != "--cap" && option != "--output" && option != "--depfile" - && option != "--bmi" && option != "--relocate") || i + 1 >= argc) { - std::println(libc::stderr(), "error: __compiler-cap: expected --sem [--cap ] [--object] [--bmi ] [--relocate ] -- ..."); + && option != "--bmi" && option != "--relocate" && option != "--source-dependencies" + && option != "--text") || i + 1 >= argc) { + std::println(libc::stderr(), "error: __compiler-cap: expected --sem [--cap ] [--object] [--bmi ] [--relocate ] [--source-dependencies ] [--text console] -- ..."); return 2; } const std::string_view value = argv[++i]; @@ -301,7 +304,14 @@ int run(int argc, char** argv) { else if (option == "--depfile") depfile = value; else if (option == "--bmi") bmi = value; else if (option == "--relocate") relocate = value; - else { + else if (option == "--source-dependencies") reading.sourceDependencies = value; + else if (option == "--text") { + if (value != "console") { + std::println(libc::stderr(), "error: __compiler-cap: --text takes `console`, not `{}`", value); + return 2; + } + reading.text = mcpplibs::os::text::encoding::console; + } else { const auto [end, ec] = std::from_chars(value.data(), value.data() + value.size(), cap); if (ec != std::errc{} || end != value.data() + value.size()) return 2; } @@ -310,7 +320,8 @@ int run(int argc, char** argv) { if (semaphore.empty() || cap < 0 || i + 1 >= argc) return 2; std::vector command; for (++i; i < argc; ++i) command.emplace_back(argv[i]); - return mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, object, relocate); + return mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, object, relocate, + reading); } ReportUnnarrowablePaths reportUnnarrowable_; @@ -1417,6 +1428,7 @@ int run(int argc, char** argv) { .option(cl::Option("companions").takes_value().value_name("SUFFIXES").help("the row's companions, removed before the link (`.wasm`)")) .option(cl::Option("aside").help("move the previous artifact aside first (a Windows host)")) .option(cl::Option("temporary").help("link under a temporary name in the artifact's directory, then rename it over the artifact (POSIX)")) + .option(cl::Option("text").takes_value().value_name("ENCODING").help("`console`: the command writes its text in the console's code page, which is read as UTF-8 (link.exe, lib.exe)")) .action([&passthrough](const cl::ParsedArgs& p) { return cmd_link_output(p, std::span(passthrough)); })) diff --git a/src/cli/cmd_build.cppm b/src/cli/cmd_build.cppm index 957dec064..7df837f41 100644 --- a/src/cli/cmd_build.cppm +++ b/src/cli/cmd_build.cppm @@ -39,6 +39,7 @@ import nlohmann.json; import mcpp.log; import mcpplibs.os.fs; import mcpplibs.os.process; // platform-step: a program's output alone +import mcpplibs.os.stdio; // link-output: a linker's text, decoded import mcpp.invocation; import mcpp.effects; import mcpp.run; @@ -315,7 +316,7 @@ export int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { } // One line, as every record of the stream is: a reader splits stdout on // newlines and parses each line, the last of which is this envelope. - std::println("{}", mcpp::wire::to_json(env).dump()); + std::println("{}", mcpp::wire::text(mcpp::wire::to_json(env))); return rc; } @@ -1864,6 +1865,10 @@ export int cmd_dyndep(const mcpplibs::cmdline::ParsedArgs& parsed) { // rename replaces a running program there, and a program started meanwhile // never meets a partial file. The publication is one command, so an edge that // holds a permit for the link holds it until the artifact is in place. +// +// `--text console` (design 2026-11-19 §20 T2): link.exe and lib.exe write their +// text in the console's code page, so the command's two streams are read +// through one pipe and written on as UTF-8, the text of everything mcpp reads. export int cmd_link_output(const mcpplibs::cmdline::ParsedArgs& parsed, std::span command) { namespace fs = std::filesystem; @@ -1872,6 +1877,23 @@ export int cmd_link_output(const mcpplibs::cmdline::ParsedArgs& parsed, std::println(libc::stderr(), "error: link-output needs --output -- "); return 2; } + const std::string text = parsed.option_or_empty("text").value(); + if (!text.empty() && text != "console") { + std::println(libc::stderr(), "error: link-output: --text takes `console`, not `{}`", text); + return 2; + } + // The linker, on this program's streams or through the decoding pipe. + auto run_linker = [&](const std::vector& argv) { + if (text.empty()) + return mcpp::run::execute(argv, {.input = mcpp::run::Input::inherit, + .streams = mcpp::run::Streams::inherit}).exit_code; + return mcpp::run::execute(argv, {.input = mcpp::run::Input::inherit, + .streams = mcpp::run::Streams::merged, + .on_line = [](std::string_view line) { + (void)mcpplibs::os::stdio::write_output(std::string(line) + "\n"); + }, + .text = mcpplibs::os::text::encoding::console}).exit_code; + }; std::error_code ec; // The row's companions (`--companions .wasm`, design 2026-11-15 N33): the // files named after the artifact that an earlier link wrote and this one @@ -1898,11 +1920,9 @@ export int cmd_link_output(const mcpplibs::cmdline::ParsedArgs& parsed, return 2; } *std::next(named) = temporary.string(); - const auto linked = mcpp::run::execute(argv, {.input = mcpp::run::Input::inherit, - .streams = mcpp::run::Streams::inherit}); - if (linked.exit_code != 0) { + if (const int linked = run_linker(argv); linked != 0) { fs::remove(temporary, ec); - return linked.exit_code; + return linked; } fs::rename(temporary, out, ec); if (ec) { @@ -1913,8 +1933,10 @@ export int cmd_link_output(const mcpplibs::cmdline::ParsedArgs& parsed, } return 0; } - if (!parsed.is_flag_set("aside")) + if (!parsed.is_flag_set("aside")) { + if (!text.empty()) return run_linker(std::vector(command.begin(), command.end())); return mcpp::run::run_exec(std::vector(command.begin(), command.end())); + } const auto dir = out.parent_path().empty() ? fs::path(".") : out.parent_path(); const auto stem = out.filename().string() + ".old-"; for (auto it = fs::directory_iterator(dir, ec); !ec && it != fs::directory_iterator(); it.increment(ec)) @@ -1931,6 +1953,7 @@ export int cmd_link_output(const mcpplibs::cmdline::ParsedArgs& parsed, break; } } + if (!text.empty()) return run_linker(std::vector(command.begin(), command.end())); return mcpp::run::run_exec(std::vector(command.begin(), command.end())); } @@ -2115,7 +2138,7 @@ export int cmd_env_configuration(const mcpplibs::cmdline::ParsedArgs& parsed) { env.data = nullptr; env.diagnostics.push_back({"MCPP_CONFIGURATION_FAILED", mcpp::wire::Severity::Error, failure}); - std::println("{}", mcpp::wire::to_json(env).dump()); + std::println("{}", mcpp::wire::text(mcpp::wire::to_json(env))); } else { mcpp::ui::error(failure); } diff --git a/src/run.cppm b/src/run.cppm index 98637f8da..d3a3a0e1b 100644 --- a/src/run.cppm +++ b/src/run.cppm @@ -18,6 +18,7 @@ export module mcpp.run; import std; import mcpplibs.os.process; +export import mcpplibs.os.text; export namespace mcpp::run { @@ -162,6 +163,11 @@ struct Options { // An explicitly declared unit whose registration precedes this start. // Its owner retains the unit until retirement and final bookkeeping. mcpplibs::os::process::owned_unit* declared_unit = nullptr; + // THE ENCODING OF THE PROGRAM'S TEXT (design 2026-11-19 §20 T2). mcpp's + // text is UTF-8; a program that writes another is read as UTF-8 here, each + // line and the kept output alike, where its bytes enter mcpp. `console`: + // the convention of the cl.exe toolchain's programs (mcpplibs.os.text). + mcpplibs::os::text::encoding text = mcpplibs::os::text::encoding::utf8; }; RunResult execute(const std::vector& argv, const Options& options = {}); @@ -180,7 +186,8 @@ Shell shell(std::string_view command); // A command line mcpp composed as text and records as such (a std module's // build commands, kept in its metadata): its standard output read, its own // redirections deciding the rest, an empty standard input. -RunResult capture_shell(std::string_view command, const Env& extraEnv = {}); +RunResult capture_shell(std::string_view command, const Env& extraEnv = {}, + mcpplibs::os::text::encoding text = mcpplibs::os::text::encoding::utf8); // A user-authored command (`[hooks]`) on mcpp's streams, in `cwd`, bounded. // 127 when the interpreter could not be started. diff --git a/src/run/run.cpp b/src/run/run.cpp index 568531054..5b8800a11 100644 --- a/src/run/run.cpp +++ b/src/run/run.cpp @@ -627,9 +627,16 @@ int stream_exec(const std::vector& argv, const Env& extraEnv, // ─── Any program, as the caller states ─────────────────────────────────── RunResult execute(const std::vector& argv, const Options& opt) { + // A program whose text is not UTF-8 is read as UTF-8 from here on. + std::optional decode; + std::function decodedLine; + if (opt.text != os::text::encoding::utf8) { + decode.emplace(opt.text); + if (opt.on_line) decodedLine = [&](std::string_view line) { opt.on_line((*decode)(line)); }; + } Spec s{ .argv = argv, .env = opt.env, .unset = opt.unset, .hostTool = opt.host_tool, .cwd = opt.cwd, .nullInput = opt.input == Input::none, .total = opt.total, .idle = opt.idle, - .onLine = opt.on_line ? &opt.on_line : nullptr, .splitOnCr = opt.split_on_cr, + .onLine = decodedLine ? &decodedLine : (opt.on_line ? &opt.on_line : nullptr), .splitOnCr = opt.split_on_cr, .shouldStop = opt.should_stop ? &opt.should_stop : nullptr, .oneString = opt.one_string, .declaredUnit = opt.declared_unit }; switch (opt.streams) { @@ -643,8 +650,8 @@ RunResult execute(const std::vector& argv, const Options& opt) { auto o = launch(s); RunResult r; r.exit_code = o.exit_code; - r.output = std::move(o.output); - r.error_output = std::move(o.error_output); + r.output = decode ? (*decode)(o.output) : std::move(o.output); + r.error_output = decode ? (*decode)(o.error_output) : std::move(o.error_output); r.timed_out = o.timed_out; r.spawn_error = o.spawn_error; if (o.spawn_error != 0) { @@ -673,10 +680,11 @@ Shell shell(std::string_view command) { } } -RunResult capture_shell(std::string_view command, const Env& extraEnv) { +RunResult capture_shell(std::string_view command, const Env& extraEnv, os::text::encoding text) { auto sh = shell(command); auto o = launch({ .argv = sh.argv, .env = extraEnv, .out = Output::stdout_err_shown, .nullInput = true, .oneString = sh.one_string }); + if (text != os::text::encoding::utf8) o.output = os::text::decoder(text)(o.output); return result_of(std::move(o), nullptr, true); } diff --git a/src/toolchain/msvc.cppm b/src/toolchain/msvc.cppm index 46b89b4ec..ef79df8a3 100644 --- a/src/toolchain/msvc.cppm +++ b/src/toolchain/msvc.cppm @@ -439,7 +439,10 @@ std::filesystem::path vc_redist_dir_for_tools_dir( // INCLUDE = \include; \Include\\{ucrt,um,shared,winrt} // LIB = \lib\; \Lib\\{ucrt,um}\ // PATH = ; (mspdb*.dll etc.) -// VSLANG = 1033 (stable English /showIncludes prefix for ninja deps=msvc) +// No VSLANG: the compiler's language is the user's environment's. Until +// 2026.11.19.1 this set VSLANG=1033 for an English /showIncludes prefix, which +// a Visual Studio without English resources ignores; the files a compile read +// now come from cl.exe's /sourceDependencies record (design 2026-11-19 §20). std::vector build_env_for_cl(const std::filesystem::path& clPath, std::string_view arch, const WindowsSdk& sdk); @@ -1524,8 +1527,6 @@ std::vector build_env_for_cl(const std::filesystem::path& clPath, path += p; } env.push_back({"PATH", std::move(path)}); - // Stable English "Note: including file:" prefix for ninja's deps=msvc. - env.push_back({"VSLANG", "1033"}); return env; } @@ -1778,7 +1779,7 @@ std::expected enrich_toolchain_from_cl(Toolchain& tc) { tc.importStdMinLevel = std_module_min_level_for_stl(tc.stdModuleSource); } - // Build environment (INCLUDE/LIB/PATH/VSLANG). SDK absence keeps + // Build environment (INCLUDE/LIB/PATH). SDK absence keeps // detection working (selection UX on SDK-less boxes); the build path // errors with guidance when envOverrides is empty. // diff --git a/src/toolchain/stdmod.cppm b/src/toolchain/stdmod.cppm index 9dd40b4af..319a38a32 100644 --- a/src/toolchain/stdmod.cppm +++ b/src/toolchain/stdmod.cppm @@ -46,6 +46,7 @@ import mcpp.toolchain.hostflags; import mcpp.toolchain.linkmodel; import mcpp.toolchain.msvc; import mcpp.toolchain.msvc_coroutines; +import mcpp.toolchain.dialect; export namespace mcpp::toolchain { @@ -121,8 +122,9 @@ namespace { std::expected run_capture_command( const std::string& cmd, - const std::vector>& env) { - auto r = mcpp::run::capture_shell(cmd, env); + const std::vector>& env, + mcpplibs::os::text::encoding text) { + auto r = mcpp::run::capture_shell(cmd, env, text); if (r.exit_code != 0) { // Include the command: its --sysroot/-isystem flags are the first // thing needed to diagnose header-resolution failures. @@ -260,9 +262,13 @@ std::expected run_commands( const std::vector& commands, const Toolchain& tc) { std::vector> env; for (auto& ev : tc.envOverrides) env.emplace_back(ev.key, ev.value); + // The compiler's text in the encoding its dialect states (design + // 2026-11-19 §20 T2): cl.exe's in the console's code page. + const auto text = mcpp::toolchain::dialect_for(tc).textEncoding == std::string_view("console") + ? mcpplibs::os::text::encoding::console : mcpplibs::os::text::encoding::utf8; std::string out; for (auto const& cmd : commands) { - if (auto r = run_capture_command(cmd, env); !r) return std::unexpected(r.error()); + if (auto r = run_capture_command(cmd, env, text); !r) return std::unexpected(r.error()); else out += *r; } return out; diff --git a/src/wire.cppm b/src/wire.cppm index af00704db..97dec0b54 100644 --- a/src/wire.cppm +++ b/src/wire.cppm @@ -313,11 +313,23 @@ inline nlohmann::json to_json(const Envelope& e) { return out; } +// THE TEXT OF A DOCUMENT THAT CARRIES WHAT A PROGRAM WROTE (design 2026-11-19 +// §20 T3). JSON is UTF-8 (RFC 8259 §8.1), and no program's bytes may end mcpp +// with an internal error, as a compiler's GBK diagnostic did when it reached +// `result.json` (mcpp#970): a sequence that is not UTF-8 is written as U+FFFD. +// A program's text is decoded where mcpp reads it (T2), so what this replaces +// is a byte a program wrote as UTF-8 and was not -- a GCC diagnostic quoting a +// Latin-1 source line; the terminal and the logs keep it as written. +template +inline std::string text(const Json& document, int indent = -1) { + return document.dump(indent, ' ', false, Json::error_handler_t::replace); +} + // Serialise and write to stdout. Two spaces, trailing newline: a client reads // it with a JSON parser either way, and a person reading a bug report should // not have to reformat it first. inline void emit(const Envelope& e) { - std::println("{}", to_json(e).dump(2)); + std::println("{}", text(to_json(e), 2)); } // ── Negotiation ──────────────────────────────────────────────────────────── diff --git a/tests/e2e/windows/msvc.cpp b/tests/e2e/windows/msvc.cpp index 6fa6e3b51..b1b32ab87 100644 --- a/tests/e2e/windows/msvc.cpp +++ b/tests/e2e/windows/msvc.cpp @@ -2,7 +2,8 @@ // (modules, `import std` at C++20, incremental); one toolset is chosen once and every consumer reads // it; a build program reads the toolset as facts and builds the build program of a cross target; // objects carry their own debug information; LTO meets export discovery; two programs each bind -// their own module of one name; clang's missing coroutines on the 32-bit ABI are explained +// their own module of one name; clang's missing coroutines on the 32-bit ABI are explained; the +// headers a compile read and the text cl.exe wrote reach mcpp in any language and code page (970) // (formerly tests/e2e/95, 99, 177, 180, 241, 760, 825, 848, 887, 889, 892, 1017, 1023). import std; import mcpplibs.mtest; @@ -474,6 +475,66 @@ int main() { f(); } expect(fatal(e2e::contains(s::text(r), "mspdbsrv") && e2e::contains(s::text(r), "/Z7"))) << "the refusal does not say why /Zi is refused and what to write: " << s::text(r); expect(fatal(!e2e::contains(s::text(r), "Compiling"))) << "the refusal came after compiling began: " << s::text(r); }; + "970 msvc reads what a compile read and what it wrote in any language and code page"_test[s::msvc] = [] { + // mcpp#970 (design 2026-11-19 §20): the headers a compile read come from cl.exe's + // /sourceDependencies record, so an edit of a header rebuilds whatever the installed language + // and the console's code page; cl.exe's text is decoded where the executor reads it, so a + // failure under a console in code page 936 reaches the terminal and result.json as the text it + // was, and nothing ends mcpp with an internal error. The project's directory is not ASCII and + // its header's has a space. A console of its own (`start /wait`) holds code page 936, so the + // console the other cases share keeps its own. + const auto root = s::case_dir() / fs::path(u8"\u4e2d\u6587\u00e9"); + s::project p(root, {{"mcpp.toml", "[package]\nname = \"cpwords\"\nversion = \"0.1.0\"\n[toolchain]\nwindows = \"msvc@system\"\n" + "[build]\ninclude_dirs = [\"inc dir\"]\n"}, + {"inc dir/hdr.h", "#pragma once\ninline int value() { return 0; }\n"}, + {"src/main.cpp", "#include \"hdr.h\"\nint main() { return value(); }\n"}}); + auto r = p.mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << "the project did not build: " << s::text(r); + const auto graph = s::read_file(ninja_of(p.dir())); + expect(fatal(e2e::contains(graph, "/sourceDependencies"))) << "cl.exe's compile writes no record"; + expect(fatal(!e2e::contains(graph, "/showIncludes") && !e2e::contains(graph, "deps = msvc"))) + << "a compile still reads the /showIncludes text"; + // The header the compile read is its input: an edit rebuilds, and fails where it should. + std::this_thread::sleep_for(std::chrono::seconds(1)); + p.write("inc dir/hdr.h", "#pragma once\nstatic_assert(false, \"edited\");\ninline int value() { return 0; }\n"); + r = p.mcpp({"build"}); + expect(fatal(failed(r))) << "an edit of the header did not rebuild the unit that includes it: " << s::text(r); + p.write("inc dir/hdr.h", "#pragma once\ninline int value() { return 0; }\n"); + r = p.mcpp({"build"}); + expect(fatal(e2e::succeeded(r))) << "the restored header did not build: " << s::text(r); + + // A failure under a console in code page 936: cl.exe writes GBK, the executor reads it as such. + p.write("src/main.cpp", "#include \"hdr.h\"\nint main() { return \u672a\u5b9a\u4e49_\u00e9; }\n"); + const auto script = s::case_dir() / "cp936.bat"; + { + std::ofstream bat(script, std::ios::binary); + bat << "@echo off\r\nchcp 936 >nul\r\ncd /d \"%CP_PROJECT%\"\r\n\"%CP_MCPP%\" build > \"%CP_OUT%\" 2>&1\r\n" + "exit /b %errorlevel%\r\n"; + } + const auto output = s::case_dir() / "cp936.txt"; + // The paths as UTF-8, which the environment carries to cmd.exe whatever its code page. + const auto utf8 = [](const fs::path& path) { + const auto text = path.u8string(); + return std::string(text.begin(), text.end()); + }; + r = s::run("cmd.exe", {"/d", "/c", "start", "/wait", "/min", "cmd.exe", "/d", "/c", utf8(script)}, + s::case_dir(), + {{"CP_PROJECT", utf8(p.dir())}, {"CP_MCPP", s::mcpp_path()}, {"CP_OUT", utf8(output)}}); + const auto said = s::read_file(output); + expect(fatal(!said.empty())) << "the build in the console of code page 936 wrote nothing: " << s::text(r); + expect(fatal(!e2e::contains(said, "internal:"))) << "mcpp ended with an internal error: " << said; + const std::string identifier = "\xE6\x9C\xAA\xE5\xAE\x9A\xE4\xB9\x89_\xC3\xA9"; + expect(fatal(e2e::contains(said, identifier) && e2e::contains(said, "C2065"))) + << "the terminal does not show cl.exe's diagnostic as the text it was: " << said; + const auto last = s::json_file(p.at("target/.last-result.json"))["data"]; + const auto result = s::json_file(fs::path(last["result"].get()))["data"]; + expect(fatal(result["status"] == "failed")) << result.dump(); + bool named = false; + for (auto const& d : result["diagnostics"]) + named = named || d["message"].get().contains(identifier); + expect(fatal(named)) << "result.json does not carry cl.exe's diagnostic as UTF-8: " << result.dump(); + }; + "241 windows ucrt runtime identity"_test[s::msvc] = [] { // The Windows SDK has an identity, `ucrt@`, in the runtime binding of the cl.exe row // (pinned msvc@system; 760 states it of the clang row): it takes part in the contract hash diff --git a/tests/unit/test_build_result.cpp b/tests/unit/test_build_result.cpp new file mode 100644 index 000000000..a0b4441fc --- /dev/null +++ b/tests/unit/test_build_result.cpp @@ -0,0 +1,52 @@ +// mcpp.build.result: the diagnostics a failed step's output yields +// (design 2026-10-26 §5.1), in every language cl.exe speaks (mcpp#970, +// design 2026-11-19 §20 T4). The Chinese lines are the bytes cl.exe 19.44 +// wrote with the zh-cn resources on windows-2022 (probe mcpp#971), decoded. + +import std; +import mcpplibs.mtest; +import mcpp.build.result; + +using namespace mcpplibs::mtest; + +namespace r = mcpp::build::result; + +UTest msvc_errors_and_notes_in_english{"BuildResult/MsvcErrorsAndNotesInEnglish", [] { + const auto d = r::parse_compiler_diagnostics( + "k.cpp\r\n" + "k.cpp(2): error C2668: 'h': ambiguous call to overloaded function\r\n" + "k.cpp(1): note: could be 'void h(long)'\r\n" + "k.cpp(2,7): warning C4189: 'unused': local variable is initialized but not referenced\r\n"); + expect(fatal(eq(d.size(), std::size_t{3}))); + expect(eq(d[0].severity, std::string("error")) && eq(d[0].path, std::string("k.cpp")) && eq(d[0].line, 2)); + expect(eq(d[0].message, std::string("'h': ambiguous call to overloaded function"))); + // A note carries no code: its location is the file's, not `k.cpp(1)`. + expect(eq(d[1].severity, std::string("note")) && eq(d[1].path, std::string("k.cpp")) && eq(d[1].line, 1)); + expect(eq(d[1].message, std::string("could be 'void h(long)'"))); + expect(eq(d[2].severity, std::string("warning")) && eq(d[2].line, 2) && eq(d[2].column, 7)); +}}; + +UTest msvc_errors_and_notes_in_chinese{"BuildResult/MsvcErrorsAndNotesInChinese", [] { + const auto d = r::parse_compiler_diagnostics( + "b.cpp(2): error C2065: \xE2\x80\x9C\xE6\x9C\xAA\xE5\xAE\x9A\xE4\xB9\x89_\xC3\xA9\xE2\x80\x9D: " + "\xE6\x9C\xAA\xE5\xA3\xB0\xE6\x98\x8E\xE7\x9A\x84\xE6\xA0\x87\xE8\xAF\x86\xE7\xAC\xA6\n" + "k.cpp(1): note: \xE5\x8F\xAF\xE8\x83\xBD\xE6\x98\xAF\xE2\x80\x9Cvoid h(long)\xE2\x80\x9D\n" + "c.cpp(1): fatal error C1083: \xE6\x97\xA0\xE6\xB3\x95\xE6\x89\x93\xE5\xBC\x80\xE5\x8C\x85\xE6\x8B\xAC" + "\xE6\x96\x87\xE4\xBB\xB6: \xE2\x80\x9C\xE7\xBC\xBA\xE5\xA4\xB1.h\xE2\x80\x9D: No such file or directory\n"); + expect(fatal(eq(d.size(), std::size_t{3}))); + expect(eq(d[0].severity, std::string("error")) && eq(d[0].path, std::string("b.cpp")) && eq(d[0].line, 2)); + expect(d[0].message.starts_with("\xE2\x80\x9C\xE6\x9C\xAA")) << d[0].message; + expect(eq(d[1].severity, std::string("note")) && eq(d[1].path, std::string("k.cpp")) && eq(d[1].line, 1)); + expect(eq(d[2].severity, std::string("error")) && eq(d[2].path, std::string("c.cpp"))); +}}; + +UTest gnu_diagnostics_are_read_as_before{"BuildResult/GnuDiagnosticsAreReadAsBefore", [] { + const auto d = r::parse_compiler_diagnostics( + "src/a.cpp:3:5: error: 'y' was not declared in this scope\n" + "src/a.cpp:1:1: note: declared here\n" + "cc1plus: error: unrecognized command-line option '-fno'\n"); + expect(fatal(eq(d.size(), std::size_t{3}))); + expect(eq(d[0].path, std::string("src/a.cpp")) && eq(d[0].line, 3) && eq(d[0].column, 5)); + expect(eq(d[1].severity, std::string("note"))); + expect(eq(d[2].path, std::string("cc1plus")) && eq(d[2].line, 0)); +}}; diff --git a/tests/unit/test_depfile.cpp b/tests/unit/test_depfile.cpp index 5f945fd18..4be5f8bd0 100644 --- a/tests/unit/test_depfile.cpp +++ b/tests/unit/test_depfile.cpp @@ -50,3 +50,76 @@ UTest windows_paths_and_crlf_lines_stay_in_the_record{"Depfile/WindowsPathsAndCr expect(eq(first_record(raw), "obj/a.o: C:/src/a.cpp \\\r\n" " C:/Program\\ Files/inc/a.h\r\n")); }}; + +// ── cl.exe's record (mcpp#970, design 2026-11-19 §20 T1) ───────────────────── +// +// The record is the one cl.exe 19.44 wrote on windows-2022 (probe mcpp#971): +// UTF-8, every path in lower case, the toolchain's and the SDK's headers +// listed with the project's. The executor writes the Make rule ninja reads. + +namespace { + +using mcpp::build::depfile::SourceRecord; +using mcpp::build::depfile::rule_from_source_dependencies; + +constexpr std::string_view kRecord = R"({ + "Version": "1.2", + "Data": { + "Source": "c:\\p970\\\u4e2d\u6587\u00e9\\b.cpp", + "ProvidedModule": "", + "Includes": [ + "c:\\p970\\\u4e2d\u6587\u00e9\\inc dir\\h\u00e9ader.h", + "c:\\program files\\microsoft visual studio\\2022\\enterprise\\vc\\tools\\msvc\\14.44.35207\\include\\vector", + "c:\\program files (x86)\\windows kits\\10\\include\\10.0.26100.0\\ucrt\\stdio.h", + "c:\\p970\\build\\gen\\config.h", + "c:\\p970\\build\\gen\\with#hash$.h" + ], + "ImportedModules": [], + "ImportedHeaderUnits": [] + } +})"; + +// The file system's spelling of a name below the build directory, by a table. +std::string spelled(const std::string& dir, const std::string& name) { + if (name == "gen") return "Gen"; + if (name == "config.h") return "Config.h"; + (void)dir; + return name; +} + +SourceRecord record_of(std::string_view text) { + return {.record = text, .target = "obj/b.obj", .buildDir = "C:/p970/Build", + .toolchainDirs = {"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Tools\\MSVC\\14.44.35207\\include\\", + "C:/Program Files (x86)/Windows Kits/10/include/10.0.26100.0/ucrt"}, + .foldCase = true, .spelled = spelled}; +} + +} // namespace + +UTest the_record_becomes_the_rule_without_the_toolchains_headers{"Depfile/TheRecordBecomesTheRuleWithoutTheToolchainsHeaders", [] { + const auto rule = rule_from_source_dependencies(record_of(kRecord)); + expect(fatal(rule.has_value())) << (rule ? std::string() : rule.error()); + expect(eq(*rule, std::string("obj/b.obj: \\\n" + " c:/p970/\xE4\xB8\xAD\xE6\x96\x87\xC3\xA9/inc\\ dir/h\xC3\xA9" "ader.h \\\n" + " C:/p970/Build/Gen/Config.h \\\n" + " C:/p970/Build/Gen/with\\#hash$$.h\n"))); +}}; + +// A header the build generated is another edge's output: its node is the build +// directory's spelling and the file's own, not the record's lower case. +UTest a_header_under_the_build_directory_keeps_its_spelling{"Depfile/AHeaderUnderTheBuildDirectoryKeepsItsSpelling", [] { + auto in = record_of(kRecord); + in.foldCase = false; // a host whose names keep their case: the build directory is not matched + const auto rule = rule_from_source_dependencies(in); + expect(fatal(rule.has_value())); + expect(ne(rule->find(" c:/p970/build/gen/config.h"), std::string::npos)) << *rule; +}}; + +UTest a_record_that_is_not_one_is_refused{"Depfile/ARecordThatIsNotOneIsRefused", [] { + expect(!rule_from_source_dependencies(record_of("not json")).has_value()); + expect(!rule_from_source_dependencies(record_of(R"({"Version": "1.2"})")).has_value()); + expect(!rule_from_source_dependencies(record_of(R"({"Data": {"Includes": [1]}})")).has_value()); + const auto none = rule_from_source_dependencies(record_of(R"({"Data": {"Includes": []}})")); + expect(fatal(none.has_value())); + expect(eq(*none, std::string("obj/b.obj:\n"))) << "a unit that includes nothing has a rule of its own"; +}}; diff --git a/tests/unit/test_ninja_backend.cpp b/tests/unit/test_ninja_backend.cpp index bd894e5b0..097407e6c 100644 --- a/tests/unit/test_ninja_backend.cpp +++ b/tests/unit/test_ninja_backend.cpp @@ -141,6 +141,50 @@ UTest bitcode_export_tools_come_from_the_selected_llvm_and_are_quoted_as_words{" expect(eq(emit_ninja_string(plan).find("coff_args ="), std::string::npos)); }}; +// mcpp#970 (design 2026-11-19 §20): cl.exe's compile writes its +// /sourceDependencies record, which the executor writes as the Make rule +// every compile edge reads; no compile reads the /showIncludes text. The +// executor, link-output and the archive edge read the text of the cl.exe +// toolchain's programs in the console's code page. A GNU toolchain's commands +// carry neither word. +UTest msvc_compiles_read_the_record_and_its_tools_text_is_decoded{"NinjaBackend/MsvcCompilesReadTheRecordAndItsToolsTextIsDecoded", [] { + auto plan = minimal_plan(); + plan.toolchain.compiler = mcpp::toolchain::CompilerId::MSVC; + plan.toolchain.binaryPath = "cl.exe"; + plan.toolchain.targetTriple = "x86_64-pc-windows-msvc"; + plan.scheduleCompilerCap = 2; + plan.compileUnits.push_back({.source = "src/main.cpp", .kind = mcpp::SourceKind::Cxx, + .object = "obj/main.obj", .packageName = "app"}); + LinkUnit lib; + lib.kind = LinkUnit::StaticLibrary; + lib.targetName = "core"; + lib.output = "lib/core.lib"; + lib.objects = {"obj/main.obj"}; + plan.linkUnits.push_back(lib); + const auto text = emit_ninja_string(plan); + auto rule = [&](std::string_view name) { + const auto start = text.find("rule " + std::string(name) + "\n"); + expect(fatal(ne(start, std::string::npos))) << name; + return text.substr(start, text.find("\n\n", start) - start); + }; + const auto object = rule("cxx_object"); + expect(ne(object.find("/sourceDependencies $out.json"), std::string::npos)) << object; + expect(ne(object.find("--source-dependencies $out.json --text console -- "), std::string::npos)) << object; + expect(ne(object.find("deps = gcc\n depfile = $out.d"), std::string::npos)) << object; + expect(eq(text.find("/showIncludes"), std::string::npos)); + expect(eq(text.find("deps = msvc"), std::string::npos)); + const auto archive = rule("cxx_archive"); + expect(ne(archive.find("$mcpp link-output --output $out --text console -- $ar"), std::string::npos)) << archive; + expect(ne(rule("cxx_link").find(" --text console -- $ld"), std::string::npos)) << rule("cxx_link"); + + auto gnu = minimal_plan(); + gnu.scheduleCompilerCap = 2; + gnu.compileUnits.push_back({.source = "src/main.cpp", .kind = mcpp::SourceKind::Cxx, .object = "obj/main.o"}); + const auto gnuText = emit_ninja_string(gnu); + expect(eq(gnuText.find("--source-dependencies"), std::string::npos)); + expect(eq(gnuText.find("--text"), std::string::npos)); +}}; + UTest ordinary_compilers_share_the_detached_schedule_bound{"NinjaBackend/OrdinaryCompilersShareTheDetachedScheduleBound", [] { auto plan = minimal_plan(); plan.toolchain.version = "16.1.0"; @@ -2248,7 +2292,13 @@ UTest names_the_consumer_directory_for_each_driver_spelling{"ConsumerIncludeScop const std::string cl = "dep.cpp(1): fatal error C1083: Cannot open include file: 'cfg7.h': " "No such file or directory\n"; - for (auto const& out : {gcc, clang, cl}) { + // The same report from cl.exe with the Chinese resources (mcpp#970, probe + // mcpp#971): the words and the quote marks are the language's. + const std::string clChinese = + "dep.cpp(1): fatal error C1083: \xE6\x97\xA0\xE6\xB3\x95\xE6\x89\x93\xE5\xBC\x80" + "\xE5\x8C\x85\xE6\x8B\xAC\xE6\x96\x87\xE4\xBB\xB6: \xE2\x80\x9C" "cfg/nested.h\xE2\x80\x9D: " + "No such file or directory\n"; + for (auto const& out : {gcc, clang, cl, clChinese}) { auto advice = mcpp::build::consumer_include_scope_advice(out, dirs); expect(fatal(!advice.empty())) << out; expect(ne(advice.find((root / "appinc").string()), std::string::npos)) << advice; @@ -2266,6 +2316,10 @@ UTest silent_when_the_header_is_not_the_consumers_or_no_header_is_missing{"Consu // A failure that is not a missing header, although cfg7.h is the consumer's. expect(mcpp::build::consumer_include_scope_advice( "dep.c:3:1: error: unknown type name 'cfg7'\n", dirs).empty()); + // A source file that cannot be opened is C1083 too, with no line of a file. + expect(mcpp::build::consumer_include_scope_advice( + "c1xx: fatal error C1083: Cannot open source file: 'cfg7.h': No such file or directory\n", + dirs).empty()); // No root include directories at all (the fast path before any plan wrote // the sidecar). expect(mcpp::build::consumer_include_scope_advice( diff --git a/tests/unit/test_wire.cpp b/tests/unit/test_wire.cpp index b8995daa8..1441d6af0 100644 --- a/tests/unit/test_wire.cpp +++ b/tests/unit/test_wire.cpp @@ -288,3 +288,18 @@ UTest null_data_is_omitted{"WireGolden/NullDataIsOmitted", [] { }}; } // namespace + +// A document that carries what a program wrote is UTF-8 whatever it wrote +// (design 2026-11-19 §20 T3): cl.exe's GBK diagnostic, which ended mcpp with +// `type_error.316` in result.json (mcpp#970), is written with U+FFFD. +UTest a_programs_bytes_never_end_the_writer{"WireText/AProgramsBytesNeverEndTheWriter", [] { + const nlohmann::json doc{{"message", std::string("error C2065: '\xCE\xB4\xB6\xA8\xD2\xE5_\xA8\xA6'")}}; + std::string text; + expect(nothrow([&] { text = w::text(doc); })); + const auto back = nlohmann::json::parse(text, nullptr, false); + expect(fatal(!back.is_discarded())) << text; + expect(ne(back["message"].get().find("\xEF\xBF\xBD"), std::string::npos)); + const nlohmann::ordered_json utf8{{"message", "\xE6\x9C\xAA\xE5\xAE\x9A\xE4\xB9\x89"}}; + expect(eq(w::text(utf8), std::string("{\"message\":\"\xE6\x9C\xAA\xE5\xAE\x9A\xE4\xB9\x89\"}"))) + << "UTF-8 is written as it is"; +}}; From d31db6b188a4d53d816b0d9ca45aac39d52ddb07 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 20:40:43 +0900 Subject: [PATCH 16/30] mcpp 2026.11.19.1 --- mcpp.toml | 2 +- modules/versioning/src/version.cppm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mcpp.toml b/mcpp.toml index 58ac88539..56f237c6e 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,6 +1,6 @@ [package] name = "mcpp" -version = "2026.11.18.1" +version = "2026.11.19.1" description = "Modern C++ build & package management tool" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/modules/versioning/src/version.cppm b/modules/versioning/src/version.cppm index e2b8c9d39..5a5ee3409 100644 --- a/modules/versioning/src/version.cppm +++ b/modules/versioning/src/version.cppm @@ -31,6 +31,6 @@ import std; export namespace mcpp { -inline constexpr std::string_view MCPP_VERSION = "2026.11.18.1"; +inline constexpr std::string_view MCPP_VERSION = "2026.11.19.1"; } // namespace mcpp From 737441d1db77d53b801b822254b46d33b09144e8 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 21:11:57 +0900 Subject: [PATCH 17/30] The pack launchers build on the mcpplibs.os release mcpp builds on Both launchers `mcpp pack` writes (bin/.exe and the single-file launcher) named mcpplibs.os 0.4.4, whose openkal ends below 0.26; with openkal-platform 0.31 as the platform of mcpp's openkal rows, a launcher could not be built and e2e 988 failed. They name one constant, kLauncherOs, the release mcpp builds on (0.4.6), and .github/tools/check_version_pins.sh holds it equal to mcpp.toml's. --- .github/tools/check_version_pins.sh | 22 ++++++++++++++++++++-- CHANGELOG.md | 3 +++ src/pack/distribution.cppm | 16 ++++++++++++---- tests/unit/test_pack_distribution.cpp | 2 +- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.github/tools/check_version_pins.sh b/.github/tools/check_version_pins.sh index 7ec089b66..0c1fb4d44 100755 --- a/.github/tools/check_version_pins.sh +++ b/.github/tools/check_version_pins.sh @@ -2,7 +2,7 @@ # # Version / pin drift guard. # -# Two invariants that used to live only in a comment: +# Three invariants that used to live only in a comment: # # 1. Every xlings version pinned anywhere in .github/ equals # `pinned::kXlingsVersion` in src/provider/xlings/xlings.cppm — which is the version @@ -11,6 +11,8 @@ # # 2. mcpp's own version is identical in all four places that carry it. # +# 3. The pack launchers build on the mcpplibs.os release mcpp builds on. +# # Why this exists: src/provider/xlings/xlings.cppm used to say "keep in lock-step with the # XLINGS_VERSION pins in release.yml / cross-build-test.yml / ci-linux-e2e.yml" # and that list was ALREADY incomplete — it omitted both composite actions, @@ -173,7 +175,23 @@ if [ -n "$v_xl" ] && [ -n "$v_toml" ] && [ "$v_xl" != "$v_toml" ]; then || bad "bootstrap pin '$v_xl' is NEWER than the version being built ('$v_toml') — CI would try to install an unreleased mcpp" fi +# ── 3. the OS library of the pack launchers ─────────────────────────────── +# +# The launchers `mcpp pack` writes build on the mcpplibs.os release mcpp builds +# on itself (design 2026-11-07, D20). The launcher's copy sat on 0.4.4 when mcpp +# and its default platform moved to openkal 0.26, and every Windows launcher was +# refused for its openkal (e2e 988, 2026.11.19.1's CI). + +os_toml=$(awk -F '"' '/^mcpplibs\.os[[:space:]]*=/{print $2; exit}' mcpp.toml) +os_launcher=$(grep -oE 'kLauncherOs[[:space:]]*=[[:space:]]*"[^"]+"' src/pack/distribution.cppm \ + | grep -oE '"[^"]+"' | tr -d '"' | head -1) +note "mcpplibs.os: mcpp builds on $os_toml, the pack launchers on $os_launcher" +[ -n "$os_toml" ] && [ -n "$os_launcher" ] \ + || bad "could not read mcpplibs.os from mcpp.toml or kLauncherOs from src/pack/distribution.cppm" +[ "$os_toml" = "$os_launcher" ] \ + || bad "src/pack/distribution.cppm builds the launchers on mcpplibs.os '$os_launcher' but mcpp.toml builds mcpp on '$os_toml'" + if [ "$fail" = 0 ]; then - echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl" >&2 + echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl; mcpplibs.os $os_toml" >&2 fi exit "$fail" diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7ef909a..c50523460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,9 @@ - A build program that pins `mcpp.plugins` below 0.26 shares no host graph with `openkal-platform@0.31.0`, whose build program links `mcpp.plugins` 0.26.1: such a pin is refused as irreconcilable on the rows openkal serves, and moves to 0.26.1. +- The launchers `mcpp pack` writes (`bin/.exe`, the single-file launcher) build on + `mcpplibs.os` 0.4.6, the release mcpp builds on, whose openkal is the one the platform of + mcpp's openkal rows names; `.github/tools/check_version_pins.sh` holds the two equal. ### Fixed diff --git a/src/pack/distribution.cppm b/src/pack/distribution.cppm index e2e95bc82..d1b06fede 100644 --- a/src/pack/distribution.cppm +++ b/src/pack/distribution.cppm @@ -88,6 +88,14 @@ std::string posix_launcher(const Launch& l); std::string exe_launcher_manifest(std::string_view name, std::string_view version); std::string exe_launcher_main(const Launch& l); +// The OS library both launchers build on: the release this mcpp builds on +// (mcpp.toml), so a pack is the same at any later time (design 2026-11-07, +// D20) and the launcher's openkal is the one the platform of mcpp's openkal +// rows names. 0.4.4 named openkal below 0.26, and with openkal-platform 0.31 +// as that platform a launcher could not be built (e2e 988). +// .github/tools/check_version_pins.sh holds the two equal. +inline constexpr std::string_view kLauncherOs = "0.4.6"; + // ─── mcpp-package.json (SPEC-014 §4.5) ──────────────────────────────────── inline constexpr std::string_view kFileName = "mcpp-package.json"; @@ -1113,9 +1121,9 @@ std::string exe_launcher_manifest(std::string_view name, std::string_view versio "[targets.{2}]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n\n" "# The OS library as one release, the one this mcpp was qualified with, so a\n" "# pack is the same at any later time (design 2026-11-07, D20).\n" - "[dependencies]\nmcpplibs.os = \"0.4.4\"\n", + "[dependencies]\nmcpplibs.os = \"{3}\"\n", name, version.empty() ? std::string_view("0.0.0") : version, - mcpp::libs::toml::escape_string(name)); + mcpp::libs::toml::escape_string(name), kLauncherOs); } std::string exe_launcher_main(const Launch& l) { @@ -1483,9 +1491,9 @@ std::string launcher_manifest(std::string_view name, std::string_view version) { "[package]\nname = \"{0}\"\nversion = \"{1}\"\n\n" "[targets.{2}]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n\n" "[build]\nsources = [\"src/main.cpp\", \"src/carried.S\"]\n\n" - "[dependencies]\nmcpplibs.os = \"0.4.4\"\nmcpplibs.mwrap.zstd = \"1.5.7\"\n", + "[dependencies]\nmcpplibs.os = \"{3}\"\nmcpplibs.mwrap.zstd = \"1.5.7\"\n", name, version.empty() ? std::string_view("0.0.0") : version, - mcpp::libs::toml::escape_string(name)); + mcpp::libs::toml::escape_string(name), kLauncherOs); } std::string_view launcher_main() { diff --git a/tests/unit/test_pack_distribution.cpp b/tests/unit/test_pack_distribution.cpp index 92e9820df..426f01498 100644 --- a/tests/unit/test_pack_distribution.cpp +++ b/tests/unit/test_pack_distribution.cpp @@ -261,7 +261,7 @@ UTest the_windows_launcher_is_a_program{"PackDistribution/TheWindowsLauncherIsAP expect(ne(src.find("(\"windows-\" + arch)"), std::string::npos)) << src; expect(ne(src.find("os::process::exec"), std::string::npos)) << src; const auto manifest = d::exe_launcher_manifest("hello", "1.0.0"); - expect(ne(manifest.find("mcpplibs.os = \"0.4.4\""), std::string::npos)) << manifest; + expect(ne(manifest.find(std::format("mcpplibs.os = \"{}\"", d::kLauncherOs)), std::string::npos)) << manifest; expect(ne(manifest.find("[targets.\"hello\"]"), std::string::npos)) << manifest; }}; From 57c02a1c1f04beb540722bf3880aef89a1245d32 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 22:13:32 +0900 Subject: [PATCH 18/30] The launchers build on the os the platform's build program links; cl.exe's record reaches ninja as lines - The pack launchers pin mcpplibs.os 0.4.5, the release the build program of openkal-platform 0.31 links through mcpp.plugins 0.26.1: one resolution holds the consumer's graph and the platform's build-program dependencies, so the launcher takes the platform's os (mcpp#972). The pin-equality invariant added in the previous commit is withdrawn. - A compile under the msvc dialect gives ninja the files it read as `mcpp.inputs:` lines read under `deps = msvc`, so a path ninja's depfile parser would split (a quote, `&`, `;`, `^`, a backquote) arrives whole; a build directory spelled with non-ASCII names is matched by its respelling. - Every JSON record the build writes (build-database envelopes, test archive manifests, run records, plan records, build-program graphs) is written as UTF-8 text. - A response file on a link line is split by the rules of the tool that reads it: CommandLineToArgvW's under the MSVC dialect, a GNU driver's otherwise (mcpp#931). - The advice for a lost header recognises C1083 whatever its quote marks and the spacing around its colon (mcpp#970). --- ...11-19-ecosystem-rule-convergence-design.md | 33 +++++++-- .github/tools/check_version_pins.sh | 22 +----- CHANGELOG.md | 24 ++++--- docs/20-toolchains.md | 2 +- docs/30-build-mcpp.md | 5 +- docs/specs/concurrency.md | 11 +-- docs/specs/manifest-semantics.md | 4 +- docs/zh/20-toolchains.md | 2 +- docs/zh/30-build-mcpp.md | 4 +- src/build/command_line.cpp | 39 +++++++++++ src/build/command_line.cppm | 8 +++ src/build/depfile.cppm | 67 +++++++++++++------ src/build/execute.cppm | 14 ++-- src/build/flags.cppm | 12 ++-- src/build/ninja_backend.cppm | 67 ++++++++++++------- src/build/prepare/build_programs.cpp | 4 +- src/build/schedule/detach_codegen.cppm | 54 +++++++-------- src/cli/cmd_build.cppm | 8 +-- src/pack/distribution.cppm | 16 +++-- tests/e2e/windows/msvc.cpp | 4 +- tests/unit/test_build_flags.cpp | 15 ++++- tests/unit/test_command_words.cpp | 14 ++++ tests/unit/test_depfile.cpp | 59 ++++++++++------ tests/unit/test_ninja_backend.cpp | 21 ++++-- 24 files changed, 337 insertions(+), 172 deletions(-) diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md index c4ff7451a..63b69b17d 100644 --- a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md @@ -835,6 +835,17 @@ xim-pkgindex 中 mcpp 自身的条目由发布流程的自动拉取请求产生 后者的引擎下界使两者不能合为一次发布。 - Windows 上执行器以改名发布 BMI 时偶遇 `being used by another process`(CI 中约 350 个 BMI 出现一次,其他进程短暂打开了 刚写出的文件)。改名对暂时的拒绝(`again`、`permission`)在 30 s 内重试(`replace_file_within`),其后的拒绝照旧报告。 +- `mcpp pack` 生成的启动器(`bin/.exe`、单文件启动器)在 mcpp 的 openkal 行的平台上构建,平台构建程序的依赖与启动器 + 自己的依赖在同一次解析中各取一个版本:启动器的 `mcpplibs.os` 为 0.4.4(平台 0.29 的构建程序所用)或 0.4.6(mcpp 自身所用) + 时都被拒为不可调和,0.4.5(平台 0.31 经 mcpp.plugins 0.26.1)可行(e2e 988 与 canaries 的打包格,本地 wine 复核)。启动器 + 因此须抄写平台内部构建工具的版本,这正是 mcpp#972 所述的根因;根本修正为按程序解析构建依赖(见 mcpp#972 的讨论)。 +- 11b 之后的独立复核:`mcpp.build-database` 的两个信封仍以 `dump` 写出(不是 UTF-8 的字节使其异常结束);ninja 读取 Make 规则 + 时在 `'`、`&`、`;`、`^` 与反引号处结束名字(cl.exe 的记录因此改为前缀行,T1);构建目录的前缀只折叠 ASCII 的大小写;C1083 + 的操作数在冒号前后无空格或只有一个冒号的语言中取不到。四者均已修正并有单元测试。MSVC 方言的响应文件按 link.exe 的规则 + 读取(Windows 的单元测试测得:按 GNU 规则读,反斜杠使原生路径失效)。 +- GNU 驱动写的 Make 规则不转义 `'`、`&`、`;`、`^` 与反引号(GCC 只转义空格、`#` 与 `$`),这类路径在 GNU 行上仍受 ninja 的 + 这一限制:依赖成为不存在的文件,所含单元每次构建都重编。这是本轮之前即有的限制,留作待决:把 GNU 驱动的记录也经执行器 + 写为前缀行,即一切编译边以同一无损通道交给 ninja。 - `-dumpdir` 的探测在不写全局缓存的命令(`self env` 等)中创建了冷 home(macOS 的 e2e 测得):这类命令在临时目录中探测, 回答不保存,home 不变(SPEC-011 §6)。 @@ -890,14 +901,19 @@ windows-2022,探针 mcpp#971(运行 44508574496),以 `CreateProcessW` **T1 依赖取自编译器为程序写出的记录。** 一次编译读取的文件取自编译器为程序写出的记录,不取自写给人看的文本。方言以数据 说明记录的形式(`inputsRecord`):GNU 驱动写 Make 规则(`-MMD -MF $out.d`,`make`),cl.exe 写 `/sourceDependencies` -的 JSON 记录(`source-dependencies`)。执行器把 JSON 记录写成同形的依赖文件,ninja 对每个编译边以 `deps = gcc` 读取。 +的 JSON 记录(`source-dependencies`)。执行器把 JSON 记录中的路径逐行写到自己的标准输出(前缀 `mcpp.inputs:`、一个 +空格与路径),cl.exe 的编译边以 `deps = msvc` 与 `msvc_deps_prefix = mcpp.inputs:` 读取,GNU 驱动的编译边仍以 +`deps = gcc` 读取它自己写的 Make 规则。不写成 Make 规则:ninja 1.12.1 读取 Make 规则时在 `'`、`&`、`;`、`^` 与反引号处 +结束一个名字(`a'b.h` 读成 `a` 与 `b.h`,实测),Windows 的路径可以含有它们(`C:\Users\O'Neil`),而前缀行的其余部分 +整体是路径;前缀是 mcpp 的,不随语言改变。 - 位于工具链环境的 `INCLUDE` 与 `EXTERNAL_INCLUDE` 目录之下的头文件不写入,与 `-MMD` 不列系统头文件相同;工具链与 SDK 的改变由工具链的身份与构建环境反映。 - 位于构建目录(执行器的工作目录,即 ninja 的 `-C`)之下的路径,写作该目录的写法加上其下各级名字在磁盘上的写法:记录把 路径改为小写,而生成的头文件是图中另一条边的输出,节点名须与之相同,否则生成器重新运行之后包含它的单元要到下一次构建 - 才重编。其余路径按记录写出,Windows 的文件系统以不区分大小写的名字打开它们。 -- 编译成功后写出依赖文件,失败时不写(ninja 不读取失败的边的依赖);记录在两种情况下都删除。 -- 依赖因此与语言、控制台的代码页无关。`/showIncludes` 与 `deps = msvc` 不再使用,`VSLANG` 不再设置:编译器的语言是 + 才重编。与构建目录只在 ASCII 以外字母的大小写上不同的路径,按磁盘上各级名字的写法再比较一次(名字比较只折叠 ASCII)。 + 其余路径按记录写出,Windows 的文件系统以不区分大小写的名字打开它们。 +- 编译成功后写出路径,失败时不写(ninja 不读取失败的边的依赖);记录在两种情况下都删除。 +- 依赖因此与语言、控制台的代码页无关。cl.exe 的 `/showIncludes` 不再使用,`VSLANG` 不再设置:编译器的语言是 用户环境的(`VSLANG` 或 Visual Studio 的界面语言),要英文诊断的用户自己设 `VSLANG=1033`。 **T2 文本在读入它的进程中按产生者的编码解码。** mcpp 内部的文本是 UTF-8。一个程序的文本在读入它的 mcpp 进程中解码, @@ -914,14 +930,17 @@ windows-2022,探针 mcpp#971(运行 44508574496),以 `CreateProcessW` - 解码不改变内容:源文件名一行、诊断及其顺序保持编译器写出的样子。代码页不能表示的字符在 cl 写出时已成为 `?`,解码 不能恢复;依赖不受影响(T1)。终端为 UTF-8 控制台(`chcp 65001`)时诊断保留全部字符。 -**T3 承载程序文本的文档总是 UTF-8。** `result.json`、`target/.last-result.json`、事件流与协议的信封承载程序写出的文本 -(诊断、失败的输出),由一个函数写出(`mcpp::wire::text`),把不是 UTF-8 的序列写作 U+FFFD(Unicode 第 3.9 节,最大子部分 +**T3 承载程序文本的文档总是 UTF-8。** `result.json`、`target/.last-result.json`、事件流、协议的信封(含 +`mcpp.build-database`)、测试运行的记录(`run.json`、`programs.ndjson`、`cases.ndjson`、`index.ndjson`)与构建程序 +的图承载程序写出的文本(诊断、失败的输出)或路径,由一个函数写出(`mcpp::wire::text`),把不是 UTF-8 的序列写作 U+FFFD(Unicode 第 3.9 节,最大子部分 的替换)。T2 之下 MSVC 的文本不再经过它;它留给声明为 UTF-8 而实际不是的字节,例如 GCC 引用一个 Latin-1 源文件的行。 终端与日志中的输出保持原样。 **T4 诊断按各语言不变的结构读取。** MSVC 的诊断为 `<路径>(<行>[,<列>]): <严重性>[ <代码>]: <正文>`,严重性为 `fatal error`、`error`、`warning`、`note`,中文版亦然(20.2)。不带代码的 `note:` 行按这一形式读取。缺失的头文件以代码 -`C1083` 与其操作数的位置识别:操作数是最后两个 `: ` 之间的一段,去掉首尾的一对引号,引号的字形随语言变化。GNU 驱动的 +`C1083` 与其操作数的位置识别:操作数是以最后一个冒号(其后是 C 运行库的原因,原因中没有冒号)之前结束的那段引号内的名字, +引号的字形(`'`、`“”`、`「」` 等)、冒号前后的空格与前面的词随语言变化(`file: 'x.h': No such`、`文件: “x.h”: No such`、 +`include : 'x.h' : No such`、`。'x.h':No such`)。GNU 驱动的 诊断不在本节:其严重性词随区域设置翻译,mcpp 不设置区域,行为不变。 ### 20.5 多视角复核 diff --git a/.github/tools/check_version_pins.sh b/.github/tools/check_version_pins.sh index 0c1fb4d44..7ec089b66 100755 --- a/.github/tools/check_version_pins.sh +++ b/.github/tools/check_version_pins.sh @@ -2,7 +2,7 @@ # # Version / pin drift guard. # -# Three invariants that used to live only in a comment: +# Two invariants that used to live only in a comment: # # 1. Every xlings version pinned anywhere in .github/ equals # `pinned::kXlingsVersion` in src/provider/xlings/xlings.cppm — which is the version @@ -11,8 +11,6 @@ # # 2. mcpp's own version is identical in all four places that carry it. # -# 3. The pack launchers build on the mcpplibs.os release mcpp builds on. -# # Why this exists: src/provider/xlings/xlings.cppm used to say "keep in lock-step with the # XLINGS_VERSION pins in release.yml / cross-build-test.yml / ci-linux-e2e.yml" # and that list was ALREADY incomplete — it omitted both composite actions, @@ -175,23 +173,7 @@ if [ -n "$v_xl" ] && [ -n "$v_toml" ] && [ "$v_xl" != "$v_toml" ]; then || bad "bootstrap pin '$v_xl' is NEWER than the version being built ('$v_toml') — CI would try to install an unreleased mcpp" fi -# ── 3. the OS library of the pack launchers ─────────────────────────────── -# -# The launchers `mcpp pack` writes build on the mcpplibs.os release mcpp builds -# on itself (design 2026-11-07, D20). The launcher's copy sat on 0.4.4 when mcpp -# and its default platform moved to openkal 0.26, and every Windows launcher was -# refused for its openkal (e2e 988, 2026.11.19.1's CI). - -os_toml=$(awk -F '"' '/^mcpplibs\.os[[:space:]]*=/{print $2; exit}' mcpp.toml) -os_launcher=$(grep -oE 'kLauncherOs[[:space:]]*=[[:space:]]*"[^"]+"' src/pack/distribution.cppm \ - | grep -oE '"[^"]+"' | tr -d '"' | head -1) -note "mcpplibs.os: mcpp builds on $os_toml, the pack launchers on $os_launcher" -[ -n "$os_toml" ] && [ -n "$os_launcher" ] \ - || bad "could not read mcpplibs.os from mcpp.toml or kLauncherOs from src/pack/distribution.cppm" -[ "$os_toml" = "$os_launcher" ] \ - || bad "src/pack/distribution.cppm builds the launchers on mcpplibs.os '$os_launcher' but mcpp.toml builds mcpp on '$os_toml'" - if [ "$fail" = 0 ]; then - echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl; mcpplibs.os $os_toml" >&2 + echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl" >&2 fi exit "$fail" diff --git a/CHANGELOG.md b/CHANGELOG.md index c50523460..6cdc3b454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,8 +63,9 @@ `openkal-platform@0.31.0`, whose build program links `mcpp.plugins` 0.26.1: such a pin is refused as irreconcilable on the rows openkal serves, and moves to 0.26.1. - The launchers `mcpp pack` writes (`bin/.exe`, the single-file launcher) build on - `mcpplibs.os` 0.4.6, the release mcpp builds on, whose openkal is the one the platform of - mcpp's openkal rows names; `.github/tools/check_version_pins.sh` holds the two equal. + `mcpplibs.os` 0.4.5, the release the build program of openkal-platform 0.31 links through + `mcpp.plugins` 0.26.1: a launcher is built over that platform, and the platform's build + dependencies are resolved with the launcher's own (mcpp#972). ### Fixed @@ -72,17 +73,24 @@ cl.exe compile read came from its `/showIncludes` lines, whose prefix is a string of the installed language (`注意: 包含文件:`), and `VSLANG=1033` cannot make it English (mcpp#970). They come from cl.exe's `/sourceDependencies` record, the same in every language and console - code page, which the executor writes as the compile's dependency file (SPEC-011 §5.1); a - header under the build directory keeps its spelling, so a generated header is the node its - generator writes. A record that cannot be read fails the compile with the reason. + code page, whose paths the executor gives ninja as lines of its output (`deps = msvc` with + mcpp's prefix `mcpp.inputs:`, SPEC-011 §5.1): ninja's reader of a Make rule ends a name at + `'`, `&`, `;`, `^` and a backquote, which a Windows path may hold. A header under the build + directory keeps its spelling, letters beyond ASCII included, so a generated header is the + node its generator writes. A record that cannot be read fails the compile with the reason. - A cl.exe failure on a Chinese console ended mcpp with `internal: unhandled exception: [json.exception.type_error.316] invalid UTF-8 byte`, and its diagnostics were unreadable: cl.exe, link.exe, lib.exe and rc.exe write their text in the code page of the console they run in, and mcpp read it as UTF-8 (mcpp#970). The executor reads their text in that page (with no console, the system's ANSI code page) and writes it on as UTF-8 (SPEC-012 §5), and every - document that carries a program's text writes a byte that is not UTF-8 as U+FFFD. A cl.exe - `note` in `result.json` carries the location of its file, and the advice for a header a - dependency lost names it in every language (`C1083`). + document that carries a program's text writes a byte that is not UTF-8 as U+FFFD: the result, + the events, the protocol envelopes (`mcpp.build-database` included) and a test run's records. + A cl.exe `note` in `result.json` carries the location of its file, and the advice for a header + a dependency lost names it in every language (`C1083`, whatever the quote marks and the space + around its colon). +- A response file on a link line under the MSVC dialect is read by link.exe's rules (those of + `CommandLineToArgvW`): the files it names by native paths are inputs of the link, where they + were read as a GNU driver reads one, a backslash taking the next character (mcpp#931). - A build interrupted in another process space -- a container job that reuses the working directory, a rebuilt development container, an agent's sandbox session -- could never be diff --git a/docs/20-toolchains.md b/docs/20-toolchains.md index c9b37b66d..b4d409c20 100644 --- a/docs/20-toolchains.md +++ b/docs/20-toolchains.md @@ -707,7 +707,7 @@ environment from the VC tools + Windows SDK (no `vcvarsall` involved), stages **The headers a compile read, and the compiler's text, in any language** *(2026.11.19.1+)*. Each compile writes cl.exe's `/sourceDependencies` record, -which mcpp writes as the compile's dependency file, so an edit of a header +whose paths mcpp gives ninja as the compile's dependencies, so an edit of a header rebuilds the units that include it whatever language the installed Visual Studio speaks and whatever code page the console uses. cl.exe, link.exe, lib.exe and rc.exe write their text in the code page of the console they run diff --git a/docs/30-build-mcpp.md b/docs/30-build-mcpp.md index 2ff08bb04..2e4b230ca 100644 --- a/docs/30-build-mcpp.md +++ b/docs/30-build-mcpp.md @@ -794,7 +794,10 @@ sources (SPEC-004 §8.1): the first `:` of a word beginning with `/`. A candidate is an input when it is an absolute path naming an existing regular file or a file an action of the build declares. `@` adds the response file and the absolute paths - it lists. `--js-library=`, `-Wl,--version-script=`, + it lists, split into words by the rules of the tool that reads it: those of + `CommandLineToArgvW` under the MSVC link dialect, where a backslash escapes + only a quote, and a GNU driver's otherwise *(2026.11.19.1+)*. + `--js-library=`, `-Wl,--version-script=`, `-Wl,-force_load,` and `/DEF:` are covered. 2. **The linker's report.** On an ELF target whose linker accepts `--dependency-file=` (GNU ld from 2.35, gold, ld.lld), the link edge diff --git a/docs/specs/concurrency.md b/docs/specs/concurrency.md index d31b29428..c39cb8cc4 100644 --- a/docs/specs/concurrency.md +++ b/docs/specs/concurrency.md @@ -138,10 +138,13 @@ CI。在 2026.10.24.1 上,同一配置的两次冷构建相隔 0.1–0.5 s 启 (`units_observable()` 为假),整次构建的编译不声明单元,仍受同一编译上限约束,并经 `degraded`(`compiler-run/no-unit`)报告一次。 11. 编译器为程序写出的、一次编译所读文件的记录不是 Make 规则时(工具链方言的 `inputsRecord`:cl.exe 的 - `/sourceDependencies`),执行器**必须**在编译成功后把它写为该边的依赖文件,ninja 以 `deps = gcc` 读取: - 工具链环境的 `INCLUDE` 与 `EXTERNAL_INCLUDE` 目录之下的头文件不写入;构建目录之下的路径写作构建目录的写法加上 - 磁盘上各级名字的写法;其余路径按记录写出。记录不能读出或依赖文件不能写出时该边**必须**失败,并写出原因; - 编译失败时不写依赖文件。记录在两种情况下都删除。依赖**禁止**取自编译器写给人看的文本(设计 2026-11-19 §20 T1)。 + `/sourceDependencies`),执行器**必须**在编译成功后把记录中的路径逐行写到自己的标准输出,每行为前缀 + `mcpp.inputs:`、一个空格与路径,该边以 `deps = msvc` 与 `msvc_deps_prefix = mcpp.inputs:` 读取:ninja 读取 Make + 规则时在 `'`、`&`、`;`、`^` 与反引号处结束一个名字,Windows 的路径可以含有它们,而这种行的其余部分整体是路径 + (以 ninja 1.12.1 测得)。工具链环境的 `INCLUDE` 与 `EXTERNAL_INCLUDE` 目录之下的头文件不写出;构建目录之下的路径 + 写作构建目录的写法加上磁盘上各级名字的写法,与构建目录只在 ASCII 以外的字母的大小写上不同的路径按磁盘上的写法 + 再比较;其余路径按记录写出。记录不能读出或路径不能写出时该边**必须**失败,并写出原因;编译失败时不写出路径。 + 记录在两种情况下都删除。依赖**禁止**取自编译器写给人看的文本(设计 2026-11-19 §20 T1)。 12. 工具链方言说明其程序的文本不是 UTF-8 时(`textEncoding = "console"`:cl.exe、link.exe、lib.exe、rc.exe), 运行该程序的执行器(`__compiler-cap`、`link-output`)**必须**以一个管道读取它的两个输出流,逐行解码为 UTF-8 后写到自己的标准输出;归档边因此也经 `link-output` 运行。解码**禁止**改变内容与顺序(SPEC-012 §5)。 diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index dc27c8d05..277599ad6 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -564,7 +564,9 @@ mcpp 2026.9.26.2,#703)。** 1. **点名的文件。** 对链接的每个词取候选:整个词;第一个 `=` 之后的部分;以 `,` 分隔的每一段, 以及段内第一个 `=` 之后的部分;MSVC 链接方言下以 `/` 开头的词中第一个 `:` 之后的部分。 - 候选以 `@` 开头时取其后的部分,并且该响应文件列出的绝对路径也是候选。一个候选是绝对路径, + 候选以 `@` 开头时取其后的部分,并且该响应文件列出的绝对路径也是候选;响应文件按读取它的工具的 + 规则分词,MSVC 链接方言下为 `CommandLineToArgvW` 的规则(反斜杠只在引号之前转义),其余为 GNU + 驱动的规则(反斜杠转义下一个字符)。一个候选是绝对路径, 并且指向一个现有的普通文件或本次规划中某个动作声明的输出时,它是携带该词的每个链接的隐式 输入,静态库的归档除外。目录不是输入;相对路径不是输入。 2. **链接器的报告。** 目标的对象格式为 ELF,且链接所运行的链接器接受 `--dependency-file=<文件>` diff --git a/docs/zh/20-toolchains.md b/docs/zh/20-toolchains.md index a28869141..f2578a762 100644 --- a/docs/zh/20-toolchains.md +++ b/docs/zh/20-toolchains.md @@ -626,7 +626,7 @@ staging 成 `.ifc` BMI,用 `/interface /TP /ifcOutput` 编译 `.cppm` 模块 `link.exe`/`lib.exe` 完成链接。 **一次编译读取的头文件与编译器的文本,与语言无关** *(2026.11.19.1+)*。每次编译写出 cl.exe 的 -`/sourceDependencies` 记录,mcpp 把它写为该编译的依赖文件,所以无论安装的 Visual Studio 是哪种语言、 +`/sourceDependencies` 记录,mcpp 把其中的路径作为该编译的依赖交给 ninja,所以无论安装的 Visual Studio 是哪种语言、 控制台用哪个代码页,修改头文件都会重新编译包含它的单元。cl.exe、link.exe、lib.exe 与 rc.exe 按它们运行 所在控制台的代码页写出文本(没有控制台时为系统的 ANSI 代码页),mcpp 按此读取,所以它们的诊断原样到达终端 与 `result.json`。mcpp 不设置 `VSLANG`:编译器使用用户环境的语言(`VSLANG`,否则为 Visual Studio 的界面语言), diff --git a/docs/zh/30-build-mcpp.md b/docs/zh/30-build-mcpp.md index f5387d7fa..1024f9ac7 100644 --- a/docs/zh/30-build-mcpp.md +++ b/docs/zh/30-build-mcpp.md @@ -666,7 +666,9 @@ stamp 不会喂给任何编译或链接边,只喂给下文说的那条 order-o 1. **链接词点名的文件。** 对链接行的每个词(`[build] ldflags`、`mcpp:link-flag=` 以及依赖传播的词) 取候选:整个词;`=` 之后的部分;以 `,` 分隔的每一段及段内 `=` 之后的部分;MSVC 链接方言下以 `/` 开头的词中第一个 `:` 之后的部分。候选是绝对路径,且指向一个现有的普通文件或构建中某个动作声明的 - 文件时,它是输入。`@<文件>` 加入该响应文件及其列出的绝对路径。`--js-library=<文件>`、 + 文件时,它是输入。`@<文件>` 加入该响应文件及其列出的绝对路径;响应文件按读取它的工具的规则分词: + MSVC 链接方言下为 `CommandLineToArgvW` 的规则,反斜杠只转义引号;其余为 GNU 驱动的规则 + *(2026.11.19.1+)*。`--js-library=<文件>`、 `-Wl,--version-script=<文件>`、`-Wl,-force_load,<文件>` 与 `/DEF:<文件>` 都在其中。 2. **链接器的报告。** 目标为 ELF、且链接器接受 `--dependency-file=<文件>`(GNU ld 2.35 起、gold、 ld.lld)时,链接边记录链接器读取的文件,与编译边记录编译器读取的头文件相同:经 `-L`、`-l` diff --git a/src/build/command_line.cpp b/src/build/command_line.cpp index 03509f3ba..2093d7ed3 100644 --- a/src/build/command_line.cpp +++ b/src/build/command_line.cpp @@ -143,6 +143,45 @@ std::vector split(std::string_view text, bool responseFile) { std::vector shell_words(std::string_view text) { return split(text, false); } std::vector response_file_words(std::string_view text) { return split(text, true); } +std::vector windows_response_file_words(std::string_view text) { + std::vector words; + std::string cur; + bool any = false, quoted = false; + std::size_t i = 0; + while (i < text.size()) { + const char c = text[i]; + if (c == '\\') { + std::size_t n = 0; + while (i + n < text.size() && text[i + n] == '\\') ++n; + const bool beforeQuote = i + n < text.size() && text[i + n] == '"'; + cur.append(beforeQuote ? n / 2 : n, '\\'); + any = true; + i += n; + if (beforeQuote && n % 2 == 1) { cur.push_back('"'); ++i; } + continue; + } + if (c == '"') { + if (quoted && i + 1 < text.size() && text[i + 1] == '"') { cur.push_back('"'); i += 2; continue; } + quoted = !quoted; + any = true; + ++i; + continue; + } + if (!quoted && (c == ' ' || c == '\t' || c == '\n' || c == '\r')) { + if (any) words.push_back(std::move(cur)); + cur.clear(); + any = false; + ++i; + continue; + } + cur.push_back(c); + any = true; + ++i; + } + if (any) words.push_back(std::move(cur)); + return words; +} + std::string display(const std::vector& argv) { std::string out; for (auto const& a : argv) { diff --git a/src/build/command_line.cppm b/src/build/command_line.cppm index 15d7a92a6..fa3297aa4 100644 --- a/src/build/command_line.cppm +++ b/src/build/command_line.cppm @@ -100,6 +100,14 @@ std::vector shell_words(std::string_view text); // as it is, and a backslash taking the next character, inside quotes or not. std::vector response_file_words(std::string_view text); +// The words of a response file as link.exe and lld-link read one, by the rules +// of CommandLineToArgvW: separated by white space outside double quotes, a +// backslash a character of the word unless backslashes precede a double quote +// (2n of them are n and the quote opens or closes; 2n+1 are n and a quote), +// and two double quotes inside quotes one of them. A native Windows path is a +// word as written (mcpp#931: the files a link's response file names). +std::vector windows_response_file_words(std::string_view text); + // Silent redirect — stdout + stderr → /dev/null (or NUL on Windows), and the // error stream alone, for a command a shell reads. constexpr std::string_view silent_redirect = mcpplibs::os::is_windows ? ">nul 2>&1" : ">/dev/null 2>&1"; diff --git a/src/build/depfile.cppm b/src/build/depfile.cppm index 46460657e..f0db7c4d4 100644 --- a/src/build/depfile.cppm +++ b/src/build/depfile.cppm @@ -42,27 +42,36 @@ inline std::string first_record(std::string_view raw) { return out; } -// THE RULE OF cl.exe'S RECORD (mcpp#970, design 2026-11-19 §20 T1). +// THE FILES cl.exe'S RECORD NAMES (mcpp#970, design 2026-11-19 §20 T1). // // cl.exe states the files a compile read in the JSON record // `/sourceDependencies ` writes, in UTF-8 and the same in every language // and console code page (measured on windows-2022, probe mcpp#971). The -// executor writes it as the Make rule every compile edge reads: +// executor gives them to ninja as lines of its own output, each `kInputsPrefix` +// and a path, which the edge reads as `deps = msvc` with that prefix: ninja +// takes the whole rest of such a line, where its reader of a Make rule ends a +// name at `'`, `&`, `;`, `^` or a backquote, which a Windows path may hold +// (measured with ninja 1.12.1). The paths: // * a header under one of `toolchainDirs` (the INCLUDE and EXTERNAL_INCLUDE // of the toolchain's environment) is left out, as -MMD leaves out a system // header; -// * a header under `buildDir`, the directory the compile ran in, is written -// as `buildDir` and then each name below it as the file system spells it +// * a header under `buildDir`, the directory the compile ran in, is named as +// `buildDir` and then each name below it as the file system spells it // (`spelled`): the record spells every path in lower case, and a generated // header is another edge's output, whose node keeps its case; -// * every other header is written as the record spells it, which the file +// * every other header is named as the record spells it, which the file // system of a cl.exe host opens without regard to case. // Paths are compared in the generic spelling (`/`), without ASCII case when -// `foldCase`. Not inline: an inline function of an imported module that -// iterates a string fails to compile in the importer under GCC 16.1. +// `foldCase`; a path that differs from `buildDir` in a letter beyond ASCII is +// compared again with its names as the file system spells them. Not inline: an +// inline function of an imported module that iterates a string fails to +// compile in the importer under GCC 16.1. +// The prefix ninja is given (`msvc_deps_prefix`): it skips the spaces after it, +// so a line is the prefix, a space and the path. +inline constexpr std::string_view kInputsPrefix = "mcpp.inputs:"; + struct SourceRecord { std::string_view record; // what /sourceDependencies wrote - std::string_view target; // the edge's output, as the rule names it std::string buildDir; // generic spelling std::vector toolchainDirs; // generic spellings bool foldCase = true; @@ -97,20 +106,26 @@ std::optional below(std::string_view path, std::string_view di return path.substr(dir.size() + 1); } -// A name in a Make rule: a space and `#` escaped, `$` doubled. -std::string escaped(std::string_view name) { - std::string out; - for (const char c : name) { - if (c == ' ' || c == '#') out += '\\'; - if (c == '$') out += '$'; - out += c; +// `path` with its first `count` names after the root as the file system spells +// them, and the rest as written; empty when it has fewer. +std::string respelled_prefix(std::string_view path, std::size_t count, const SourceRecord& in) { + const auto root = path.find('/'); + if (root == std::string_view::npos) return {}; + std::string out(path.substr(0, root + 1)); + std::size_t at = root + 1; + for (std::size_t n = 0; n < count; ++n) { + if (at >= path.size()) return {}; + const auto end = std::min(path.find('/', at), path.size()); + const std::string name(path.substr(at, end - at)); + out += (n == 0 ? "" : "/") + in.spelled(out, name); + at = end + 1; } - return out; + return at < path.size() ? out + '/' + std::string(path.substr(at)) : out; } } // namespace detail -std::expected rule_from_source_dependencies(const SourceRecord& in) { +std::expected, std::string> inputs_from_source_dependencies(const SourceRecord& in) { const auto doc = nlohmann::json::parse(in.record, nullptr, false); if (doc.is_discarded() || !doc.is_object()) return std::unexpected(std::string("the record is not a JSON object")); @@ -118,26 +133,34 @@ std::expected rule_from_source_dependencies(const Sour if (data == doc.end() || !data->is_object() || !data->contains("Includes") || !(*data)["Includes"].is_array()) return std::unexpected(std::string("the record has no Data.Includes list")); const std::string buildDir = detail::generic(in.buildDir); + const auto buildDepth = static_cast(std::ranges::count(buildDir, '/')); std::vector system; for (auto const& d : in.toolchainDirs) if (!d.empty()) system.push_back(detail::generic(d)); - std::string rule = detail::escaped(in.target) + ":"; + std::vector out; for (auto const& entry : (*data)["Includes"]) { if (!entry.is_string()) return std::unexpected(std::string("an entry of Data.Includes is not a path")); - const std::string path = detail::generic(entry.get()); + std::string path = detail::generic(entry.get()); if (std::ranges::any_of(system, [&](const std::string& d) { return detail::below(path, d, in.foldCase).has_value(); })) continue; + auto rest = detail::below(path, buildDir, in.foldCase); + if (!rest && in.foldCase && in.spelled && std::ranges::any_of(path, [](char c) { return (c & 0x80) != 0; })) + if (auto spelled = detail::respelled_prefix(path, buildDepth, in); + !spelled.empty() && detail::below(spelled, buildDir, true)) { + path = std::move(spelled); + rest = detail::below(path, buildDir, true); + } std::string named = path; - if (const auto rest = detail::below(path, buildDir, in.foldCase); rest && in.spelled) { + if (rest && in.spelled) { named = buildDir; for (auto part : std::views::split(*rest, '/')) { const std::string name(std::string_view(part.begin(), part.end())); named += '/' + in.spelled(named, name); } } - rule += " \\\n " + detail::escaped(named); + out.push_back(std::move(named)); } - return rule + "\n"; + return out; } } // namespace mcpp::build::depfile diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 5df045be7..969832aa8 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -266,7 +266,7 @@ std::expected write(const Manifest& m, const std::vectorheld}, {"failed", c.checks->failed}}; - cases << line.dump() << "\n"; + cases << mcpp::wire::text(line) << "\n"; } } } @@ -4694,8 +4694,8 @@ void write_mtest_store(const std::filesystem::path& dir, const std::vector link_selection_words(std::string_view rendered) { return out; } -// The absolute paths of existing files a response file names. -std::vector response_file_paths(const std::filesystem::path& rsp) { +// The absolute paths of existing files a response file names, its words read +// by the rules of the tool that reads it: link.exe's under the MSVC dialect, +// the GNU drivers' otherwise (where a backslash takes the next character). +std::vector response_file_paths(const std::filesystem::path& rsp, bool msvcDialect) { std::vector out; std::ifstream in(rsp, std::ios::binary); if (!in) return out; const std::string text { std::istreambuf_iterator(in), std::istreambuf_iterator() }; - for (auto const& word : mcpp::build::command_line::response_file_words(text)) { + const auto words = msvcDialect ? mcpp::build::command_line::windows_response_file_words(text) + : mcpp::build::command_line::response_file_words(text); + for (auto const& word : words) { std::error_code ec; if (!word.starts_with('-') && std::filesystem::path(word).is_absolute() && std::filesystem::is_regular_file(word, ec)) @@ -581,7 +585,7 @@ link_word_inputs(std::string_view word, bool msvcDialect, // rewrites nothing when the action rebuilds them (clangd's ok image, // 2026-11-02: a stale image after its libraries changed). if (responseFile && file) - for (auto& named : response_file_paths(p)) out.push_back(std::move(named)); + for (auto& named : response_file_paths(p, msvcDialect)) out.push_back(std::move(named)); } return out; } diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 44af318fa..84a209313 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -37,6 +37,7 @@ import mcpp.toolchain.detect; import mcpp.toolchain.model; import mcpp.modgraph.glob; import mcpp.toolchain.dialect; +import mcpp.build.depfile; // the prefix of the lines cl.exe's record is given in import mcpp.toolchain.provider; import mcpp.toolchain.registry; import mcpp.toolchain.triple; // shared_soname_flag decides by TARGET, not host @@ -1158,26 +1159,32 @@ std::string missing_header_in(std::string_view line) { return std::string(line.substr(open + 1, at - open - 1)); return {}; } - // cl.exe, in every language: `(): fatal error C1083: : - // : `. The words, the quote marks and the - // reason are the installed language's (`'x.h'`, `“x.h”`); the location, - // the code and the place of the name are not (design 2026-11-19 §20 T4). - // A missing source file is C1083 too, reported with no line of a file. + // cl.exe, in every language: `(): fatal error C1083: + // :`. The words, the quote marks, the space + // around the colon and the reason are the installed language's + // (`file: 'x.h': No such`, `文件: “x.h”: No such`, `include : 'x.h' : No + // such`, `。'x.h':No such`); the location, the code and the place of the + // name are not (design 2026-11-19 §20 T4): it is the quoted name that ends + // where the last colon begins the reason, which names no colon. constexpr std::string_view kC1083 = "): fatal error C1083: "; if (auto at = line.find(kC1083); at != std::string_view::npos) { - const auto rest = line.substr(at + kC1083.size()); - const auto first = rest.find(": "); - const auto last = rest.rfind(": "); - if (first == std::string_view::npos || last <= first + 2) return {}; - auto quoted = rest.substr(first + 2, last - first - 2); - // One mark at each end, a character of UTF-8 each. - std::size_t lead = 1; - if (static_cast(quoted[0]) >= 0x80) - while (lead < quoted.size() && (static_cast(quoted[lead]) & 0xC0) == 0x80) ++lead; - std::size_t tail = quoted.size() - 1; - while (tail > lead && (static_cast(quoted[tail]) & 0xC0) == 0x80) --tail; - if (tail <= lead) return {}; - return std::string(quoted.substr(lead, tail - lead)); + auto quoted = line.substr(at + kC1083.size()); + const auto reason = quoted.rfind(':'); + if (reason == std::string_view::npos) return {}; + quoted = quoted.substr(0, reason); + while (!quoted.empty() && quoted.back() == ' ') quoted.remove_suffix(1); + // The closing mark and the opening one it pairs with, in UTF-8. + constexpr std::array, 6> kMarks{{ + {"'", "'"}, {"\"", "\""}, {"\xE2\x80\x9D", "\xE2\x80\x9C"}, {"\xE2\x80\x99", "\xE2\x80\x98"}, + {"\xE3\x80\x8D", "\xE3\x80\x8C"}, {"\xC2\xBB", "\xC2\xAB"}}}; + for (auto const& [close, open] : kMarks) { + if (!quoted.ends_with(close)) continue; + const auto inner = quoted.substr(0, quoted.size() - close.size()); + const auto start = inner.rfind(open); + if (start == std::string_view::npos || start + open.size() >= inner.size()) return {}; + return std::string(inner.substr(start + open.size())); + } + return {}; } constexpr std::string_view kGcc = "fatal error: "; constexpr std::string_view kEnoent = ": No such file or directory"; @@ -2129,12 +2136,15 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // msvc); the rule *structure* is shared across compilers. // THE FILES A COMPILE READ COME FROM THE COMPILER'S RECORD FOR PROGRAMS // (mcpp#970, design 2026-11-19 §20 T1). cl.exe writes its JSON record - // (`/sourceDependencies $out.json`), which the executor writes as the Make - // rule every compile edge reads. Until 2026.11.19.1 these edges read the - // `/showIncludes` lines through ninja's `deps = msvc`: their prefix is a - // string of the installed language and their paths text of the console's - // code page, so a Visual Studio with only Chinese resources recorded no - // header at all, and `VSLANG=1033` could not make it English. + // (`/sourceDependencies $out.json`), which the executor gives ninja as + // lines of its own output, `kInputsPrefix` and a path each, read as + // `deps = msvc` with that prefix: ninja's reader of a Make rule ends a name + // at `'`, `&`, `;`, `^` or a backquote, which a Windows path may hold, and + // takes the rest of such a line whole. Until 2026.11.19.1 these edges read + // cl.exe's own `/showIncludes` lines: their prefix is a string of the + // installed language and their paths text of the console's code page, so a + // Visual Studio with only Chinese resources recorded no header at all, and + // `VSLANG=1033` could not make it English. const bool msvcDialect = dial.id == std::string_view("msvc"); const bool sourceRecord = dial.inputsRecord == std::string_view("source-dependencies"); const std::string compile_tail = std::format( @@ -2222,9 +2232,14 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, (needsGnuModuleFilter && mcpplibs::os::is_windows) ? "$mcpp depfile-filter --raw $out.d.raw --out $out.d -- " : ""; - // Both records arrive as the Make rule `$out.d`: the GNU driver writes it, - // and the executor writes cl.exe's. + // The GNU driver writes the Make rule `$out.d`; the executor gives + // cl.exe's record as lines of its output. auto append_cxx_deps = [&] { + if (sourceRecord) { + append(" deps = msvc\n"); + append(std::format(" msvc_deps_prefix = {}\n", mcpp::build::depfile::kInputsPrefix)); + return; + } append(" deps = gcc\n"); append(" depfile = $out.d\n"); }; diff --git a/src/build/prepare/build_programs.cpp b/src/build/prepare/build_programs.cpp index d8ab7211d..4b0a44965 100644 --- a/src/build/prepare/build_programs.cpp +++ b/src/build/prepare/build_programs.cpp @@ -202,7 +202,7 @@ std::expected step9_root_build_program(PrepareState& state) { nlohmann::json list = nlohmann::json::array(); for (auto i : order) list.push_back(state.graph_package_entry(i, true, nullptr)); doc["packages"] = std::move(list); - const auto text = doc.dump(2) + "\n"; + const auto text = mcpp::wire::text(doc, 2) + "\n"; const auto graphPath = bpEnv.artifactsDir / "graph.json"; std::error_code gec; std::filesystem::create_directories(graphPath.parent_path(), gec); @@ -448,7 +448,7 @@ write_graph_document(PrepareState& state, std::size_t subject, list.push_back(std::move(entry)); } doc["packages"] = std::move(list); - const auto text = doc.dump(2) + "\n"; + const auto text = mcpp::wire::text(doc, 2) + "\n"; const auto graphPath = artifactsDir / "graph.json"; std::error_code gec; std::filesystem::create_directories(graphPath.parent_path(), gec); diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm index 983cb24a9..19380cf2b 100644 --- a/src/build/schedule/detach_codegen.cppm +++ b/src/build/schedule/detach_codegen.cppm @@ -202,10 +202,10 @@ int await_unit(const std::filesystem::path& slot, const std::filesystem::path& o // // `reading` is what the executor reads for the compiler (design 2026-11-19 // §20): the record of the files the compile read when the compiler writes one -// that is not the Make rule `depfile` (cl.exe's `/sourceDependencies`, written -// as `depfile` once the compile succeeds), and the encoding of the compiler's -// text when it is not UTF-8, in which case its two streams are read through -// one pipe and written on as UTF-8. +// that is not the Make rule `depfile` (cl.exe's `/sourceDependencies`, given to +// ninja as lines of this executor's output once the compile succeeds), and the +// encoding of the compiler's text when it is not UTF-8, in which case its two +// streams are read through one pipe and written on as UTF-8. struct Reading { std::filesystem::path sourceDependencies; mcpplibs::os::text::encoding text = mcpplibs::os::text::encoding::utf8; @@ -1378,12 +1378,11 @@ std::string on_disk_name(const std::string& dir, const std::string& name) { return name; } -// The compiler's record of the files it read, written as the Make rule ninja -// reads (§20 T1), and then removed. A failed compile leaves no rule: ninja +// The compiler's record of the files it read, given to ninja as lines of this +// executor's output (§20 T1: `kInputsPrefix` and a path, which the edge reads +// as `deps = msvc`), and then removed. A failed compile gives none: ninja // reads none for a failed edge. -std::expected write_inputs_rule(const std::filesystem::path& record, - const std::filesystem::path& depfile, - const std::filesystem::path& output, bool compiled) { +std::expected write_inputs(const std::filesystem::path& record, bool compiled) { std::error_code ec; if (!compiled) { std::filesystem::remove(record, ec); @@ -1394,9 +1393,7 @@ std::expected write_inputs_rule(const std::filesystem::path& mcpp::home::spelling(record))); const std::string text{std::istreambuf_iterator(in), std::istreambuf_iterator()}; in.close(); - // The rule names the edge's output as the edge does: the text ninja gave. - const std::string target = mcpplibs::os::fs::generic_utf8(output); - mcpp::build::depfile::SourceRecord source{.record = text, .target = target}; + mcpp::build::depfile::SourceRecord source{.record = text}; source.buildDir = mcpplibs::os::fs::generic_utf8(std::filesystem::current_path(ec)); for (const char* variable : {"INCLUDE", "EXTERNAL_INCLUDE"}) if (auto value = mcpp::invocation::get(variable)) @@ -1404,14 +1401,17 @@ std::expected write_inputs_rule(const std::filesystem::path& source.toolchainDirs.push_back(mcpplibs::os::fs::generic_utf8(dir)); source.foldCase = mcpplibs::os::conventions().case_insensitive_names; source.spelled = on_disk_name; - auto rule = mcpp::build::depfile::rule_from_source_dependencies(source); - if (!rule) return std::unexpected(std::format("{}: {}", mcpp::home::spelling(record), rule.error())); - { - std::ofstream out(depfile, std::ios::binary | std::ios::trunc); - out << *rule; - out.close(); - if (!out) return std::unexpected(std::format("{} cannot be written", mcpp::home::spelling(depfile))); + auto inputs = mcpp::build::depfile::inputs_from_source_dependencies(source); + if (!inputs) return std::unexpected(std::format("{}: {}", mcpp::home::spelling(record), inputs.error())); + std::string lines; + for (auto const& path : *inputs) { + lines += mcpp::build::depfile::kInputsPrefix; + lines += ' '; + lines += path; + lines += '\n'; } + if (!lines.empty() && !mcpplibs::os::stdio::write_output(lines)) + return std::unexpected(std::string("the files it read cannot be written to ninja")); std::filesystem::remove(record, ec); return {}; } @@ -1439,15 +1439,13 @@ int run_capped(const std::vector& argv, // failed build with no cause (Termux, 2026-11-18). const auto out_name=output.empty() ? std::string("a compile") : mcpp::home::spelling(output); const bool decoding = reading.text != mcpplibs::os::text::encoding::utf8; - // The compiler's record of the files it read, as the rule ninja reads - // (§20 T1); the status the edge ends with. + // The compiler's record of the files it read, given to ninja (§20 T1); + // the status the edge ends with. auto read_inputs = [&](int compiled) { if (reading.sourceDependencies.empty()) return compiled; - auto ruled = write_inputs_rule(reading.sourceDependencies, depfile, output, - compiled == 0 && actor->owner.active()); - if (ruled) return compiled; - std::println(std::cerr, "mcpp: writing the files the compile read as its dependency file for {}: {}", - out_name, ruled.error()); + auto given = write_inputs(reading.sourceDependencies, compiled == 0 && actor->owner.active()); + if (given) return compiled; + std::println(std::cerr, "mcpp: giving ninja the files the compile of {} read: {}", out_name, given.error()); return 1; }; auto ended=[&](int status,std::string_view step,std::string_view why) { @@ -1508,8 +1506,8 @@ int run_capped(const std::vector& argv, .should_stop=[&] {return !actor->owner.active();},.declared_unit=unit.get(),.text=reading.text}); if(auto retired=retire_owned(*unit,actor->owner);!retired) return ended(1,"retiring the compiler unit",retired.error()); - // A compile whose record cannot be written as its rule failed: a rule - // that is not there would leave the files it read untracked. + // A compile whose record cannot be given to ninja failed: files it + // read that ninja does not know of would leave the edge untracked. const int status=read_inputs(result.exit_code); if(auto settled=settle(status==0&&actor->owner.active());!settled) return ended(1,"publishing the compile's outputs",settled.error()); diff --git a/src/cli/cmd_build.cppm b/src/cli/cmd_build.cppm index 7df837f41..5c5b7d5b8 100644 --- a/src/cli/cmd_build.cppm +++ b/src/cli/cmd_build.cppm @@ -752,12 +752,12 @@ export int cmd_emit_build_database(const mcpplibs::cmdline::ParsedArgs& parsed) return 1; } take_closing_notes(); - const auto text = mcpp::wire::to_json(mcpp::wire::Envelope{ + const auto text = mcpp::wire::text(mcpp::wire::to_json(mcpp::wire::Envelope{ .kind = "mcpp.build-database", .effects = {Effect::ReadProject}, .data = nullptr, .diagnostics = diagnostics, - }).dump(2) + "\n"; + }), 2) + "\n"; (void)publish(text); return 1; }; @@ -1065,7 +1065,7 @@ export int cmd_emit_build_database(const mcpplibs::cmdline::ParsedArgs& parsed) nlohmann::json specJson{{"name", spec}}; if (spec == "s1") specJson["version"] = std::string(mcpp::build::database::kProfileVersion); - if (const auto rc = publish(mcpp::wire::to_json(mcpp::wire::Envelope{ + if (const auto rc = publish(mcpp::wire::text(mcpp::wire::to_json(mcpp::wire::Envelope{ .kind = "mcpp.build-database", .effects = std::move(effects), .data = nlohmann::json{ @@ -1075,7 +1075,7 @@ export int cmd_emit_build_database(const mcpplibs::cmdline::ParsedArgs& parsed) {"inputs-fingerprint", std::move(rendered.inputsFingerprint)}, }, .diagnostics = diagnostics, - }).dump(2) + "\n"); rc != 0) return rc; + }), 2) + "\n"); rc != 0) return rc; return hasError ? 1 : 0; } diff --git a/src/pack/distribution.cppm b/src/pack/distribution.cppm index d1b06fede..cdd0045fb 100644 --- a/src/pack/distribution.cppm +++ b/src/pack/distribution.cppm @@ -88,13 +88,15 @@ std::string posix_launcher(const Launch& l); std::string exe_launcher_manifest(std::string_view name, std::string_view version); std::string exe_launcher_main(const Launch& l); -// The OS library both launchers build on: the release this mcpp builds on -// (mcpp.toml), so a pack is the same at any later time (design 2026-11-07, -// D20) and the launcher's openkal is the one the platform of mcpp's openkal -// rows names. 0.4.4 named openkal below 0.26, and with openkal-platform 0.31 -// as that platform a launcher could not be built (e2e 988). -// .github/tools/check_version_pins.sh holds the two equal. -inline constexpr std::string_view kLauncherOs = "0.4.6"; +// The OS library both launchers build on, one release so a pack is the same at +// any later time (design 2026-11-07, D20). It is the release the build program +// of the platform of mcpp's openkal rows links: a launcher is built over that +// platform, and the platform's build dependencies are resolved with the +// launcher's own, one version of each package (mcpp#972). openkal-platform +// 0.31's build program links mcpp.plugins 0.26.1, whose mcpplibs.os is 0.4.5; +// 0.4.4 (that of 0.29's) and 0.4.6 were each refused as irreconcilable (e2e +// 988, the canaries' pack cells). The canaries and e2e 988 hold the two equal. +inline constexpr std::string_view kLauncherOs = "0.4.5"; // ─── mcpp-package.json (SPEC-014 §4.5) ──────────────────────────────────── diff --git a/tests/e2e/windows/msvc.cpp b/tests/e2e/windows/msvc.cpp index b1b32ab87..7a4e9649a 100644 --- a/tests/e2e/windows/msvc.cpp +++ b/tests/e2e/windows/msvc.cpp @@ -492,8 +492,8 @@ int main() { f(); } expect(fatal(e2e::succeeded(r))) << "the project did not build: " << s::text(r); const auto graph = s::read_file(ninja_of(p.dir())); expect(fatal(e2e::contains(graph, "/sourceDependencies"))) << "cl.exe's compile writes no record"; - expect(fatal(!e2e::contains(graph, "/showIncludes") && !e2e::contains(graph, "deps = msvc"))) - << "a compile still reads the /showIncludes text"; + expect(fatal(!e2e::contains(graph, "/showIncludes") && e2e::contains(graph, "msvc_deps_prefix = mcpp.inputs:"))) + << "a compile reads the /showIncludes text, or not the lines the executor gives from the record"; // The header the compile read is its input: an edit rebuilds, and fails where it should. std::this_thread::sleep_for(std::chrono::seconds(1)); p.write("inc dir/hdr.h", "#pragma once\nstatic_assert(false, \"edited\");\ninline int value() { return 0; }\n"); diff --git a/tests/unit/test_build_flags.cpp b/tests/unit/test_build_flags.cpp index c6df10ebe..f1ddd4a9f 100644 --- a/tests/unit/test_build_flags.cpp +++ b/tests/unit/test_build_flags.cpp @@ -353,9 +353,11 @@ UTest every_place_a_word_carries_a_path_is_a_link_input{"LinkWordInputs/EveryPla const auto a = dir.path / "a.a", m = dir.path / "v.map", js = dir.path / "lib.js", rsp = dir.path / "libs.rsp"; touch(a); touch(m); touch(js); + // The GNU drivers read a backslash as taking the next character, so a + // response file for them names paths with `/` (#261). { std::ofstream out(rsp, std::ios::binary); - out << a.string() << "\n"; + out << a.generic_string() << "\n"; expect(fatal(bool(out.flush()))) << "the response file was not written: " << rsp; } const std::set none; @@ -373,6 +375,17 @@ UTest every_place_a_word_carries_a_path_is_a_link_input{"LinkWordInputs/EveryPla expect(link_word_inputs("-Wl,-force_load," + a.string(), false, none) == std::vector{a}); const auto viaResponseFile = link_word_inputs("@" + rsp.string(), false, none); expect(viaResponseFile == std::vector{rsp, a}) << "@" << rsp.string() << " gave:" << named(viaResponseFile); + // link.exe reads its response file by the rules of CommandLineToArgvW: a + // native path, quoted where it has a space, is one word as written. + const auto spaced = dir.path / "a b.lib", msvcRsp = dir.path / "link.rsp"; + touch(spaced); + { + std::ofstream out(msvcRsp, std::ios::binary); + out << a.string() << "\r\n\"" << spaced.string() << "\"\r\n"; + expect(fatal(bool(out.flush()))) << "the response file was not written: " << msvcRsp; + } + const auto viaMsvc = link_word_inputs("@" + msvcRsp.string(), true, none); + expect(viaMsvc == std::vector{msvcRsp, a, spaced}) << "@" << msvcRsp.string() << " gave:" << named(viaMsvc); }}; UTest a_directory_or_a_relative_path_is_no_link_input{"LinkWordInputs/ADirectoryOrARelativePathIsNone", [] { diff --git a/tests/unit/test_command_words.cpp b/tests/unit/test_command_words.cpp index 8709c6601..46a455a47 100644 --- a/tests/unit/test_command_words.cpp +++ b/tests/unit/test_command_words.cpp @@ -11,6 +11,7 @@ using namespace mcpplibs::mtest; using words = std::vector; using mcpp::build::command_line::response_file_words; +using mcpp::build::command_line::windows_response_file_words; using mcpp::build::command_line::shell_words; UTest a_shell_reads_quotes_and_backslashes{"CommandWords/AShellReadsQuotesAndBackslashes", [] { @@ -28,6 +29,19 @@ UTest a_response_file_takes_the_character_after_every_backslash{"CommandWords/AR expect(eq(response_file_words("a\\ b \"c\\\\d\""), (words{"a b", "c\\d"}))); }}; +// link.exe's rules (CommandLineToArgvW): a backslash is a character unless +// backslashes precede a double quote, and a native path is a word as written. +UTest a_windows_response_file_keeps_its_backslashes{"CommandWords/AWindowsResponseFileKeepsItsBackslashes", [] { + expect(eq(windows_response_file_words("C:\\lib\\a.lib\r\n\"C:\\x y\\b.lib\"\r\n/OUT:c.exe"), + (words{"C:\\lib\\a.lib", "C:\\x y\\b.lib", "/OUT:c.exe"}))); + // 2n backslashes before a quote are n and the quote opens or closes; 2n+1 + // are n and a quote; two quotes inside quotes are one. + expect(eq(windows_response_file_words("a\\\\\"b c\" d\\\"e \"f\"\"g\""), + (words{"a\\b c", "d\"e", "f\"g"}))); + expect(windows_response_file_words(" \t\r\n").empty()); + expect(eq(windows_response_file_words("\"\""), (words{""}))) << "an empty quoted word is a word"; +}}; + UTest a_word_quoted_for_a_shell_reads_back_as_itself{"CommandWords/AWordQuotedForAShellReadsBackAsItself", [] { for (std::string w : {"plain", "a b", "it's", "a\\b", "$x", "\"q\"", ""}) expect(eq(shell_words(mcpp::build::command_line::quote_posix(w)), words{w})) << w; diff --git a/tests/unit/test_depfile.cpp b/tests/unit/test_depfile.cpp index 4be5f8bd0..aacb91329 100644 --- a/tests/unit/test_depfile.cpp +++ b/tests/unit/test_depfile.cpp @@ -55,12 +55,13 @@ UTest windows_paths_and_crlf_lines_stay_in_the_record{"Depfile/WindowsPathsAndCr // // The record is the one cl.exe 19.44 wrote on windows-2022 (probe mcpp#971): // UTF-8, every path in lower case, the toolchain's and the SDK's headers -// listed with the project's. The executor writes the Make rule ninja reads. +// listed with the project's. The executor gives ninja the paths as lines of +// its output (`deps = msvc` with mcpp's prefix). namespace { using mcpp::build::depfile::SourceRecord; -using mcpp::build::depfile::rule_from_source_dependencies; +using mcpp::build::depfile::inputs_from_source_dependencies; constexpr std::string_view kRecord = R"({ "Version": "1.2", @@ -72,23 +73,26 @@ constexpr std::string_view kRecord = R"({ "c:\\program files\\microsoft visual studio\\2022\\enterprise\\vc\\tools\\msvc\\14.44.35207\\include\\vector", "c:\\program files (x86)\\windows kits\\10\\include\\10.0.26100.0\\ucrt\\stdio.h", "c:\\p970\\build\\gen\\config.h", - "c:\\p970\\build\\gen\\with#hash$.h" + "c:\\p970\\build\\gen\\with#hash$.h", + "c:\\users\\o'neil & co\\a;b^c`d.h" ], "ImportedModules": [], "ImportedHeaderUnits": [] } })"; -// The file system's spelling of a name below the build directory, by a table. +// The file system's spelling of a name, by a table: the directories of the +// build, and a name whose letters beyond ASCII differ only in case. std::string spelled(const std::string& dir, const std::string& name) { if (name == "gen") return "Gen"; if (name == "config.h") return "Config.h"; + if (name == "\xD0\xB4\xD0\xBC") return "\xD0\x94\xD0\x9C"; // дм as Дм (void)dir; return name; } SourceRecord record_of(std::string_view text) { - return {.record = text, .target = "obj/b.obj", .buildDir = "C:/p970/Build", + return {.record = text, .buildDir = "C:/p970/Build", .toolchainDirs = {"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Tools\\MSVC\\14.44.35207\\include\\", "C:/Program Files (x86)/Windows Kits/10/include/10.0.26100.0/ucrt"}, .foldCase = true, .spelled = spelled}; @@ -96,13 +100,17 @@ SourceRecord record_of(std::string_view text) { } // namespace -UTest the_record_becomes_the_rule_without_the_toolchains_headers{"Depfile/TheRecordBecomesTheRuleWithoutTheToolchainsHeaders", [] { - const auto rule = rule_from_source_dependencies(record_of(kRecord)); - expect(fatal(rule.has_value())) << (rule ? std::string() : rule.error()); - expect(eq(*rule, std::string("obj/b.obj: \\\n" - " c:/p970/\xE4\xB8\xAD\xE6\x96\x87\xC3\xA9/inc\\ dir/h\xC3\xA9" "ader.h \\\n" - " C:/p970/Build/Gen/Config.h \\\n" - " C:/p970/Build/Gen/with\\#hash$$.h\n"))); +// Every path the record names but the toolchain's, as a word of its own: a +// space, `#`, `$`, `'`, `&`, `;`, `^` and a backquote included, which ninja's +// reader of a Make rule could not carry (it ends a name at the last five). +UTest the_record_names_its_files_without_the_toolchains_headers{"Depfile/TheRecordNamesItsFilesWithoutTheToolchainsHeaders", [] { + const auto inputs = inputs_from_source_dependencies(record_of(kRecord)); + expect(fatal(inputs.has_value())) << (inputs ? std::string() : inputs.error()); + expect(eq(*inputs, std::vector{ + "c:/p970/\xE4\xB8\xAD\xE6\x96\x87\xC3\xA9/inc dir/h\xC3\xA9" "ader.h", + "C:/p970/Build/Gen/Config.h", + "C:/p970/Build/Gen/with#hash$.h", + "c:/users/o'neil & co/a;b^c`d.h"})); }}; // A header the build generated is another edge's output: its node is the build @@ -110,16 +118,27 @@ UTest the_record_becomes_the_rule_without_the_toolchains_headers{"Depfile/TheRec UTest a_header_under_the_build_directory_keeps_its_spelling{"Depfile/AHeaderUnderTheBuildDirectoryKeepsItsSpelling", [] { auto in = record_of(kRecord); in.foldCase = false; // a host whose names keep their case: the build directory is not matched - const auto rule = rule_from_source_dependencies(in); - expect(fatal(rule.has_value())); - expect(ne(rule->find(" c:/p970/build/gen/config.h"), std::string::npos)) << *rule; + const auto inputs = inputs_from_source_dependencies(in); + expect(fatal(inputs.has_value())); + expect(std::ranges::contains(*inputs, std::string("c:/p970/build/gen/config.h"))); +}}; + +// A build directory whose name differs from the record's beyond ASCII in case +// alone (the record spells every path in lower case) is found by the names as +// the file system spells them. +UTest a_build_directory_with_letters_beyond_ascii_is_found{"Depfile/ABuildDirectoryWithLettersBeyondAsciiIsFound", [] { + auto in = record_of(R"({"Data": {"Includes": ["c:\\\u0434\u043c\\build\\gen\\config.h"]}})"); + in.buildDir = "C:/\xD0\x94\xD0\x9C/Build"; + const auto inputs = inputs_from_source_dependencies(in); + expect(fatal(inputs.has_value())) << (inputs ? std::string() : inputs.error()); + expect(eq(*inputs, std::vector{"C:/\xD0\x94\xD0\x9C/Build/Gen/Config.h"})); }}; UTest a_record_that_is_not_one_is_refused{"Depfile/ARecordThatIsNotOneIsRefused", [] { - expect(!rule_from_source_dependencies(record_of("not json")).has_value()); - expect(!rule_from_source_dependencies(record_of(R"({"Version": "1.2"})")).has_value()); - expect(!rule_from_source_dependencies(record_of(R"({"Data": {"Includes": [1]}})")).has_value()); - const auto none = rule_from_source_dependencies(record_of(R"({"Data": {"Includes": []}})")); + expect(!inputs_from_source_dependencies(record_of("not json")).has_value()); + expect(!inputs_from_source_dependencies(record_of(R"({"Version": "1.2"})")).has_value()); + expect(!inputs_from_source_dependencies(record_of(R"({"Data": {"Includes": [1]}})")).has_value()); + const auto none = inputs_from_source_dependencies(record_of(R"({"Data": {"Includes": []}})")); expect(fatal(none.has_value())); - expect(eq(*none, std::string("obj/b.obj:\n"))) << "a unit that includes nothing has a rule of its own"; + expect(none->empty()) << "a unit that includes nothing names nothing"; }}; diff --git a/tests/unit/test_ninja_backend.cpp b/tests/unit/test_ninja_backend.cpp index 097407e6c..f1993a13c 100644 --- a/tests/unit/test_ninja_backend.cpp +++ b/tests/unit/test_ninja_backend.cpp @@ -142,8 +142,8 @@ UTest bitcode_export_tools_come_from_the_selected_llvm_and_are_quoted_as_words{" }}; // mcpp#970 (design 2026-11-19 §20): cl.exe's compile writes its -// /sourceDependencies record, which the executor writes as the Make rule -// every compile edge reads; no compile reads the /showIncludes text. The +// /sourceDependencies record, which the executor gives ninja as lines with +// mcpp's prefix (`deps = msvc`); no compile reads the /showIncludes text. The // executor, link-output and the archive edge read the text of the cl.exe // toolchain's programs in the console's code page. A GNU toolchain's commands // carry neither word. @@ -170,9 +170,9 @@ UTest msvc_compiles_read_the_record_and_its_tools_text_is_decoded{"NinjaBackend/ const auto object = rule("cxx_object"); expect(ne(object.find("/sourceDependencies $out.json"), std::string::npos)) << object; expect(ne(object.find("--source-dependencies $out.json --text console -- "), std::string::npos)) << object; - expect(ne(object.find("deps = gcc\n depfile = $out.d"), std::string::npos)) << object; + expect(ne(object.find("deps = msvc\n msvc_deps_prefix = mcpp.inputs:\n"), std::string::npos)) << object; + expect(eq(object.find("depfile = $out.d"), std::string::npos)) << object; expect(eq(text.find("/showIncludes"), std::string::npos)); - expect(eq(text.find("deps = msvc"), std::string::npos)); const auto archive = rule("cxx_archive"); expect(ne(archive.find("$mcpp link-output --output $out --text console -- $ar"), std::string::npos)) << archive; expect(ne(rule("cxx_link").find(" --text console -- $ld"), std::string::npos)) << rule("cxx_link"); @@ -2298,7 +2298,18 @@ UTest names_the_consumer_directory_for_each_driver_spelling{"ConsumerIncludeScop "dep.cpp(1): fatal error C1083: \xE6\x97\xA0\xE6\xB3\x95\xE6\x89\x93\xE5\xBC\x80" "\xE5\x8C\x85\xE6\x8B\xAC\xE6\x96\x87\xE4\xBB\xB6: \xE2\x80\x9C" "cfg/nested.h\xE2\x80\x9D: " "No such file or directory\n"; - for (auto const& out : {gcc, clang, cl, clChinese}) { + // The shapes the words may take in other languages: a space before the + // colon, none after it, and the name after a sentence with one colon. + const std::string clSpaced = + "dep.cpp(1): fatal error C1083: impossible d'ouvrir le fichier include : 'cfg7.h' : " + "No such file or directory\n"; + const std::string clTight = + "dep.cpp(1): fatal error C1083: include \xE3\x83\x95\xE3\x82\xA1\xE3\x82\xA4\xE3\x83\xAB\xE3\x80\x82" + "'cfg7.h':No such file or directory\n"; + const std::string clSentence = + "dep.cpp(1): fatal error C1083: \xED\x8F\xAC\xED\x95\xA8 \xED\x8C\x8C\xEC\x9D\xBC. 'cfg/nested.h': " + "No such file or directory\n"; + for (auto const& out : {gcc, clang, cl, clChinese, clSpaced, clTight, clSentence}) { auto advice = mcpp::build::consumer_include_scope_advice(out, dirs); expect(fatal(!advice.empty())) << out; expect(ne(advice.find((root / "appinc").string()), std::string::npos)) << advice; From 400ec91873371336f3bfe1d9c5911d10eeaa0662 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 23:38:33 +0900 Subject: [PATCH 19/30] The Windows bare-metal llvm rows build; 663 reads the std record its plan names llvm 23.1.3 revision 4 carries ld.lld.exe in the Windows archive (design 2026-11-19 section 11.2), so i686-none-elf and x86_64-none-elf with llvm build on windows-x86_64 where the table stated lld-required-absent (target matrix scan of run 44531373098). e2e 663 took the newest std-module.json that mentions libcxx from the home the shard's programs share; an x86_64-openkal record written beside it (openkal-llvm-runtime's llvm/libcxx/modules) was read in two runs of the Linux shard. It now reads the record build.ninja names, as its mtest port in toolchain::layers does. --- .../663_a_graph_libcxx_over_the_payloads_c_library.sh | 11 +++++++++-- tests/matrix/expected.tsv | 8 ++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/e2e/663_a_graph_libcxx_over_the_payloads_c_library.sh b/tests/e2e/663_a_graph_libcxx_over_the_payloads_c_library.sh index 4984df0c8..f536dff29 100755 --- a/tests/e2e/663_a_graph_libcxx_over_the_payloads_c_library.sh +++ b/tests/e2e/663_a_graph_libcxx_over_the_payloads_c_library.sh @@ -74,8 +74,15 @@ grep -q -- 'xim-x-llvm[^ ]*libc++\(abi\)\?\.a' "$ninja" \ # names the glibc payload. Before this assertion the command carried only # `--no-default-config -nostdinc++` and the runner's /usr/include stood in, # a host dependency no report showed. -stdjson=$(grep -l 'llvm.libcxx\|libcxx' "$MCPP_HOME"/build-cache/v1/std/*/std-module.json 2>/dev/null | xargs -r ls -t | head -1) -[ -n "$stdjson" ] || fail "no std-module.json records the package's module under $MCPP_HOME/build-cache/v1/std" +# The record is the one this build's plan names. The cache is the home's, which +# the programs running beside this one share: the newest record that mentions +# libcxx was an x86_64-openkal one (openkal-llvm-runtime's llvm/libcxx/modules) +# in a shard of mcpp#966's Linux e2e, the same race toolchain::layers 663 reads +# around. +stdkey=$(grep -oE 'build-cache/v1/std/[0-9a-f]+' "$ninja" | head -1) +stdjson="$MCPP_HOME/$stdkey/std-module.json" +[ -n "$stdkey" ] && [ -f "$stdjson" ] \ + || fail "no std-module.json records the package's module under $MCPP_HOME/build-cache/v1/std" "$ninja" grep -q 'xim-x-glibc' "$stdjson" \ || fail "the package std module's command does not name the payload's glibc" "$stdjson" bin=$(ls target/*/*/bin/app | head -1) diff --git a/tests/matrix/expected.tsv b/tests/matrix/expected.tsv index be23e56c0..da42d1306 100644 --- a/tests/matrix/expected.tsv +++ b/tests/matrix/expected.tsv @@ -97,6 +97,10 @@ # 建出;gcc 与 msvc 是 `capability-pin`;graph 体系里图中钉的版本在平台版本组之外, # `platform-version-group`。 # +# 2026.11.19.1 改变的格(设计 2026-11-19 §11.2,一个工具链载荷在每台宿主上读同样的输入):llvm 23.1.3 +# 修订 4 的 Windows 归档在 `bin/` 带 `ld.lld.exe`,`windows-x86_64` 上 `i686-none-elf` 与 `x86_64-none-elf` +# 配 llvm 由 `lld-required-absent` 变为建出(运行 44531373098 实测)。 +# # 中间五列是报告器输出,不参与支持判定。现有报告器把 aarch64-ios 的 # c-abi 输出为 glibc(payload),这不代表 iOS 产物实际链接 glibc。Apple # 目标的实际依赖由目标构建与运行门检查,不能从该报告字段推导。 @@ -423,7 +427,7 @@ payload windows-x86_64 i686-linux-gnu llvm@23.1.3 - - - - - unsupported tier-pla payload windows-x86_64 i686-linux-gnu msvc@system - - - - - unsupported tier-planned payload windows-x86_64 i686-linux-musl llvm@23.1.3 i686-unknown-linux-musl none musl(platform) libc++(platform) openkal-llvm-runtime@0.20.3 ok none payload windows-x86_64 i686-linux-musl msvc@system - - - - - unsupported capability-pin -payload windows-x86_64 i686-none-elf llvm@23.1.3 - - - - - unsupported lld-required-absent +payload windows-x86_64 i686-none-elf llvm@23.1.3 i686-none-elf none - - - ok none payload windows-x86_64 i686-none-elf msvc@system - - - - - unsupported capability-pin payload windows-x86_64 i686-openkal llvm@23.1.3 i686-unknown-none-elf none musl(platform) libc++(platform) openkal-llvm-runtime@0.20.3 ok none payload windows-x86_64 i686-openkal msvc@system - - - - - unsupported capability-pin @@ -457,7 +461,7 @@ payload windows-x86_64 x86_64-linux-musl llvm@23.1.3 - - - - - unsupported conve payload windows-x86_64 x86_64-linux-musl msvc@system - - - - - unsupported convention-unreplaced payload windows-x86_64 x86_64-macos llvm@23.1.3 x86_64-apple-macos14.0 none musl(platform) libc++(platform) openkal-llvm-runtime@0.20.3 ok none payload windows-x86_64 x86_64-macos msvc@system - - - - - unsupported capability-pin -payload windows-x86_64 x86_64-none-elf llvm@23.1.3 - - - - - unsupported lld-required-absent +payload windows-x86_64 x86_64-none-elf llvm@23.1.3 x86_64-none-elf none - - - ok none payload windows-x86_64 x86_64-none-elf msvc@system - - - - - unsupported capability-pin payload windows-x86_64 x86_64-openkal llvm@23.1.3 x86_64-unknown-none-elf none musl(platform) libc++(platform) openkal-llvm-runtime@0.20.3 ok none payload windows-x86_64 x86_64-openkal msvc@system - - - - - unsupported capability-pin From cd4ce3c90531ca58cf3a0f9eca1df1601eb95baa Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 23:53:31 +0900 Subject: [PATCH 20/30] A path or git dependency's build is cached under the name of its sources (mcpp#974) A dependency's sources have one name, read by the global dependency cache, the host tool store and the build programs' identity alike: an index package's version, a git dependency's resolved commit, and for a path dependency the digest of its tree's content (`+path.`), which leaves out target/, .git/, .mcpp/, compile_commands.json and nested packages. The tool store's modification-time stamp is gone. The project's own trees -- the root, the selected members, any package below the build's directory or below the directory of a build above it (own_roots, handed to build-program and host-tool sub-builds) -- are built in place. Every other path and git dependency enters the global cache, so two projects over one framework and a CI run that restores the build cache share one build. A path tree is read again before its entry is written; one that changed while it compiled is not stored and is reported as a degradation; the entry the same tree left under its previous digest is removed. A cache hit is reported under the package's qualified name, so it reads `Cached`. SPEC-001 section 10.4, usage chapters 04 and 30 in both languages, design record section 22, e2e bmi/19 rewritten for the new behaviour. --- ...11-19-ecosystem-rule-convergence-design.md | 194 +++++++++++++++++- CHANGELOG.md | 13 ++ docs/04-mcpp-toml.md | 32 ++- docs/09-commands-by-scenario.md | 7 +- docs/30-build-mcpp.md | 20 +- docs/specs/package-identity.md | 19 +- docs/zh/04-mcpp-toml.md | 22 +- docs/zh/09-commands-by-scenario.md | 4 +- docs/zh/30-build-mcpp.md | 12 +- modules/buildmcpp/src/tool_store.cppm | 63 +----- src/build/prepare.cppm | 9 + src/build/prepare/features.cpp | 38 ++-- src/build/prepare/host_graph.cpp | 38 +++- src/build/prepare/plan.cpp | 106 ++++++++-- src/build/prepare/state.cppm | 16 ++ src/build/source_digest.cppm | 23 ++- tests/e2e/modules/bmi.cpp | 64 +++--- tests/e2e/modules/programs.cpp | 5 +- tests/unit/test_source_digest.cpp | 21 +- tests/unit/test_tool_store.cpp | 91 +------- 20 files changed, 538 insertions(+), 259 deletions(-) diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md index 63b69b17d..70d305a39 100644 --- a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md @@ -5,9 +5,11 @@ status: active # 生态规则收敛方案:2026.11.18.1 之后十个 issue 的五类成因、七项决定与实施范围 -**日期:** 2026-11-19 · **版本:** v5(实施中。用户决定:全部在本轮实施,每个仓库一个拉取请求,提交作为检查点, -从成因解决而不留变通;N1 至 N3 按建议;mcpp#970 并入本轮,随 2026.11.19.1 发布。第 15 节为七项决定的依据,第 16 节为两轮自评, -第 17 节为已定事项,第 19 节为实施的状态与测量,第 20 节为 #970 的方案) · **基线:** mcpp 2026.11.18.1(`3b886163`)、openkal 0.25.3、openkal-linux 0.26.3(`363ccbb`)、 +**日期:** 2026-11-19 · **版本:** v7(实施中。用户决定:全部在本轮实施,每个仓库一个拉取请求,提交作为检查点, +从成因解决而不留变通;N1 至 N3 按建议;mcpp#970 并入本轮,随 2026.11.19.1 发布;mcpp#972 中的构建程序的解析范围、平台系列、 +#965、#967 并入本轮,映像启动映像并入 openkal-image 0.7.1;mcpp#974(路径依赖的缓存)作为缺陷与通用机制并入本轮。 +第 15 节为七项决定的依据,第 16 节为两轮自评,第 17 节为已定事项,第 19 节为实施的状态与测量,第 20 节为 #970 的方案, +第 21 节为 #972 的方案,第 22 节为 #974 的方案) · **基线:** mcpp 2026.11.18.1(`3b886163`)、openkal 0.25.3、openkal-linux 0.26.3(`363ccbb`)、 openkal-macos 0.23.5(`9dbc36f`,索引中为 0.23.4)、openkal-platform 0.29.2(引擎默认)与 0.30.1、mcpplibs.os 0.4.4、 mcpp.plugins 0.26.0、mcpp-index `3ee5f08`、xlings `4f5df092` · **前文:** [第二部分之后的补充方案](2026-11-17-ninja-cascade-publish-supplement-design.md)、 @@ -977,6 +979,188 @@ windows-2022,探针 mcpp#971(运行 44508574496),以 `CreateProcessW` - **e2e(Windows,MSVC,970)。** 一个非 ASCII 目录中、头文件目录带空格的工程:编译规则带记录而不读 `/showIncludes`; 头文件改为 `static_assert(false)` 后构建失败,改回后成功;在一个代码页为 936 的新控制台(`start /wait`)中构建失败时, mcpp 不以内部错误结束,终端与 `result.json` 中的诊断含正确的汉字。 -- **规则的形状(单元)。** cl.exe 的编译规则带 `/sourceDependencies $out.json`、执行器的两个词与 `deps = gcc`,链接与归档经 - `link-output --text console`;GNU 工具链的命令不带这两个词。 +- **规则的形状(单元)。** cl.exe 的编译规则带 `/sourceDependencies $out.json`、执行器的两个词、`deps = msvc` 与 + `msvc_deps_prefix = mcpp.inputs:`,链接与归档经 `link-output --text console`;GNU 工具链的命令不带这两个词。 - **语言包。** CI 不安装(6.4 分钟);20.2 的测量由单元测试的样本覆盖。 + +## 21. 构建程序的解析范围、平台系列与两项引擎缺陷(#972) + +**来源。** mcpp#972 与其补充评论(cmd.exe 解释器在 29 个目标上的验证)。按"排除不通用、不合规范的部分,从架构与成因解决" +分类后,用户决定:#965、#967、构建程序的解析范围(A)与平台系列的扩展随 2026.11.19.1 发布;映像启动映像(openkal-image#42) +并入 openkal-image 0.7.1。#956、#957 已在本轮修复(第 7 节);无进程的系统(Web、UEFI、DOS)与 DJGPP 的 `fork` 不是缺陷; +`x86_64-windows-musl` 上的 "not recognized" 经分诊属于消费者:该行 `presents = "posix"`,消费者的 POSIX 分支以 `stat` 的属主 +执行位判断能否执行,Windows 不记录这一属性(openkal fs.h),而 openkal-musl 的 `access(X_OK)` 与 `posix_spawnp`(`;` 与 +`PATHEXT`)给出正确答案;盘符被正确识别。openkal-macos#53(PureDarwin 的卷不在声明执行属性的格式名单中)留在该仓库。 + +### 21.1 现状 + +- **一张解析表。** `ResolvedKey` 为命名空间、名字与变体;目标图、每个构建程序的 `[build-dependencies]` 闭包与宿主的读法都写入 + 同一张表,一个包一个版本(`graph.cpp`)。构建依赖不改名(N29),所以两个版本被拒绝("a build loads one version of it")。 +- **R10.2。** 宿主工具链与标准相同的构建程序是一个工作区的成员,由一个子构建规划;`resolution_pins` 按名字把外层的版本告诉 + 子构建,一个名字一个版本。 +- **测得。** 打包的 launcher(目标侧依赖 mcpplibs.os)与平台 0.31 的插件(宿主侧,经 mcpp.plugins 0.26.1 依赖 mcpplibs.os 0.4.5) + 共用一张表:launcher 写 0.4.4 与 0.4.6 都被拒绝,只能写平台构建程序的 0.4.5(e2e 988)。#972 的消费者的 `build.mcpp` 用 + mcpp.plugins 0.22.1,平台 0.31.0 的插件用 0.26.1,同样被拒绝,消费者只能抄写平台内部的版本。 +- **平台系列。** 以范围选择的平台取范围内最新、引擎下界满足的版本(SPEC-013 §14);版本组不容许目标图的某个请求时,整个 + 构建以 `platform-version-group` 被拒绝。索引的平台列为 `@*`,消费者因此在自己没有改动时停止构建。 +- **#965。** `resolve_clang_driver` 在载荷旁没有 cfg 时于取 base 的 C++ 头文件之前返回;`-nostdinc++` 只在 `bypassCfg` + (要求载荷有 cfg)时写出。Windows 的 LLVM 载荷没有 `clang++.cfg`,所以 base 提供的 libstdc++ 在 Windows 主机上两者都失去 + (mcpp-platforms 的 Hurd、Haiku 两行)。libc++ 的 base 不受影响,因为 clang 的 FreeBSD、OpenBSD 驱动自己找到 + `usr/include/c++/v1`。 +- **#967。** 索引的平台列(`platform_rows`)在本地索引中未命中时,行被拒绝为未知目标;依赖未命中时的刷新(`decide_for_miss`) + 不经过这里。 + +### 21.2 成因 + +1. **解析的范围是一次准备,而 C++ 模块的约束是一个程序。** 一个程序(一次链接)内一个包只能有一个版本;两个程序之间没有这个 + 约束。执行上的共享(R10.2,多个程序使用的库只编译一次)被当成了解析上的约束。 +2. **平台系列的选择只看引擎下界,不看本次构建能否使用该版本。** +3. **一个安装的事实决定了一个目标的事实。** 载荷旁有没有 cfg 文件决定了谁提供 C++ 头文件。 +4. **索引的一种查询没有走索引未命中的统一策略。** + +### 21.3 方案 + +**A1 解析的范围是程序。** +- 目标侧:命令选择的成员、它们经 `[dependencies]` 到达的包与平台加入的包是一个范围;版本组只约束它(SPEC-013 §7.5 不变)。 +- 宿主侧:每个构建程序(一个包的 `build.mcpp` 或其规则合成的程序、平台的 `[platform] plugin`)以该包的 `[build-dependencies]` + 及其闭包为一个范围,以该包的身份命名(`build:<包>@<版本或来源>`);一个构建依赖自己的 `[build-dependencies]` 是它自己的 + 构建程序的范围。 +- 请求在其范围内合并;不同范围的同一个包各有记录,互不约束。范围内的合并规则、变体与改名的边界不变;根的 + `override = true` 选择整个构建的版本,对所有范围生效。 +- **共享。** 宿主范围的请求按约束取版本时,若另一个范围已取得满足该约束的版本,取其中最新者,否则取索引中最新者。没有冲突的 + 构建因此与此前取得相同的版本,构建程序的身份不变,升级不引起重编。 + +**A2 执行:构建程序图按版本划分。** R10.2 的图仍按宿主工具链与标准分组;组内的程序按闭包中的版本划分:一个程序加入第一个 +在共有的每个包上版本相同的划分,否则另起一个。第一个划分在原目录,其余在 `-<摘要>`(摘要取该划分与第一个划分不同的 +包与版本)。每个划分一个子构建,`resolution_pins` 取该划分的版本;划分内版本相同的包只编译一次,划分之间由全局依赖缓存按身份 +共享。R10.3 的程序身份本来覆盖闭包中每个包的版本。 + +**A3 锁。** 一个身份一个条目;同一身份在另一个范围取得不同的版本时,该版本另成一个条目,名字带 `+b<范围摘要>`,`program` +写出该范围。合并与 `--locked` 按身份比较,不变。 + +**A4 诊断。** 冲突只在一个范围内发生:消息写出该构建程序(`the build program of <包>`)与每个请求者。 + +**A5 平台系列取可用的最新版本(SPEC-013 §14 第 2 条)。** 以系列选择、锁中没有记录的平台,解析因版本组被拒绝时,选择移到 +系列中下一个更早、引擎下界满足的版本并重新准备;跳过的版本连同拒绝它的请求写入报告,与跳过引擎下界不满足的版本同一形式。 +全部不容许时拒绝,消息列出每个版本与拒绝它的请求。精确版本与锁中记录的版本按写出的取。 + +**A6 launcher 回到 mcpp 自己的 mcpplibs.os。** A1 之后 launcher 的目标侧依赖与平台插件的宿主侧依赖互不约束; +`kLauncherOs` 与 mcpp 自己的版本相同,由 `check_version_pins.sh` 检查。 + +**B1 #965:C++ 层的词由提供者决定。** `resolve_clang_driver` 先取 base 的 C++ 头文件,再看载荷有没有 cfg;`-nostdinc++` 与该层的 +目录在 C++ 层不由载荷提供时写出,与宿主无关;`--no-default-config` 仍由 cfg 的存在决定。 + +**B2 #967:平台列的未命中走索引未命中的策略。** 内置表与本地索引都没有答案的行,由 `decide_for_miss` 决定刷新(`--offline`、 +`auto_refresh`、去抖、每次运行一次),刷新后重新扫描;仍未命中时拒绝,消息写出是否刷新及原因。扫描的缓存在刷新后作废。 + +**C1 映像启动映像(openkal-image 0.7.1)。** okrun 绑定 `kal_process_spawn`:映像文件或 okb 经 runner 指定的加载器启动, +`okrun [--require-signature] [--trust ]... --argv0= -- <映像> <其余参数>`;启动前按加载器的检查拒绝不能发生的 +启动(`kal_err_not_program`、`kal_err_permission`),句柄是加载器的、状态是映像的;其他文件按原样启动;不指定加载器的程序 +(携带映像的程序)拒绝映像文件;映像的名字(`kal_env_image`)为解析后的绝对名。不采纳:binfmt_misc(只有 Linux,需要 root)、 +映像自启动(ELF 的魔数须在开头,映像只导入 `kal_*`)、在 openkal 实现或 openkal-musl 中识别映像(它们不认识上层的加载器)。 + +### 21.4 多视角复核 + +- **架构。** 解析的范围与 C++ 的约束一致;执行上的共享由身份决定,不约束解析。平台系列与已有的跳过同一形式;#965 把安装的 + 事实与目标的事实分开;#967 让索引的每种查询走一条策略。 +- **稳定性。** 平台发布新版本或改变其内部构建工具的版本,不使消费者停止构建;冲突只在一个程序内报告。 +- **简洁。** 不新增键;消费者与 launcher 都不再抄写平台内部的版本。 +- **用户体验。** 跳过的平台版本与原因写在报告中;冲突的消息点名程序。 +- **兼容与无感升级。** 没有冲突的构建版本不变、程序身份不变;只有此前被拒绝的构建得到新结果;锁只在两个版本并存时多出条目。 +- **跨平台。** 与系统无关;#965 在每个宿主上相同。 +- **测试覆盖。** 见 21.6。 + +### 21.5 不采纳的做法 + +- **构建依赖改名(N29)。** 构建程序以作者写的名字导入模块。 +- **只分开目标侧与宿主侧,宿主侧仍一张表。** #972 的冲突在两个构建程序之间,不能解决。 +- **宿主侧完全不共享。** 没有冲突的构建版本会改变,程序身份随之改变,升级后重编。 +- **冲突时才给后来的程序另一条记录。** 一个共享包的依赖与该程序自己的请求可能取到两个版本,一个程序内出现两个版本。 +- **以锁或新键固定平台系列。** 锁与 `[target.<选择器>] platform` 已经存在;问题是最新不等于可用。 +- **B(范围代替精确版本)。** 0.x 没有兼容承诺,范围不能合并。 + +### 21.6 验证 + +- **单元。** 不同范围的同名包各有记录;共享的取版本规则;划分;锁条目的名字与 `program`;无 cfg 时 base 的 libstdc++ 的词; + 平台列未命中时的刷新决定。 +- **e2e。** 工程的 `build.mcpp` 与平台插件用同一个库的两个版本(项目自己的索引),构建成功,两个程序各自编译;launcher 以 + mcpp 的 os 构建(988);平台系列跳过版本组不容许的版本(项目索引中的两个平台版本);平台列在刷新后命中(#967)。 +- **openkal-image。** `examples/starts` 在四台机器上:启动自己(argv[0] 如启动者所给)、`exits.oke` 与一个系统程序;签名要求下 + 未签名的映像被拒绝为 `kal_err_permission`;不指定加载器时被拒绝为 `kal_err_not_program`。 + +## 22. 源码的名字与路径依赖的缓存(#974) + +**来源。** mcpp#974:经 `path` 到达的库从不进入全局构建缓存,依赖它的每个工程、每次干净构建都重新编译它;HuxerUI 的一致性 CI +中八个工程各自编译整个框架,每个约三分钟,调试与发布构建各一次,恢复 `~/.mcpp/build-cache` 也无济于事。用户决定:作为缺陷与 +通用机制,从根与框架层面解决,修复并入 #966。 + +### 22.1 现状 + +- **依赖缓存。** 全局依赖缓存(`build-cache/v1/pkg`)只收索引包:`path`、`git` 依赖与工作区成员在任何深度都不缓存,依赖 + 它们的包也不缓存(`localTaint`),理由是这些源码在版本不变时可以改变,以 `name@version` 为键察觉不到。 +- **工具 store。** 宿主工具已按来源命名:`git` 包按解析的提交,`path` 包按树的印记(`tool_store::tree_stamp`:每个文件的 + 相对路径、大小与修改时间),新条目写出时删除同一棵树的旧条目(mcpp#874)。 +- **构建程序。** 程序的身份(R10.3)按树内容的摘要(`source_tree_digest`,以 stat 行缓存,mcpp#797)。 +- 同一件事(一个包的源码是什么)因此有三种答法:依赖缓存没有答案,工具 store 用修改时间,构建程序用内容。 + +### 22.2 成因 + +1. **源码的名字没有一个定义。** 索引包的版本命名源码;`git` 与 `path` 包的版本不命名源码,依赖缓存于是整个放弃这些包,而没有 + 换一个能命名源码的名字。工具 store 与构建程序各自补了一个名字,依赖缓存没有。 +2. **"可缓存"按来源的种类判断,而应按"是不是正在编辑的树"判断。** 不缓存的真正理由只对工程自己的树成立:它每次编辑都是新键, + 进入缓存只增加写入与回收;工程之外的 `path` 依赖(框架、共享库)恰是被多个工程复用的部分。 +3. **修改时间不是内容。** 检出、复制与 CI 的恢复改变修改时间而不改变内容,印记因而不命中;修改时间粒度之内的编辑又可能不被 + 看见。 + +### 22.3 方案 + +**E1 一个源码的名字。** 索引包为 `name@version`;`git` 包为 `+git.<解析的提交>`;`path` 包为 `+path.<树内容摘要的前 16 位>`。 +摘要覆盖依赖根下每个普通文件的相对路径与内容,不计 `target/`、`.git/`、`.mcpp/`、`compile_commands.json` 与带自己 +`mcpp.toml` 的子目录(另一个包,以它自己的名字计入)。依赖缓存、工具 store 与构建程序的身份调用同一个函数 +(`source_identity_suffix`、`tree_content_digest`),一次准备中每棵树只读一次;`tree_stamp` 删除。 + +**E2 工程自己的树在原处构建。** 正在编辑的树是:根包、被选择的工作区成员、本次构建的根(`workRoot`)之下的任何包;构建程序 +与宿主工具的子构建继承上层构建的根(`own_roots`),上层工程的树在子构建中同样在原处构建。不以名字识别的包使依赖它的包也不进入 +缓存(`localTaint` 的规则不变,判据换成 `sources_named`)。 + +**E3 摘要以内容为准,读取以 stat 为界。** 沿用 `source_tree_digest` 的 stat 行:大小与修改时间未变的文件沿用记录的内容摘要, +修改时间距记录时刻过近的文件(racily clean)重新读取。stat 行按树的路径保存在 `build-cache/digests/`;条目的键与树的位置无关, +同一份源码在两处检出共用一个条目。 + +**E4 写入前复核。** `path` 依赖的条目在编译完成之后、写入之前重新计算摘要;与键中的不同(编译期间被编辑)时不写入,作为降级 +报告(`build/cache`,写出包与树),本次构建的产物照常使用。 + +**E5 一棵树一个条目。** 条目旁记录构建它的树(`source`);写入新条目时,同一个包、同一棵树、另一个摘要的条目被删除,与工具 +store 的规则(mcpp#874)相同。另一处检出是另一棵树,保留自己的条目,由 `mcpp cache gc` 按最后使用回收。 + +**E6 报告。** 命中的包一行 `Cached <包> v<版本> (<相对路径>, N units)`,与索引包同一形式;报告以包的限定名对应(此前缓存命中的 +记录用依赖键,默认命名空间的 `greet` 与报告的 `mcpplibs.greet` 对不上,命中的包被显示为 `Compiling`)。`mcpp cache list` +显示 `<版本>+path.<摘要>`。 + +### 22.4 多视角复核 + +- **架构。** 一个名字、一个函数、三个使用者;是否缓存由"是不是工程自己的树"决定,不由来源的种类决定。 +- **稳定性。** 内容摘要不受检出与恢复的影响;编辑中的构建不写入错误的条目;同一棵树的旧条目不累积。 +- **简洁。** 删除 `tree_stamp`;不新增键与选项。 +- **用户体验。** 多个工程共用一个框架时框架只编译一次;CI 恢复 `build-cache` 后命中;命中的包在报告中显示为 `Cached`。 +- **兼容与无感升级。** 索引包的键不变;`git` 与 `path` 依赖在升级后的第一次构建写入缓存;工具 store 中以印记命名的旧条目不再 + 命中,按新键重建一次,旧条目由回收删除;`--cache local` 与 `--cache off` 的含义不变。 +- **跨平台。** 摘要以相对路径的 UTF-8 形式(`generic_u8string`)与内容计算,与宿主无关。 +- **测试覆盖。** 见 22.6。 + +### 22.5 不采纳的做法 + +- **编译器启动器(ccache、sccache,#974 的备选)。** 只缓存单个编译,不缓存 BMI 与模块之间的次序;对模块单元的命中依赖编译器 + 对路径的规范化;是引擎之外的第二个缓存,与依赖缓存的身份不一致。 +- **继续用修改时间的印记。** 见成因 3。 +- **工程自己的树也进入缓存。** 每次编辑一个新条目,就地的增量构建本来更快。 +- **以绝对路径命名条目。** 同一份源码的两处检出(两次 CI 运行、两个工程)不能共用。 + +### 22.6 验证 + +- **单元。** 编辑与撤销编辑回到原摘要;`compile_commands.json`、构建输出与嵌套的包不计入;工具 store 的 scratch 目录。 +- **e2e。** `bmi/19`:兄弟目录中的 `path` 依赖进入缓存,干净构建显示 `Cached mylibA v0.1.0 (../mylibA, 1 unit)`;工程目录之下的 + `path` 依赖在原处编译、不进入缓存;编辑后的新条目替换同一棵树的旧条目;传递的 `path` 依赖进入缓存;`cache list` 以 `+path.` + 显示;对传递依赖的编辑到达程序。`programs/857`:工作区之下的宿主模块库不进入缓存。两例在本地通过。 +- **手工(#974 的复现)。** app1、app2 经 `path` 使用 greet:第二个工程显示 `Cached greet v0.1.0 (../greet, 1 unit)`;编辑 greet + 后重建一次并替换条目;把 greet 放入工程目录之下,在原处编译。 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cdc3b454..170d2ea05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,19 @@ ### Changed +- A `path` or `git` dependency's build enters the global cache, named by its sources: a `git` + dependency by its resolved commit, a `path` dependency by the digest of its tree's content + (`+path.` in `mcpp cache list`). Two projects that reach one library through + `path`, a clean build, and a CI run that restores `$MCPP_HOME/build-cache` reuse one entry + while the library's sources are the same, wherever the tree is checked out; before, every + dependent compiled it again (mcpp#974). The project's own trees -- the root, its workspace + members, any package below the project's directory or the directory of the build a build + program or host tool serves -- are built in place. A tree is read again before its entry is + written, and one that changed while it compiled is not stored, which the build reports as a + degradation (`build/cache`); the entry the same tree left under its previous digest is + removed as the new one is written. A host tool from a `path` package and the build + programs' identity name the tree by the same digest of content, where the tool store used + modification times, so a fresh checkout reuses a tool's entry too. - `mcpplibs.openkal-platform@0.31.0` is the default platform of the rows openkal serves (0.29.2 in 2026.11.18.1): the openkal 0.26 family, in which a start is bounded by the environment alone and one refused for its size is `E2BIG`, and an observation of a unit diff --git a/docs/04-mcpp-toml.md b/docs/04-mcpp-toml.md index dae71dc9b..4d02a96cb 100644 --- a/docs/04-mcpp-toml.md +++ b/docs/04-mcpp-toml.md @@ -1884,14 +1884,30 @@ Precedence: `--cache ` **>** `MCPP_BUILD_CACHE` **>** `[build] cache` **>* `global`. An unrecognized value is reported (an error under `--strict`) rather than silently falling back to `global`. -What is **not** cached: `path` and `git` dependencies, at any depth, and -workspace members. Their sources can change without their `name@version` -changing, so no key over that identity could notice the change. - -The libraries a build program imports follow the same rule (2026.10.11.1+). The -engine interface (`mcpp.core`) and index packages are kept in the global cache; -a `path` or `git` package or a workspace member is compiled in the build -programs' graph under `target/.build-mcpp/host/`, never in the global cache. +An entry is named by the sources it was built from *(2026.11.19.1+)*: an index +package by its `name@version`, a `git` dependency by its resolved commit, and a +`path` dependency by the digest of its tree's content, shown as +`+path.` in `mcpp cache list`. The digest covers every file's +relative path and content below the dependency's root, except `target/`, +`.git/`, `.mcpp/`, `compile_commands.json` and a subdirectory with its own +`mcpp.toml`. Two projects that reach one library through `path`, a clean build, +and a CI run that restores `$MCPP_HOME/build-cache` reuse one entry while the +library's sources are the same, wherever its tree is checked out; an edit is a +new key, and the entry built from the same tree under the previous digest is +removed as the new one is written. A `path` dependency's tree is read again +before its entry is written, and a tree that changed while it compiled is not +stored. + +What is **not** cached: the project's own trees, which are what is being edited +-- the root package, its workspace members, and any package below the project's +directory (or below the directory of the build a build program or a host tool is +built for). Before 2026.11.19.1, `path` and `git` dependencies were not cached at +any depth (mcpp#974). + +The libraries a build program imports follow the same rule (2026.10.11.1+): the +engine interface (`mcpp.core`), index packages and named `git` and `path` +dependencies are kept in the global cache, and the project's own packages are +compiled in the build programs' graph under `target/.build-mcpp/host/`. `--cache local` and `--cache off` keep all of them in the project. See [30 — The build programs' graph](30-build-mcpp.md). diff --git a/docs/09-commands-by-scenario.md b/docs/09-commands-by-scenario.md index b58bb4420..d0a26d8e5 100644 --- a/docs/09-commands-by-scenario.md +++ b/docs/09-commands-by-scenario.md @@ -101,9 +101,10 @@ cache gc`): an ok image's prepared copies and snapshots for another boot After a successful build, at most once a day, every entry -- package, host tool or `std` -- that no build has used for `[cache] unused_days` days (config.toml; default 30, `0` turns it off) is removed. A hit marks an entry used. A `path` -host tool's entry is replaced when its tree changes: the entry built from the -same tree under the previous stamp is removed as the new one is written -(2026.10.31.3+). Before a build, mcpp warns when the file system it writes to +host tool's or `path` dependency's entry is replaced when its tree changes: the +entry built from the same tree under the previous digest of its content is +removed as the new one is written (host tools 2026.10.31.3+, dependencies and +the content digest 2026.11.19.1+). Before a build, mcpp warns when the file system it writes to has less free space than the larger of 2 GiB and 1.25 times what the configuration's last build wrote (recorded in `/.mcpp-written`); the warning states both (2026.11.8.1+). `mcpp cache prune` and `mcpp cache gc diff --git a/docs/30-build-mcpp.md b/docs/30-build-mcpp.md index 2e4b230ca..ddf9fc741 100644 --- a/docs/30-build-mcpp.md +++ b/docs/30-build-mcpp.md @@ -1789,16 +1789,20 @@ Four properties worth knowing: published version is immutable, so for a tool that arrives from an index the version alone identifies its sources. A `git` tool is keyed by the commit it resolved to. A `path` tool has no version that moves when its sources do, so -it is keyed by a stamp of its tree: every regular file's relative path, size -and modification time, with `target/`, `.git/`, `.mcpp/` and the compile -database excluded. An edit to the tool's emitter therefore reaches the -consumer on the next build, and so does its reversal, while a tree that did -not change is a store hit and is not rebuilt. Measured on -[`examples/12-a-new-device-language`](../examples/12-a-new-device-language/); +it is keyed by the digest of its tree's content, the identity the dependency +cache uses ([04 §2.10](04-mcpp-toml.md)): every regular file's relative path +and content, with `target/`, `.git/`, `.mcpp/`, the compile database and a +subdirectory with its own `mcpp.toml` excluded *(2026.11.19.1+; before, each +file's path, size and modification time)*. An edit to the tool's emitter +therefore reaches the consumer on the next build, and so does its reversal, +while a tree whose content did not change is a store hit and is not rebuilt, +also after a checkout or a copy that changed every modification time. Measured +on [`examples/12-a-new-device-language`](../examples/12-a-new-device-language/); before this rule the key held the version alone and a change to the emitter left `mcpp run` printing the previous answer until the version was bumped. -Entries accumulate as a tree is edited; `mcpp cache clean` empties the store, -which lives at `/tool//@[+]/`. +The entry built from the same tree under the previous digest is removed when +the new one is written; `mcpp cache clean` empties the store, which lives at +`/tool//@[+]/`. **The paths a tool build writes are bounded (2026.9.15.2+).** The sub-build runs in `/tool/.build//`, a directory named by the entry diff --git a/docs/specs/package-identity.md b/docs/specs/package-identity.md index 722f24b69..753a9034a 100644 --- a/docs/specs/package-identity.md +++ b/docs/specs/package-identity.md @@ -5,7 +5,7 @@ | **规范编号** | SPEC-001 | | **标题** | 包身份(`package.namespace` / `package.name`)、`[dependencies]` 选择器与匹配机制 | | **状态** | **评审中(Review)** —— 已实现 | -| **版本** | 1.9 | +| **版本** | 1.10 | | **最后修改** | 2026-11-19 | | **最低实现版本** | 描述符身份:mcpp **0.0.106**;精确 selector:mcpp **2026.8.10.1**(xlings >= 0.4.69) | | **作者/维护** | mcpp-community | @@ -569,12 +569,29 @@ mcpp 侧的使用: 修订号与索引条目不一致不使载荷不完整(1.6 把它算入完整性,1.7 起移出)。mcpp **禁止**写入载荷, 包括标记文件;一个载荷的路径按提供者的回答使用,**禁止**被规范化(链接条目的路径即其身份)。 +### 10.4 源码的名字(mcpp 2026.11.19.1+,mcpp#974) + +1. 一个依赖的源码由以下之一命名:索引包——`name@version`(载荷在不可变的 store 中,§10.3);`git` 依赖—— + 解析得到的提交;`path` 依赖——其源码树内容的摘要,写作 `+path.<摘要>`。摘要覆盖根目录下每个普通 + 文件的相对路径与内容,不含 `target`、`.git`、`.mcpp`、引擎写在清单旁的 `compile_commands.json`,以及含有 + 自己 `mcpp.toml` 的子目录(另一个包)。 +2. 全局依赖缓存、宿主工具的缓存与构建程序的身份**必须**以同一个名字识别源码;以修改时间代替内容的名字 + **禁止**作为身份:同一份源码在另一处检出(如另一次 CI 运行)时名字相同,一次修改与它的撤销回到同一个名字。 +3. 工程自己的树——根包、被选择的工作区成员、位于本次构建的根或其上层构建(构建程序、宿主工具所服务的构建)的根 + 之下的包——不以名字识别,在原处构建,**禁止**进入全局缓存。 +4. `path` 依赖的条目在写入之前重新计算摘要,与键中的不同时**禁止**写入,并**必须**作为降级报告(`build/cache`, + 写出该包与它的树);同一棵树在另一个摘要下的条目在写入新条目时移除。 +5. 一个不以名字识别的包使依赖它的包也不进入全局缓存。 + +**已实现**(mcpp 2026.11.19.1)。此前全局依赖缓存只收索引包,宿主工具的缓存以修改时间命名 `path` 包的树。 + --- ## 11. 变更记录 | 版本 | 日期 | 变更 | |---|---|---| +| 1.10 | 2026-11-19 | 新增 §10.4:源码的名字——索引包的版本、git 依赖的提交、path 依赖的内容摘要;全局依赖缓存、宿主工具的缓存与构建程序的身份共用;工程自己的树在原处构建(mcpp 2026.11.19.1,mcpp#974) | | 1.9 | 2026-11-19 | 新增 §3.5:描述符 `mcpp` 字段的两种形式、一个读取函数、发布写出指针形式(mcpp 2026.11.19.1,mcpp#960) | | 1.8 | 2026-11-09 | 新增 §5.5:git 引用至多一个 ref、缺省分支、命令行的字符串形式(mcpp 2026.11.9.1) | | 1.5 | 2026-09-27 | 新增 §10.1:版本位的回答与 xlings 一致(安装记录优先,其次按 xlings 版本文法选择;已记录而缺失的载荷不算已安装),mcpp#712、#716;新增 §10.2:打包修订 `revision` 的含义与 mcpp 侧的三处使用(mcpp 2026.9.27.1,xlings 2026.9.27.1) | diff --git a/docs/zh/04-mcpp-toml.md b/docs/zh/04-mcpp-toml.md index 3eb50f653..58efbbe38 100644 --- a/docs/zh/04-mcpp-toml.md +++ b/docs/zh/04-mcpp-toml.md @@ -1684,14 +1684,20 @@ cache = "global" # "global" (default) | "local" | "off" `[build] cache` **>** `global`。一个无法识别的取值会被报告(`--strict` 下是错误),而不是静默回落到 `global`。 -**不会**被缓存的:`path` 与 `git` 依赖,不论层级多深,以及 -workspace 成员。它们的源码可以在 `name@version` 不变的情况下改变, -所以没有任何基于那个身份的键能察觉到变化。 - -构建程序 import 的库遵循同一条规则(2026.10.11.1+)。引擎接口(`mcpp.core`)与索引包保存在 -全局缓存中;`path` 或 `git` 包、或 workspace 成员在 `target/.build-mcpp/host/` 下的构建程序图 -中编译,绝不进入全局缓存。`--cache local` 与 `--cache off` 把它们全部保存在工程内。见 -[30 — 构建程序图](30-build-mcpp.md)。 +条目以构建它的源码命名 *(2026.11.19.1+)*:索引包以 `name@version`,`git` 依赖以解析得到的提交, +`path` 依赖以其源码树内容的摘要,在 `mcpp cache list` 中写作 `+path.<摘要>`。摘要覆盖依赖根目录下 +每个文件的相对路径与内容,不含 `target/`、`.git/`、`.mcpp/`、`compile_commands.json` 与含自己 `mcpp.toml` +的子目录。两个经 `path` 使用同一个库的工程、一次 clean 之后的构建、以及恢复 `$MCPP_HOME/build-cache` 的 +CI 运行,只要库的源码相同就复用同一个条目,与源码树检出在哪里无关;一次修改是一个新键,同一棵树在先前摘要下 +的条目在写入新条目时移除。`path` 依赖的条目在写入前重新读取源码树,编译期间改变了的树不写入。 + +**不会**被缓存的:工程自己的树,即正在编辑的那些——根包、workspace 成员,以及位于工程目录之下(或构建程序、 +宿主工具所服务的那次构建的目录之下)的任何包。2026.11.19.1 之前,`path` 与 `git` 依赖不论层级多深都不缓存 +(mcpp#974)。 + +构建程序 import 的库遵循同一条规则(2026.10.11.1+):引擎接口(`mcpp.core`)、索引包与以名字识别的 `git`、 +`path` 依赖保存在全局缓存中,工程自己的包在 `target/.build-mcpp/host/` 下的构建程序图中编译。 +`--cache local` 与 `--cache off` 把它们全部保存在工程内。见[30 — 构建程序图](30-build-mcpp.md)。 查看与回收: diff --git a/docs/zh/09-commands-by-scenario.md b/docs/zh/09-commands-by-scenario.md index 27f448183..17bcbb758 100644 --- a/docs/zh/09-commands-by-scenario.md +++ b/docs/zh/09-commands-by-scenario.md @@ -84,8 +84,8 @@ host 工具条目。一份 `std` BMI 被机器上每个工程共享,实现以" 构建成功后,每天至多一次,移除 `[cache] unused_days` 天(config.toml;默认 30, `0` 关闭)内没有构建使用过的条目,包、host 工具与 `std` 均在其列;命中即记为使用。 -`path` host 工具的源树变化时,写入新条目的同时移除同一源树在旧 stamp 下的条目 -(2026.10.31.3+)。构建前,若所写入的文件系统可用空间少于 2 GiB 与"该配置上一次构建写入量的 1.25 倍"中的较大者 +`path` host 工具或 `path` 依赖的源树变化时,写入新条目的同时移除同一源树在其内容先前摘要下的条目 +(host 工具 2026.10.31.3+,依赖与内容摘要 2026.11.19.1+)。构建前,若所写入的文件系统可用空间少于 2 GiB 与"该配置上一次构建写入量的 1.25 倍"中的较大者 (记录在 `<输出目录>/.mcpp-written`),mcpp 给出警告并说明两者(2026.11.8.1+)。`mcpp cache prune` 与 `mcpp cache gc --older-than` 也按年龄收集 `std` 模块条目;`gc --max-size` 只对包与宿主工具做 预算(2026.11.8.1+)。 diff --git a/docs/zh/30-build-mcpp.md b/docs/zh/30-build-mcpp.md index 1024f9ac7..ef02ec53f 100644 --- a/docs/zh/30-build-mcpp.md +++ b/docs/zh/30-build-mcpp.md @@ -1492,13 +1492,15 @@ grpc = { version = "1.83.0", tools = ["grpc_cpp_plugin"] } **这个键里有源码,以每种来源能给出的形式。** 已发布的版本不可变,所以对来自索引的 工具,版本本身就标识了源码。`git` 工具按解析出的 commit 键控。`path` 工具没有随源码 -移动的版本,所以按树的印记键控:每个普通文件的相对路径、大小与修改时间,不计 -`target/`、`.git/`、`.mcpp/` 与 compile database。于是改动工具的 emitter 在下一次构建 -到达消费者,改回去也到达,而没有变化的树仍是 store 命中、不重建。在 +移动的版本,所以按树内容的摘要键控,与依赖缓存是同一个身份([04 §2.10](04-mcpp-toml.md)): +每个普通文件的相对路径与内容,不计 `target/`、`.git/`、`.mcpp/`、compile database 与带 +自己 `mcpp.toml` 的子目录 *(2026.11.19.1 起;此前是每个文件的路径、大小与修改时间)*。 +于是改动工具的 emitter 在下一次构建到达消费者,改回去也到达,而内容没有变化的树仍是 +store 命中、不重建,检出或复制改变了全部修改时间之后也是。在 [`examples/12-a-new-device-language`](../../examples/12-a-new-device-language/) 上实测;此前这个键只有版本,改动 emitter 之后 `mcpp run` 打印的是上一次的答案,直到 -版本被抬高。树被编辑时条目会累积;`mcpp cache clean` 清空 store,路径是 -`/tool//@[+]/`。 +版本被抬高。新条目写出时,同一棵树在先前摘要下的条目被删除;`mcpp cache clean` 清空 +store,路径是 `/tool//@[+]/`。 **工具构建写出的路径长度有界(2026.9.15.2+)。** 子构建在 `/tool/.build//` 中进行,这个目录由条目与消费工程命名,不嵌套在 diff --git a/modules/buildmcpp/src/tool_store.cppm b/modules/buildmcpp/src/tool_store.cppm index 53f082fa4..555185e93 100644 --- a/modules/buildmcpp/src/tool_store.cppm +++ b/modules/buildmcpp/src/tool_store.cppm @@ -100,28 +100,10 @@ struct Key { std::string key_hex(const Key& k); nlohmann::json to_json(const Key& k); -// THE IDENTITY OF A SOURCE TREE THAT HAS NO VERSION OF ITS OWN. -// -// An index package's `name@version` names immutable content, so the version -// alone identifies the bytes a tool was built from. A `path` package changes -// under an unchanged version, and a `git` package at a branch moves; the store -// keyed on the version alone then keeps a binary built from sources that no -// longer exist. Measured (2026-09-08, examples/12): an emitter change in a path -// tool package reached the consumer only after the package's version was -// bumped, and until then every build reported success over the previous -// compiler's output. -// -// A `git` package is keyed by its resolved commit, which is immutable content. -// A `path` package is keyed by this stamp: every regular file's relative path, -// size and modification time, in sorted order, hashed. It is what ninja itself -// trusts to decide a rebuild, costs one `stat` per file rather than a read, -// and moves in both directions -- an edit and its reversal each produce a new -// key, which is what the criterion demands. Build products, the version -// control directory and the engine's own scratch are excluded, since they -// change without the sources changing. So is any directory below the root -// that holds its own mcpp.toml: that is another package, keyed by its own -// stamp when the tool depends on it and no input otherwise (#705). -std::string tree_stamp(const std::filesystem::path& root); +// A SOURCE TREE THAT HAS NO VERSION OF ITS OWN is named in a key by the digest +// of its content (`+path.`), which the engine computes once for +// the build programs, this store and the dependency cache (mcpp.build +// source_digest, mcpp#974). A `git` package is keyed by its resolved commit. // /tool//@// std::filesystem::path entry_dir(const std::filesystem::path& cacheRoot, const Key& k); @@ -220,43 +202,6 @@ nlohmann::json to_json(const Key& k) { return j; } -std::string tree_stamp(const fs::path& root) { - std::vector rows; - std::error_code ec; - fs::recursive_directory_iterator it(root, fs::directory_options::skip_permission_denied, ec); - for (; it != fs::recursive_directory_iterator(); it.increment(ec)) { - if (ec) break; - const auto& p = it->path(); - const auto name = p.filename(); - if (it->is_directory(ec)) { - // A directory holding its own mcpp.toml is ANOTHER PACKAGE (#705). - // If the tool depends on it, that package is in the key through - // `upstreamKeys` with its own stamp; if not, nothing in it is an - // input of this tool. The case that forced this is a consumer - // nested inside the tool's tree -- a fixture, an example, a - // workspace member: every build of the consumer wrote under its - // own directory and so rebuilt the tool it was building with. - if (name == "target" || name == ".git" || name == ".mcpp" - || fs::is_regular_file(p / "mcpp.toml", ec)) - it.disable_recursion_pending(); - continue; - } - if (!it->is_regular_file(ec)) continue; - if (name == "compile_commands.json") continue; - // UTF-8, not the code page: this is a walk of a tree mcpp does not - // control, and a stamp is an identity (check_narrow_conversions.sh). - const auto rel8 = p.lexically_relative(root).generic_u8string(); - const std::string rel(rel8.begin(), rel8.end()); - const auto sz = fs::file_size(p, ec); - const auto mt = fs::last_write_time(p, ec).time_since_epoch().count(); - rows.push_back(std::format("{}|{}|{}", rel, sz, mt)); - } - std::ranges::sort(rows); - std::string joined; - for (auto const& r : rows) { joined += r; joined += '\n'; } - return mcpp::toolchain::hash_string(joined); -} - std::string key_hex(const Key& k) { return mcpp::toolchain::hash_string(to_json(k).dump()); } diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index 44e1e4f3c..d21302448 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -664,6 +664,10 @@ export struct BuildContext { struct CacheTask { mcpp::bmi_cache::CacheKey key; mcpp::bmi_cache::DepArtifacts artifacts; + // A `path` dependency's tree, the file of its stat rows and the digest + // its key holds (mcpp#974): read again before the entry is written. + std::filesystem::path treeRoot, treeRows; + std::string treeDigest; }; std::vector depsToPopulate; @@ -890,6 +894,11 @@ export struct BuildOverrides { // outermost first. A request for one of them is the tool's own build asking // for itself, refused at its first repetition (#649 E6). std::vector tool_chain_sources; + // The project trees of the builds above this one (mcpp#974): a package + // below one of them is that project's own and is built in place, as one + // below this build's root is, rather than shared through the global cache. + // Each sub-build is given its parent's, and its parent's root. + std::vector own_roots; // The toolchain spec a host-tool sub-build uses, decided by the build that // requested the tool and recorded in the tool's store key (#710). Beats // every other source, `--toolchain` included, because the requesting build diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index e66bcfd2e..29dbf9b66 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -1469,10 +1469,10 @@ step6_check_tool_self_request(PrepareState& state, HostToolCtx& ctx) { // The dependency closure a host tool's own build resolves (mcpp#874): its // package's `[dependencies]`, transitively, read from the manifests. A path -// package is its version and its tree's stamp, and is walked into; a git or -// index package is what the manifest asks for. Dev-dependencies are never in -// it: the tool's build builds no tests. -static std::vector tool_upstream_keys(const std::filesystem::path& root, +// package is its version and the digest of its tree's content (mcpp#974), and +// is walked into; a git or index package is what the manifest asks for. +// Dev-dependencies are never in it: the tool's build builds no tests. +static std::vector tool_upstream_keys(PrepareState& state, const std::filesystem::path& root, const mcpp::manifest::Manifest& manifest) { namespace fs = std::filesystem; std::vector keys; @@ -1494,7 +1494,7 @@ static std::vector tool_upstream_keys(const std::filesystem::path& const auto at = fs::weakly_canonical(dir / spec.path, ec); auto dm = mcpp::manifest::load(at / "mcpp.toml", {.insideWorkspace = true}); keys.push_back(std::format("{}@{}+path.{}", name, dm ? dm->package.version : std::string{}, - mcpp::build::tool_store::tree_stamp(at))); + tree_content_digest(state, at).substr(0, 16))); if (dm && seen.insert(at).second) work.emplace_back(at, std::move(*dm)); } } @@ -1551,24 +1551,14 @@ step6_resolve_tool_key(PrepareState& state, HostToolCtx& ctx) { ctx.key.packageName = ctx.depName; // THE VERSION IDENTIFIES THE SOURCES ONLY FOR AN INDEX // PACKAGE. A `git` package is keyed by its commit and a - // `path` package by a stamp of its tree, because both - // change under an unchanged version and the store then - // serves a binary built from sources that no longer exist - // (#630, item 6; measured 2026-09-08 with examples/12). - // The same rule applies to every upstream below. + // `path` package by the digest of its tree's content, + // because both change under an unchanged version and the + // store then serves a binary built from sources that no + // longer exist (#630, item 6; measured 2026-09-08 with + // examples/12). The same rule applies to every upstream + // below, and is the dependency cache's (mcpp#974). auto source_keyed_version = [&](std::size_t pkgIdx) { - const auto& man = state.packages[pkgIdx].manifest.package; - std::string v = man.version; - if (pkgIdx >= 1 && pkgIdx - 1 < state.dep_cache_identities.size()) { - const auto& id = state.dep_cache_identities[pkgIdx - 1]; - if (id.sourceKind == "git" && !id.sourceRef.empty()) - v += "+git." + id.sourceRef; - else if (id.sourceKind == "path") - v += "+path." + mcpp::build::tool_store::tree_stamp( - id.sourceRef.empty() ? state.packages[pkgIdx].root - : std::filesystem::path(id.sourceRef)); - } - return v; + return state.packages[pkgIdx].manifest.package.version + source_identity_suffix(state, pkgIdx); }; ctx.key.version = source_keyed_version(ctx.depIdx); ctx.key.targetName = ctx.toolName; @@ -1592,7 +1582,7 @@ step6_resolve_tool_key(PrepareState& state, HostToolCtx& ctx) { // `mcpp build`), and a root's dev-dependencies are walked while its tests // are built. Either way the key changed between commands, the tool's path // with it, and every action that ran the tool ran again. - ctx.key.upstreamKeys = tool_upstream_keys(state.packages[ctx.depIdx].root, + ctx.key.upstreamKeys = tool_upstream_keys(state, state.packages[ctx.depIdx].root, state.packages[ctx.depIdx].manifest); std::ranges::sort(ctx.key.upstreamKeys); @@ -1678,6 +1668,8 @@ step6_build_tool(PrepareState& state, HostToolCtx& ctx) { sub.tool_depth = state.overrides.tool_depth + 1; sub.tool_chain_sources = state.overrides.tool_chain_sources; sub.tool_chain_sources.push_back(ctx.toolSource); + sub.own_roots = state.overrides.own_roots; + sub.own_roots.push_back(state.workRoot); // The PRISTINE manifest the resolver produced for this // package — `packages[depIdx].manifest` is a copy that // feature activation has already mutated, and re-activating diff --git a/src/build/prepare/host_graph.cpp b/src/build/prepare/host_graph.cpp index d5a6f8a1f..601a88e43 100644 --- a/src/build/prepare/host_graph.cpp +++ b/src/build/prepare/host_graph.cpp @@ -46,6 +46,7 @@ import mcpp.manifest; import mcpp.config; // resolve_project_index_path import mcpp.modgraph.scanner; import mcpplibs.os.platform; +import mcpp.home; import mcpp.home.files; import mcpp.pm.mangle; import mcpp.toolchain.fingerprint; @@ -251,6 +252,32 @@ std::string standard_of(const PrepareState& state, std::size_t i) { return s.empty() ? std::string("c++23") : s; } +} // namespace + +fs::path tree_digest_rows(const fs::path& root) { + return mcpp::home::root() / "build-cache" / "digests" + / (mcpp::toolchain::hash_string(path_text(root)).substr(0, 16) + ".tsv"); +} + +std::string tree_content_digest(PrepareState& state, const fs::path& root) { + const auto key = path_text(root); + if (auto it = state.treeDigests.find(key); it != state.treeDigests.end()) return it->second; + auto digest = mcpp::build::source_tree_digest(root, tree_digest_rows(root)); + state.treeDigests.emplace(key, digest); + return digest; +} + +std::string source_identity_suffix(PrepareState& state, std::size_t pkgIdx) { + if (pkgIdx == 0 || pkgIdx - 1 >= state.dep_cache_identities.size()) return {}; + const auto& id = state.dep_cache_identities[pkgIdx - 1]; + if (id.sourceKind == "git" && !id.sourceRef.empty()) return "+git." + id.sourceRef; + if (id.sourceKind != "path") return {}; + const fs::path root = id.sourceRef.empty() ? state.packages[pkgIdx].root : fs::path(id.sourceRef); + return "+path." + tree_content_digest(state, root).substr(0, 16); +} + +namespace { + // The identity of program `i`: what its compile and link read. std::string program_identity(PrepareState& state, std::size_t i, const ProgramSource& src, const HostToolchain& tc) { @@ -268,12 +295,9 @@ std::string program_identity(PrepareState& state, std::size_t i, const ProgramSo std::string source = ident ? ident->sourceKind + ":" + ident->sourceRef : ""; std::string tree; // A tree that can change in place: anything but an index package or - // a pinned git checkout. - if (!ident || (ident->sourceKind != "version" && ident->sourceKind != "git")) { - const auto cache = host_root(state) / "digests" - / (mcpp::toolchain::hash_string(path_text(pkg.root)) + ".tsv"); - tree = mcpp::build::source_tree_digest(pkg.root, cache); - } + // a pinned git checkout (`source_identity_suffix` names the same). + if (!ident || (ident->sourceKind != "version" && ident->sourceKind != "git")) + tree = tree_content_digest(state, pkg.root); const auto& f = d < state.buildProgramFeaturesByPackage.size() ? state.buildProgramFeaturesByPackage[d] : std::vector{}; text += std::format("package {} {} {} [{}] {}\n", qualified(pkg.manifest), @@ -750,6 +774,8 @@ build_group(PrepareState& state, const fs::path& dir, const std::vector sub.strict = state.overrides.strict; sub.tool_depth = state.overrides.tool_depth + 1; sub.tool_chain_sources = state.overrides.tool_chain_sources; + sub.own_roots = state.overrides.own_roots; + sub.own_roots.push_back(state.workRoot); sub.tool_chain = state.overrides.tool_chain.empty() ? std::string("root → build programs") : state.overrides.tool_chain + " → build programs"; diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index 43a0b3440..313bd3d96 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -17,6 +17,7 @@ import mcpp.runtime.elf; // DT_NEEDED of a library the line names import mcpp.build.refusal; import mcpp.build.version_floor; import mcpp.home; +import mcpp.build.source_digest; import mcpplibs.os.platform; import mcpp.invocation; import mcpp.effects; @@ -96,6 +97,23 @@ namespace mcpp::build { static void step13_report_packages(PrepareState& state, BuildContext& ctx); +// WHETHER A PACKAGE'S SOURCES ARE NAMED, so that its build is shared through the +// global cache by that name (mcpp#974): an index package by its version, a git +// dependency by its resolved commit, a path dependency by the digest of its +// tree's content. The project's own trees -- the root, a selected member, any +// package below this build's root or the root of a build above it -- are named +// by nothing and built in place: they are what is being edited. +static bool sources_named(const PrepareState& state, std::size_t i) { + if (i == 0 || i - 1 >= state.dep_cache_identities.size()) return false; + const auto& id = state.dep_cache_identities[i - 1]; + if (id.sourceKind == "version") return true; + if (id.sourceKind == "git") return !id.sourceRef.empty(); + if (id.sourceKind != "path" || state.packages[i].selectedMember) return false; + std::vector own = state.overrides.own_roots; + own.push_back(state.workRoot); + return !mcpp::build::path_is_under_any(state.packages[i].root, own); +} + static std::expected step13_source_packages(PrepareState& state, BuildContext& ctx) { { std::error_code ec; @@ -1758,6 +1776,9 @@ static std::expected step13_dependency_cache(PrepareState& st pa.indexName = state.dep_cache_identities[idx - 1].indexName; pa.packageName = state.dep_cache_identities[idx - 1].packageName; pa.version = state.dep_cache_identities[idx - 1].version; + // A git or path dependency's sources by their own name (mcpp#974), + // so its key, and every key that folds it in, moves with them. + if (sources_named(state, idx)) pa.version += source_identity_suffix(state, idx); } if (pa.packageName.empty()) { // The root package, or a package with no resolution identity. @@ -1793,10 +1814,7 @@ static std::expected step13_dependency_cache(PrepareState& st pa.cflags.push_back(state.ltoOffWord); pa.cxxflags.push_back(state.ltoOffWord); } - const bool selfIsIndex = idx > 0 - && idx - 1 < state.dep_cache_identities.size() - && state.dep_cache_identities[idx - 1].sourceKind == "version"; - if (!selfIsIndex) localTaint[idx] = 1; + if (!sources_named(state, idx)) localTaint[idx] = 1; for (auto& e : state.dependencyEdges) { if (e.consumerPackageIndex != idx || serves_root_tests(e)) continue; // `idx`'s dependencies precede it in `keyOrder`, so their @@ -1844,8 +1862,13 @@ static std::expected step13_dependency_cache(PrepareState& st // opposite — anything it cannot prove came from the immutable // xpkgs store stays out, because the failure mode here is a // silently wrong object rather than a rejected command. - if (!depIdent || depIdent->sourceKind != "version") continue; - // ...and neither may anything it was built against be local. + // + // A GIT OR PATH DEPENDENCY NAMES ITS SOURCES TOO (mcpp#974): its + // resolved commit, or the digest of its tree's content, is in its + // key (`sources_named`), so a change to the sources is a change of + // key. The project's own trees are not named and stay out. + if (!depIdent || !sources_named(state, i)) continue; + // ...and neither may anything it was built against be unnamed. if (localTaint[i]) continue; // ...and the package's sources must ACTUALLY be in the immutable // store, not merely labelled as coming from it. @@ -1875,13 +1898,14 @@ static std::expected step13_dependency_cache(PrepareState& st // and the package stops looking like a store package at all. The // question here is where the payload was INSTALLED, which is a // statement about the path, not about the inode. - if (!mcpp::build::path_is_under_any(pkgRoot.root, state.storeRoots)) + const bool fromStore = depIdent->sourceKind == "version"; + if (fromStore && !mcpp::build::path_is_under_any(pkgRoot.root, state.storeRoots)) continue; const auto& depName = depIdent->packageName; - const auto& depVer = depIdent->version.empty() + const auto depVer = (depIdent->version.empty() ? pkgRoot.manifest.package.version - : depIdent->version; + : depIdent->version) + (fromStore ? std::string{} : source_identity_suffix(state, i)); auto bmiT = mcpp::toolchain::bmi_traits(*state.tc); mcpp::bmi_cache::CacheKey key { @@ -1964,7 +1988,12 @@ static std::expected step13_dependency_cache(PrepareState& st } mcpp::bmi_cache::touch_accessed(key); state.usedCacheRecords.insert(key.entryFile().string()); - ctx.cachedDeps.push_back({depName, depVer, unitIdx.size()}); + // By the name the package report gives it: a dependency key + // is not always its qualified name (`greet` for the default + // namespace's `mcpplibs.greet`), and the line then read + // `Compiling` for a package the cache served. + ctx.cachedDeps.push_back({mcpp::build::qualified_package_name(pkgRoot.manifest), depVer, + unitIdx.size()}); continue; // no populate task; it is already cached } // A valid entry that does not hold what we asked for means the @@ -1984,7 +2013,17 @@ static std::expected step13_dependency_cache(PrepareState& st probe.layoutMismatch.front())); } for (auto u : unitIdx) ctx.plan.compileUnits[u].fillsCache = true; - ctx.depsToPopulate.push_back({ std::move(key), std::move(arts) }); + BuildContext::CacheTask task{ std::move(key), std::move(arts) }; + // The tree is read again before its entry is written: a source + // edited while it compiled is not stored under the digest of the + // sources before the edit (mcpp#974). + if (depIdent->sourceKind == "path") { + task.treeRoot = depIdent->sourceRef.empty() ? pkgRoot.root + : std::filesystem::path(depIdent->sourceRef); + task.treeRows = tree_digest_rows(task.treeRoot); + task.treeDigest = tree_content_digest(state, task.treeRoot); + } + ctx.depsToPopulate.push_back(std::move(task)); } } // ────────────────────────────────────────────────────────────────── @@ -2009,8 +2048,7 @@ static void step13_split_debuginfo(PrepareState& state, BuildContext& ctx) { return; std::set fromIndex; for (std::size_t i = 1; i < state.packages.size(); ++i) - if (i - 1 < state.dep_cache_identities.size() - && state.dep_cache_identities[i - 1].sourceKind == "version") + if (sources_named(state, i)) fromIndex.insert(mcpp::build::qualified_package_name(state.packages[i].manifest)); const auto split = mcpp::manifest::flag_element("-gsplit-dwarf"); for (auto& cu : ctx.plan.compileUnits) { @@ -2430,10 +2468,52 @@ void focus_on_member(BuildContext& ctx) { ctx.projectRoot = m.root; } +// AN EDITED TREE'S PREVIOUS ENTRY IS REPLACED (mcpp#974), as a `path` host +// tool's is (tool_store::write_entry, mcpp#874). Every edit of a `path` +// dependency is a new key, and none would be collected before `[cache] +// unused_days`: the tree an entry was built from is written beside it, and an +// entry of the same package from the same tree under another digest is removed +// as this one is written. Another checkout of the package is another tree, and +// keeps its entries. +static void replace_tree_siblings(const mcpp::bmi_cache::CacheKey& key, const std::filesystem::path& tree) { + namespace fs = std::filesystem; + const auto plus = key.version.find("+path."); + if (plus == std::string::npos) return; + const auto origin = tree.lexically_normal().generic_u8string(); + const std::string text(origin.begin(), origin.end()); + std::error_code ec; + std::ofstream(key.dir() / "source", std::ios::binary | std::ios::trunc) << text; + const auto versionDir = key.dir().parent_path(); + const auto prefix = std::format("{}@{}", key.packageName, key.version.substr(0, plus + 6)); + auto from_this_tree = [&](const fs::path& entry) { + std::ifstream in(entry / "source", std::ios::binary); + return in && std::string{std::istreambuf_iterator(in), {}} == text; + }; + for (auto const& v : fs::directory_iterator(versionDir.parent_path(), ec)) { + const auto name = v.path().filename().u8string(); + if (v.path() == versionDir || !std::string(name.begin(), name.end()).starts_with(prefix)) continue; + std::error_code kec; + for (auto const& entry : fs::directory_iterator(v.path(), kec)) + if (from_this_tree(entry.path())) fs::remove_all(entry.path(), kec); + fs::remove(v.path(), kec); // only when nothing else is left in it + } +} + void populate_dependency_cache(BuildContext& ctx) { if (ctx.cacheMode != CacheMode::Global) ctx.depsToPopulate.clear(); for (auto& task : ctx.depsToPopulate) { + if (!task.treeRoot.empty()) { + if (mcpp::build::source_tree_digest(task.treeRoot, task.treeRows) != task.treeDigest) { + const auto tree = task.treeRoot.lexically_normal().generic_u8string(); + mcpp::diag::degraded("build/cache", std::format( + "the sources of {} changed while it was built ({}), so its build was not stored in the global cache", + task.key.packageName, std::string(tree.begin(), tree.end())), + "the next build compiles it again, from the sources it then finds"); + continue; + } + } auto pr = mcpp::bmi_cache::populate_from(task.key, ctx.outputDir, task.artifacts); + if (pr && !task.treeRoot.empty()) replace_tree_siblings(task.key, task.treeRoot); if (!pr) { mcpp::ui::warning(std::format( "bmi cache populate failed for {}@{}: {}", diff --git a/src/build/prepare/state.cppm b/src/build/prepare/state.cppm index 231c69700..11484cc98 100644 --- a/src/build/prepare/state.cppm +++ b/src/build/prepare/state.cppm @@ -752,6 +752,9 @@ struct PrepareState { std::string bootstrapSpec; std::vector> dep_manifests; std::vector dep_cache_identities; + // The content digest of each tree that can change in place, once per + // preparation (`tree_content_digest`, mcpp#974). + std::map treeDigests; std::map root_git_lock_identities; // Git dependencies reached through another package, by qualified // identity; written to mcpp.lock beside the root's and read back as their @@ -1107,6 +1110,19 @@ void step4b_note_exact_pin(PrepareState& state, const WorkItem& item, const mcpp std::expected makePackageRoot( PrepareState& state, const std::filesystem::path& packageRoot, const mcpp::manifest::Manifest& manifest); +// THE IDENTITY OF A PACKAGE'S SOURCES (mcpp#974): what names the bytes a +// package's build reads where its version does not. An index package's version +// does, its payload being immutable: empty. A `git` package's resolved commit: +// `+git.`. A `path` package's tree by the digest of its content: +// `+path.`. One answer for the dependency cache, the tool store and the +// build programs' identity (host_graph.cpp). +std::string source_identity_suffix(PrepareState& state, std::size_t pkgIdx); +// The content digest of a tree that can change in place, once per tree in a +// preparation; the stat rows that spare reading unchanged files are kept under +// the home, shared by every project that reaches the tree. +std::string tree_content_digest(PrepareState& state, const std::filesystem::path& root); +// Where those stat rows are kept for `root`. +std::filesystem::path tree_digest_rows(const std::filesystem::path& root); // Where the build program of package `i` keeps its files: its cache, its // graph document, and the program mcpp synthesises for it. The root's and a // workspace member's are below their own target/; a dependency's is below the diff --git a/src/build/source_digest.cppm b/src/build/source_digest.cppm index 3ea667c96..08db121f8 100644 --- a/src/build/source_digest.cppm +++ b/src/build/source_digest.cppm @@ -21,9 +21,20 @@ // recorded without a hash, and the next walk reads the file. // // WHAT IS WALKED. Every regular file below the root, except `target`, `.git` -// and `.mcpp`, and except a subdirectory holding its own `mcpp.toml`: that is -// another package (a workspace member), with its own digest where something -// depends on it -- the tool store's rule (SPEC-004 §10.1). +// and `.mcpp`, the compile database the engine writes beside a project's +// manifest (`compile_commands.json`), and a subdirectory holding its own +// `mcpp.toml`: that is another package (a workspace member), with its own +// digest where something depends on it (SPEC-004 §10.1). +// +// ONE IDENTITY FOR A TREE WITHOUT A VERSION (mcpp#974). The build programs' +// identity, the tool store's key and the dependency cache's key all name a +// `path` package by this digest: a content, not a time, so an edit and its +// reversal return to the same key, and two checkouts of the same sources -- +// a CI run after another -- are one entry. +// +// THE ROWS FILE IS SHARED. Two preparations may walk one tree at once (two +// projects that reach it), so each writes its rows under a name of its own and +// renames it over the file. export module mcpp.build.source_digest; @@ -99,6 +110,7 @@ std::string source_tree_digest(const fs::path& root, const fs::path& statCache) continue; } if (!it->is_regular_file(tec)) continue; + if (it->path().filename() == "compile_commands.json") continue; const auto rel = utf8(it->path().lexically_relative(root)); Row r; r.size = it->file_size(tec); @@ -127,7 +139,9 @@ std::string source_tree_digest(const fs::path& root, const fs::path& statCache) if (moved && !statCache.empty()) { fs::create_directories(statCache.parent_path(), ec); - const auto tmp = statCache.string() + ".tmp"; + const auto tmp = std::format("{}.{:016x}.tmp", statCache.string(), + (static_cast(std::random_device{}()) << 32) + ^ static_cast(std::random_device{}())); { std::ofstream out(tmp, std::ios::binary | std::ios::trunc); out << "mcpp-source-digest-v1\n"; @@ -135,6 +149,7 @@ std::string source_tree_digest(const fs::path& root, const fs::path& statCache) out << std::format("{}\t{}\t{}\t{}\n", rel, r.size, r.mtime, r.hash); } fs::rename(tmp, statCache, ec); + if (ec) fs::remove(tmp, ec); } return digest; } diff --git a/tests/e2e/modules/bmi.cpp b/tests/e2e/modules/bmi.cpp index 90f7ba5d6..792a2b177 100644 --- a/tests/e2e/modules/bmi.cpp +++ b/tests/e2e/modules/bmi.cpp @@ -1,7 +1,7 @@ -// BMIs and their cache: path dependencies stay out of the build cache, a custom index's nested -// objects are cached and staged, purview and header edits rebuild, staging neither cascades nor -// rewrites an equal destination, the split schedule end to end, a staged BMI that waits for a -// module compiled here, and the std module compiled once by two builds that find it missing +// BMIs and their cache: a path dependency outside the project is cached by its tree's digest, a +// custom index's nested objects are cached and staged, purview and header edits rebuild, staging +// neither cascades nor rewrites an equal destination, the split schedule end to end, a staged BMI +// that waits for a module compiled here, and the std module compiled once by two builds that find it missing // (formerly tests/e2e/19, 49, 118, 170, 171, 234, 849, 1039). 1039 removes the shared home's std // entry that 170 and 171 stage from; in one program its `machine` resource keeps them apart. import std; @@ -24,13 +24,15 @@ std::optional graph_of(const fs::path& root) { return std::nullopt; } -// Any path below `root` whose text contains `part` (`find -path "**"`). -bool any_path(const fs::path& root, std::string_view part) { +// The build cache's entries whose name begins with `prefix` (`//*`), sorted. +std::vector entries_of(const fs::path& pkg, std::string_view prefix) { + std::vector out; std::error_code ec; - if (!fs::exists(root, ec)) return false; - for (auto it = fs::recursive_directory_iterator(root, ec); it != fs::recursive_directory_iterator(); it.increment(ec)) - if (it->path().string().contains(part)) return true; - return false; + for (const auto& ns : fs::directory_iterator(pkg, ec)) + for (const auto& e : fs::directory_iterator(ns.path(), ec)) + if (e.path().filename().string().starts_with(prefix)) out.push_back(e.path()); + std::ranges::sort(out); + return out; } // `sed 's///'` over a file: the first occurrence on each line. @@ -99,39 +101,55 @@ std::string lua_xpm(std::string_view name) { E2E bmi{"bmi", [] { "19 bmi cache reuse"_test = [] { - // Path dependencies, direct and transitive, never enter the build cache nor `cache - // list`, and an edit to a transitive one reaches the binary. + // A path dependency outside the project, direct or transitive, is kept in the build + // cache under the digest of its tree (mcpp#974): a clean build takes it from there, an + // edit is a new key that replaces the entry built from the same tree, the edit reaches + // the binary, and `cache list` names the entry by `+path.`. A package below the + // project's own directory is what is being edited, and is built in place. const s::options home{.where = s::home::inherited}; const auto mcpp_home = s::case_home(s::home::inherited); + const auto pkg = mcpp_home / "build-cache/v1/pkg"; const auto tmp = s::case_dir(); const auto run = [&](const fs::path& dir, std::vector args) { return s::mcpp(std::move(args), dir, home); }; + const auto lib = [](std::string_view name) { + return std::format("[package]\nname = \"{0}\"\nversion = \"0.1.0\"\n[language]\nstandard = \"c++23\"\nmodules = true\nimport_std = true\n[modules]\nsources = [\"src/**/*.cppm\"]\n[targets.{0}]\nkind = \"lib\"\n", name); + }; expect(fatal(e2e::succeeded(run(tmp, {"new", "mylibA"})))); s::project a(tmp / "mylibA", {{"src/greet.cppm", "export module mylibA.greet;\nimport std;\nexport auto greet() -> void { std::println(\"hi\"); }\n"}, - {"mcpp.toml", "[package]\nname = \"mylibA\"\nversion = \"0.1.0\"\n[language]\nstandard = \"c++23\"\nmodules = true\nimport_std = true\n[modules]\nsources = [\"src/**/*.cppm\"]\n[targets.mylibA]\nkind = \"lib\"\n"}}); + {"mcpp.toml", lib("mylibA")}}); a.remove("src/main.cpp"); expect(fatal(e2e::succeeded(run(tmp, {"new", "myapp"})))); - s::project app(tmp / "myapp", {{"src/main.cpp", "import std;\nimport mylibA.greet;\nint main() { greet(); return 0; }\n"}, - {"mcpp.toml", "[package]\nname = \"myapp\"\nversion = \"0.1.0\"\n[language]\nstandard = \"c++23\"\nmodules = true\nimport_std = true\n[modules]\nsources = [\"src/**/*.cppm\", \"src/**/*.cpp\"]\n[targets.myapp]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n[dependencies.mylibA]\npath = \"../mylibA\"\n"}}); + s::project app(tmp / "myapp", {{"src/main.cpp", "import std;\nimport mylibA.greet;\nimport mylibC.own;\nint main() { greet(); return own(); }\n"}, + {"vendor/mylibC/src/own.cppm", "export module mylibC.own;\nexport auto own() -> int { return 0; }\n"}, + {"vendor/mylibC/mcpp.toml", lib("mylibC")}, + {"mcpp.toml", "[package]\nname = \"myapp\"\nversion = \"0.1.0\"\n[language]\nstandard = \"c++23\"\nmodules = true\nimport_std = true\n[modules]\nsources = [\"src/**/*.cppm\", \"src/**/*.cpp\"]\n[targets.myapp]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n[dependencies.mylibA]\npath = \"../mylibA\"\n[dependencies.mylibC]\npath = \"vendor/mylibC\"\n"}}); auto r = app.mcpp({"build", "-v"}, home); expect(fatal(e2e::succeeded(r))) << s::text(r); - expect(fatal(fs::is_directory(mcpp_home / "build-cache/v1"))) << "missing " << (mcpp_home / "build-cache/v1").string(); - expect(fatal(!any_path(mcpp_home / "build-cache/v1/pkg", "mylibA"))) << "path dep mylibA was populated into the build cache (must be skipped)"; - expect(fatal(s::no_line(s::text(r), R"(Cached mylibA|Compiling mylibA .*cached [0-9]+ unit)"))) << "path dep wrongly labeled Cached\n" << s::text(r); + const auto firstA = entries_of(pkg, "mylibA@0.1.0+path."); + expect(fatal(firstA.size() == 1)) << "mylibA has " << firstA.size() << " entries in the build cache, expected one\n" << s::text(r); + expect(fatal(entries_of(pkg, "mylibC@").empty())) << "mylibC, below the project's directory, entered the build cache"; + + app.remove("target"); + r = app.mcpp({"build"}, home); + expect(fatal(e2e::succeeded(r))) << s::text(r); + expect(fatal(s::has_line(s::text(r), R"(Cached +mylibA v0\.1\.0 \(\.\./mylibA, [0-9]+ units?\))"))) << "a clean build did not take mylibA from the cache\n" << s::text(r); + expect(fatal(s::has_line(s::text(r), R"(Compiling +mylibC )"))) << "mylibC was not compiled in place\n" << s::text(r); expect(fatal(e2e::succeeded(run(tmp, {"new", "mylibB"})))); s::project b(tmp / "mylibB", {{"src/inner.cppm", "export module mylibB.inner;\nexport auto inner() -> int { return 7; }\n"}, - {"mcpp.toml", "[package]\nname = \"mylibB\"\nversion = \"0.1.0\"\n[language]\nstandard = \"c++23\"\nmodules = true\nimport_std = true\n[modules]\nsources = [\"src/**/*.cppm\"]\n[targets.mylibB]\nkind = \"lib\"\n"}}); + {"mcpp.toml", lib("mylibB")}}); b.remove("src/main.cpp"); a.write("src/greet.cppm", "export module mylibA.greet;\nimport std;\nimport mylibB.inner;\nexport auto greet() -> void { std::println(\"hi {}\", inner()); }\n"); a.write("mcpp.toml", a.read("mcpp.toml") + "[dependencies.mylibB]\npath = \"../mylibB\"\n"); - app.remove("target"); r = app.mcpp({"build"}, home); expect(fatal(e2e::succeeded(r))) << s::text(r); - for (std::string_view name : {"mylibA", "mylibB"}) - expect(fatal(!any_path(mcpp_home / "build-cache/v1/pkg", name))) << "local package " << name << " entered the build cache"; + const auto nowA = entries_of(pkg, "mylibA@0.1.0+path."); + expect(fatal(nowA.size() == 1 && nowA != firstA)) << "the edited mylibA did not replace its entry: " << nowA.size() << " entries\n" << s::text(r); + expect(fatal(entries_of(pkg, "mylibB@0.1.0+path.").size() == 1)) << "the transitive mylibB did not enter the build cache"; const auto list = s::text(app.mcpp({"cache", "list"}, home)); for (std::string_view name : {"mylibA", "mylibB"}) - expect(fatal(s::lacks(list, name))) << "cache list shows local package " << name << "\n" << list; + expect(fatal(s::has_line(list, std::format(R"({}[^\n]*\+path\.)", name)))) << "cache list does not name " << name << " by its tree\n" << list; + expect(fatal(s::lacks(list, "mylibC"))) << "cache list shows mylibC\n" << list; b.write("src/inner.cppm", "export module mylibB.inner;\nexport auto inner() -> int { return 99; }\n"); r = app.mcpp({"run"}, home); expect(fatal(e2e::succeeded(r))) << s::text(r); diff --git a/tests/e2e/modules/programs.cpp b/tests/e2e/modules/programs.cpp index e8c8feaeb..c4d5d0cfd 100644 --- a/tests/e2e/modules/programs.cpp +++ b/tests/e2e/modules/programs.cpp @@ -174,7 +174,8 @@ main = "main.cpp" "857 a host module several members import is compiled once"_test = [] { // A path host-module library three members' build programs import is compiled once per // standard: two graphs (m1 and m2 share one, m3 has its own), none per member, and none of - // its modules enters the global cache, which holds the engine's interface. + // its modules enters the global cache, which holds the engine's interface: the library is + // below the workspace's directory, one of the project's own trees (mcpp#974). const s::options home{.where = s::home::inherited, .env = {{"MCPP_COMPILE_COMMANDS", "compile_commands.json"}}}; s::project ws({{"mcpp.toml", "[workspace]\nmembers = [\"m1\", \"m2\", \"m3\"]\n"}, {"rules/mcpp.toml", "[package]\nname = \"rule857\"\nversion = \"0.1.0\"\n[targets.rule857]\nkind = \"lib\"\n"}, @@ -215,7 +216,7 @@ main = "main.cpp" const fs::path cache_root = cache_dir.substr(0, cache_dir.find('\n')); bool leaked = false; for (const auto& e : s::find_files(cache_root / "pkg", R"(entry\.json)")) leaked = leaked || s::read_file(e).contains("t857.rules"); - expect(fatal(!leaked)) << "D: a module of a path dependency was written below the global cache root"; + expect(fatal(!leaked)) << "D: a module of a path dependency below the workspace was written below the global cache root"; expect(fatal(!s::find_files(cache_root / "pkg/_engine", R"(entry\.json)").empty())) << "D: the engine interface is not in the global cache\n" << s::text(r); }; diff --git a/tests/unit/test_source_digest.cpp b/tests/unit/test_source_digest.cpp index 2e178076e..a9ff730bd 100644 --- a/tests/unit/test_source_digest.cpp +++ b/tests/unit/test_source_digest.cpp @@ -57,7 +57,26 @@ UTest build_output_and_nested_packages_are_not_part_of_it{"SourceDigest/BuildOut t.write(".git/HEAD", "ref"); t.write("member/mcpp.toml", "[package]\n"); t.write("member/src/m.cppm", "export module m;\n"); - expect(eq(t.digest(), a)); + t.write(".mcpp/stamp", "x"); + t.write("compile_commands.json", "[]"); + expect(eq(t.digest(), a)) << "build output, .git, .mcpp, the compile database and a nested package"; +}}; + +// A CONTENT, NOT A TIME (mcpp#974): the tool store and the dependency cache key +// a `path` package by this digest, so an edit and its reversal return to the +// first key, and a copy of the same sources elsewhere -- another checkout, a +// later CI run -- has it as well. +UTest an_edit_and_its_reversal_return_to_the_first_digest{"SourceDigest/AnEditAndItsReversalReturnToTheFirstDigest", [] { + Tree t; + t.write("src/gen.cpp", "int main() { return 41; }\n"); + const auto before = t.digest(); + t.write("src/gen.cpp", "int main() { return 42; } // edited\n"); + expect(ne(t.digest(), before)); + t.write("src/gen.cpp", "int main() { return 41; }\n"); + expect(eq(t.digest(), before)); + Tree copy; + copy.write("src/gen.cpp", "int main() { return 41; }\n"); + expect(eq(copy.digest(), before)) << "the same sources in another directory"; }}; UTest an_added_or_removed_file_changes_it{"SourceDigest/AnAddedOrRemovedFileChangesIt", [] { diff --git a/tests/unit/test_tool_store.cpp b/tests/unit/test_tool_store.cpp index 85c796746..55ee1c780 100644 --- a/tests/unit/test_tool_store.cpp +++ b/tests/unit/test_tool_store.cpp @@ -1,14 +1,6 @@ -// The tool store's stamp for a source tree that has no version of its own. -// -// A `path` tool package changes under an unchanged version, and the store -// keyed on the version alone served a binary built from sources that no longer -// existed (measured 2026-09-08, examples/12). The stamp is what ninja itself -// trusts: every regular file's relative path, size and modification time. The -// tests state the three properties the key needs: the same tree gives the same -// stamp, a change to a source moves it, and a change under the build products -// does not. - -#include +// The tool store's paths and keys. The key of a tool built from a `path` +// package names its tree by the digest of its content (mcpp#974), which +// test_source_digest.cpp covers. import std; import mcpplibs.mtest; @@ -18,83 +10,6 @@ using namespace mcpplibs::mtest; namespace fs = std::filesystem; -namespace { - -struct Tree { - fs::path root; - explicit Tree(std::string_view name) : root(fs::temp_directory_path() / name) { - fs::remove_all(root); - fs::create_directories(root / "src"); - fs::create_directories(root / "target" / "x"); - write("mcpp.toml", "[package]\nname = \"t\"\nversion = \"0.1.0\"\n"); - write("src/gen.cpp", "int main() { return 41; }\n"); - write("target/x/gen.o", "object"); - } - ~Tree() { std::error_code ec; fs::remove_all(root, ec); } - void write(std::string_view rel, std::string_view content) { - std::ofstream(root / rel) << content; - } -}; - -// Two writes in one second have equal mtimes at second resolution on some -// filesystems; a size change is the part of the stamp that cannot be -// masked by that, so every edit below also changes the length. -} // namespace - -UTest the_same_tree_gives_the_same_stamp{"ToolStoreStamp/TheSameTreeGivesTheSameStamp", [] { - Tree t("mcpp_tool_store_stamp_same"); - const auto a = mcpp::build::tool_store::tree_stamp(t.root); - const auto b = mcpp::build::tool_store::tree_stamp(t.root); - expect(eq(a, b)); - expect(eq(a.size(), 16u)) << "sixteen hex digits, the fingerprint module's shape"; -}}; - -UTest an_edit_to_a_source_moves_the_stamp_in_both_directions{"ToolStoreStamp/AnEditToASourceMovesTheStampInBothDirections", [] { - Tree t("mcpp_tool_store_stamp_edit"); - const auto before = mcpp::build::tool_store::tree_stamp(t.root); - t.write("src/gen.cpp", "int main() { return 42; } // edited\n"); - const auto edited = mcpp::build::tool_store::tree_stamp(t.root); - expect(ne(before, edited)); - // The reversal is a new state too: the file's length is back, its - // modification time is not, and a key that only ever grew would miss - // exactly this step. - t.write("src/gen.cpp", "int main() { return 41; }\n"); - const auto reverted = mcpp::build::tool_store::tree_stamp(t.root); - expect(ne(edited, reverted)); -}}; - -UTest build_products_and_the_version_control_directory_do_not_count{"ToolStoreStamp/BuildProductsAndTheVersionControlDirectoryDoNotCount", [] { - Tree t("mcpp_tool_store_stamp_target"); - const auto before = mcpp::build::tool_store::tree_stamp(t.root); - t.write("target/x/gen.o", "a different object of another length"); - fs::create_directories(t.root / ".git"); - t.write(".git/HEAD", "ref: refs/heads/main\n"); - fs::create_directories(t.root / ".mcpp"); - t.write(".mcpp/stamp", "x"); - t.write("compile_commands.json", "[]"); - const auto after = mcpp::build::tool_store::tree_stamp(t.root); - expect(eq(before, after)) << "build products, .git, .mcpp and the compile database are excluded"; -}}; - -// #705: a consumer nested in the tool's tree -- a fixture, an example -- wrote -// its own sources and outputs under the tool's root, so every build of the -// consumer moved the tool's key and rebuilt the tool. A directory with its own -// mcpp.toml is another package and is not this tool's input. A directory -// without one is still an ordinary part of the tree. -UTest a_nested_package_is_not_part_of_the_tree{"ToolStoreStamp/ANestedPackageIsNotPartOfTheTree", [] { - Tree t("mcpp_tool_store_stamp_nested"); - fs::create_directories(t.root / "tests" / "consumer" / "src"); - t.write("tests/consumer/mcpp.toml", "[package]\nname = \"consumer\"\n"); - const auto before = mcpp::build::tool_store::tree_stamp(t.root); - t.write("tests/consumer/src/main.cpp", "int main() { return 0; }\n"); - t.write("tests/consumer/build.mcpp", "// generated by the consumer's prepare\n"); - expect(eq(before, mcpp::build::tool_store::tree_stamp(t.root))); - - fs::create_directories(t.root / "tests" / "data"); - t.write("tests/data/input.txt", "an ordinary input of the tool"); - expect(ne(before, mcpp::build::tool_store::tree_stamp(t.root))); -}}; - // The sub-build scratch (mcpp#641, item 3). Every object path the sub-build // writes is appended to it, so it spends no length on names the entry already // records; it still separates two consumers of one entry, and a re-run by the From 42b7f094af2d9cd5ee7b5a5278cef7f6c87797e3 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 23:59:56 +0900 Subject: [PATCH 21/30] A base's C++ layer and the graph's layers reach a clang driver with no cfg (mcpp#965) Which library is the C++ layer is the target's fact; a driver configuration file beside the compiler is a fact of one installation. resolve_clang_driver took a base package's C++ headers only after it found the payload's cfg, and the isolation words were gated on bypassing that cfg, so on a Windows host, whose LLVM payload has none, the std module of a libstdc++ base (the Hurd and Haiku rows of mcpp-platforms) was precompiled with neither the base's include_dirs nor -nostdinc++. The driver model now names the base's headers and its library whatever the cfg, host_compile_tokens states the clang driver's search whenever it bypasses a cfg or there is none to trust: -nostdinc++ with the base's directories and -stdlib= for a base's C++ layer, and -nostdinc++ and -nostdlibinc for the layers the graph supplies. Only --no-default-config follows the file. GCC keeps its own search. SPEC-002 v1.10 section 2.5, usage chapter 22 in both languages, unit case HostFlags/ABasesCxxLayerReachesADriverWithNoCfg. --- CHANGELOG.md | 8 +++ docs/22-target-side.md | 10 +++- docs/specs/target-side.md | 9 ++- docs/zh/22-target-side.md | 7 ++- modules/toolchain-model/src/linkmodel.cppm | 45 ++++++++++---- src/toolchain/hostflags.cppm | 32 +++++++--- tests/unit/test_hostflags.cpp | 70 ++++++++++++++++++++++ 7 files changed, 156 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170d2ea05..668b32275 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,14 @@ ### Fixed +- On a Windows host the `std` module of a base package that supplies libstdc++ (the Hurd and + Haiku rows of mcpp-platforms) was precompiled with neither the base's `include_dirs` nor + `-nostdinc++`, and stopped at `'bits/stdc++.h' file not found`: the words that close the + clang driver's own search and name the C++ layer's headers were written only when the + toolchain payload had a driver configuration file to bypass, and the Windows LLVM payload has + none. They follow the layers on every host: `-nostdinc++` and the base's directories for a + base's C++ layer, `-stdlib=` naming its library, and `-nostdinc++` and `-nostdlibinc` for + the layers the graph supplies; only `--no-default-config` follows the file (mcpp#965). - A header edit rebuilt nothing on a Visual Studio without English resources: the files a cl.exe compile read came from its `/showIncludes` lines, whose prefix is a string of the installed language (`注意: 包含文件:`), and `VSLANG=1033` cannot make it English (mcpp#970). diff --git a/docs/22-target-side.md b/docs/22-target-side.md index 45077113b..87d5bb674 100644 --- a/docs/22-target-side.md +++ b/docs/22-target-side.md @@ -992,7 +992,8 @@ layer's host locations.** `mcpp.toolchain.hostflags` reads each layer's origin and closes the corresponding driver search: a graph-supplied `c-abi` drops the compiler's own C-library search path (`-nostdlibinc`), and a graph-supplied `c++-abi` drops its C++ search path (`-nostdinc++`) — both independently of -whether the *other* layer is also the graph's. Before this, a package that +whether the *other* layer is also the graph's, and of whether the toolchain +payload has a driver configuration file beside it *(2026.11.19.1+, mcpp#965)*. Before this, a package that happened to compile only because a host header filled a gap the graph did not would build on one machine and fail, differently, on another; the two determined outcomes now are "found in the graph" and "not found", never "found @@ -1321,7 +1322,12 @@ platform's or a provider's `sysroot`), the base answers the layers it states: - the report shows them as `prebuilt`, with the base's `@`; - the C++ headers, the `std` module and the C++ libraries of the link are the base's; the toolchain payload's C++ runtime and its archives, built for the payload's own host, take no - part. A base that states no `std-module` leaves the build without `import std`; + part. A base that states no `std-module` leaves the build without `import std`. Under clang + every unit and the `std` module's precompile carry `-nostdinc++` and the base's + `include_dirs`, with `-stdlib=` naming the layer's library, on every host: whether the + payload has a driver configuration file beside it (the Windows LLVM payload has none) + decides only whether `--no-default-config` is written *(2026.11.19.1+; before, a Windows + host gave a libstdc++ base's `std` module neither, mcpp#965)*; - a package that supplies the `c++-abi` layer instead (a dependency's `provides`, a platform's or a provider's `layers`) takes the base's place for that layer alone: the base's C++ headers leave the compile and its `ldflags` leave the link, so an image has one C++ runtime, while diff --git a/docs/specs/target-side.md b/docs/specs/target-side.md index 4b887cdf4..d8f2c9e65 100644 --- a/docs/specs/target-side.md +++ b/docs/specs/target-side.md @@ -4,8 +4,8 @@ |---|---| | 规范编号 | SPEC-002 | | 标题 | 目标侧模型与能力声明(`mcpp:` 保留命名空间) | -| 状态 | 评审中 v1.8 | -| 最后修改 | 2026-11-10 | +| 状态 | 评审中 v1.10 | +| 最后修改 | 2026-11-19 | | 对应实现 | mcpp >= 2026.8.24.2;§2.2 的对象形式与 §7 mcpp >= 2026.10.26.1 | | 相关设计文档 | `.agents/docs/2026-08-24-target-side-design.md`、`.agents/docs/2026-10-23-target-base-and-execution-host-design.md`、`.agents/docs/2026-10-26-control-contract-runtime-and-ide-ecosystem-design.md` §6 | | 使用文档 | [docs/22 - 目标侧](../zh/22-target-side.md) | @@ -136,6 +136,10 @@ mcpp:<层名>[=<实现名>] - 基座声明 `c++-abi` 且该层由基座供给时,C++ 头文件、`std` 模块与链接的 C++ 库**必须**取自基座;另一包供给该层时, 基座的 C++ 头文件与 `ldflags` **禁止**进入该构建,C 库与编译器运行时仍取自基座(mcpp >= 2026.11.17.1);工具链载荷的 C++ 运行时及为其宿主 构建的归档**禁止**进入该构建。基座未给出 `std-module` 时,该构建没有 `std` 模块。 +- 基座供给 `c++-abi` 时,clang 的每个单元与 `std` 模块的预编译**必须**带 `-nostdinc++` 与基座的 `include_dirs`, + `-stdlib=` 取该层的库名;这与工具链载荷旁有无驱动的配置文件(`<驱动>.cfg`)无关,配置文件只决定是否写出 + `--no-default-config`。依赖图供给 `c-abi` 或 `c++-abi` 时的 `-nostdlibinc` 与 `-nostdinc++` 同此 + (mcpp >= 2026.11.19.1,mcpp#965;此前没有配置文件的载荷,即 Windows 的 LLVM 载荷,两者都不写出)。 - `provides` 是声明(§2.4):测量回答不了 C 库时,`mcpp:c-abi` 回答它。 - driver-internal 的 MinGW-w64 行上,基座包是 clang 的 sysroot;静态链接取基座的归档而非其导入库(SPEC-016 §3)。 @@ -413,3 +417,4 @@ hosted 与否是 C 环境的一个声明:提供 `hosted-standard-library` 的 sy | v1.2 | 2026-10-23 | 随 mcpp 2026.10.24.1:§1.3 `prebuilt` 放宽为被点名、按内容识别的预制目录;§2.4 改为由测量回答、`provides` 声明或核对,标为已实现;规则一增加基座是一个供给者与 c-abi 独占律;规则二增加基座上的编译器运行库与可测量的下限;规则三增加可叠加层;新增 §6 目标侧执行(组合、运行器契约、物理边界)。 | | v1.9 | 2026-11-17 | 随 mcpp 2026.11.17.1(设计 2026-11-16 §5、§7):§6.2 闭包的 `run_files`(schema 2)与 `MCPP_RUNTIME_FILES` 为派生的并集;新增 §6.2.1 工作目录;§2.5 另一包供给 `c++-abi` 时基座的 `ldflags` 不进入链接,包根由清单标明(版本目录带 `mcpp.toml` 即包根)。 | | v1.8 | 2026-11-12 | 随 mcpp 2026.11.12.1:§2.5 基座包只有一种读法(带清单即包根 sysroot,布局只对无清单的目录探测)。 | +| v1.10 | 2026-11-19 | 随 mcpp 2026.11.19.1:§2.5 基座的 C++ 层与依赖图供给的层的隔离词在每个宿主上写出,与载荷旁有无驱动的配置文件无关;`-stdlib=` 取该层的库名(mcpp#965)。 | diff --git a/docs/zh/22-target-side.md b/docs/zh/22-target-side.md index f5663c768..25c51eaae 100644 --- a/docs/zh/22-target-side.md +++ b/docs/zh/22-target-side.md @@ -831,7 +831,8 @@ include_dirs = ["config/picolibc"] `mcpp.toolchain.hostflags` 读取每一层的来源,并关闭对应的编译器隐式搜索: 图供给 `c-abi` 时去掉编译器自带的 C 库搜索路径(`-nostdlibinc`),图供给 `c++-abi` 时去掉它的 C++ 搜索路径(`-nostdinc++`)——两者各自独立判断, -不取决于另一层是否也来自图。在此之前,一个只因为宿主头文件恰好补上了某个 +不取决于另一层是否也来自图,也不取决于工具链载荷旁有没有驱动的配置文件 +*(2026.11.19.1 起,mcpp#965)*。在此之前,一个只因为宿主头文件恰好补上了某个 缺口才能编译的包,会在一台机器上构建成功、在另一台机器上以不同方式失败; 现在确定的结果只有两种——「在图里找到」与「没找到」,不再有「用了这台机器上 恰好装着的那份 SDK」。包应当通过下面的层谓词来适配,而不是依赖宿主机器 @@ -1109,7 +1110,9 @@ std-module-flags = ["-D__fenv_static="] - 报告把这些层标为 `prebuilt`,并附基座的 `<名>@<版本>`; - C++ 头文件、`std` 模块与链接的 C++ 库取自基座;工具链载荷为其自身宿主构建的 C++ 运行时及其归档不参与。基座未给出 - `std-module` 时,该构建没有 `import std`; + `std-module` 时,该构建没有 `import std`。clang 下每个单元与 `std` 模块的预编译在每个宿主上都带 `-nostdinc++` 与基座的 + `include_dirs`,`-stdlib=` 取该层的库名;载荷旁有没有驱动的配置文件(Windows 的 LLVM 载荷没有)只决定是否写出 + `--no-default-config` *(2026.11.19.1 起;此前 Windows 宿主上 libstdc++ 基座的 `std` 模块两者都得不到,mcpp#965)*; - 另一个包供给 `c++-abi` 层时(依赖的 `provides`、平台或提供者的 `layers`),它只在该层取代基座:基座的 C++ 头文件 不进入编译,其 `ldflags` 不进入链接,一个映像只有一个 C++ 运行时;C 库与编译器运行时仍取自基座(2026.11.17.1+)。 系统自带的 C++ 库不带模块源码时(NetBSD 的 libstdc++),以这种方式得到 `import std`:在其 C 库之上构建的 libc++; diff --git a/modules/toolchain-model/src/linkmodel.cppm b/modules/toolchain-model/src/linkmodel.cppm index 945ccb0bc..2eaf166fd 100644 --- a/modules/toolchain-model/src/linkmodel.cppm +++ b/modules/toolchain-model/src/linkmodel.cppm @@ -255,8 +255,11 @@ struct ClangDriverModel { bool hasCfg = false; // sibling .cfg exists std::filesystem::path cfgPath; std::filesystem::path llvmRoot; // /../ - std::vector cxxIncludes; // libc++ header dirs + std::vector cxxIncludes; // the C++ layer's header dirs std::vector libDirs; // libc++/compiler-rt libs + // The library those headers are, as `-stdlib=` names it: the payload's + // libc++, or a base package's library (empty: a name clang has no word for). + std::string stdlib = "libc++"; // "--no-default-config" "-nostdinc++" "-isystem<...>" (compile side), as // argv tokens. -stdlib=libc++ is deliberately left to callers: compile @@ -267,8 +270,18 @@ struct ClangDriverModel { // the flag would invalidate every user's std BMIs for no behavioural gain. std::vector compile_tokens(const PathEscape& esc, bool stdlibSelect = false) const { - std::vector out{"--no-default-config", "-nostdinc++"}; - if (stdlibSelect) out.push_back("-stdlib=libc++"); + std::vector out{"--no-default-config"}; + for (auto& t : cxx_tokens(esc, stdlibSelect)) out.push_back(std::move(t)); + return out; + } + + // The C++ layer's half of the above, without the cfg bypass: what a driver + // with no cfg beside it is told when the C++ layer is not its own payload's + // (a base package's, mcpp#965). The library selected is the layer's. + std::vector cxx_tokens(const PathEscape& esc, + bool stdlibSelect = false) const { + std::vector out{"-nostdinc++"}; + if (stdlibSelect && !stdlib.empty()) out.push_back("-stdlib=" + stdlib); for (auto& inc : cxxIncludes) out.push_back("-isystem" + esc(inc)); return out; } @@ -381,21 +394,31 @@ ClangDriverModel resolve_clang_driver(const Toolchain& tc) { const auto root = tc.binaryPath.parent_path().parent_path(); dm.hasCfg = std::filesystem::exists(root / "include" / "c++" / "v1", ec); } - if (!dm.hasCfg) return dm; - dm.llvmRoot = tc.binaryPath.parent_path().parent_path(); - // A TARGET WHOSE C LIBRARY A PLATFORM DESCRIBES TAKES NONE OF THE - // PAYLOAD'S OWN TARGET SIDE (E1): its libc++ headers and runtime - // directories are built for the payload's host. Measured on a FreeBSD row - // a platform defined (2026-11-10): the payload's `include/c++/v1` reached C - // units (`'__config_site' file not found`), and its `lib/` - // reached the link and the artifact's RPATH. + if (dm.hasCfg) dm.llvmRoot = tc.binaryPath.parent_path().parent_path(); // A BASE PACKAGE'S C++ HEADERS STAND WHERE THE PAYLOAD'S WOULD (E5), so // every reader of this model -- a unit, the std module, a build program -- // compiles against the library the base links. + // + // WHETHER OR NOT THE PAYLOAD CARRIES A CFG (mcpp#965). Which library is the + // C++ layer is the target's fact; a cfg beside the driver is a fact of one + // installation. The Windows LLVM payload carries none, and this answer was + // taken only after the cfg was found: on a Windows host the std module of a + // libstdc++ base (the Hurd and Haiku rows of mcpp-platforms) was + // precompiled with neither the base's directories nor `-nostdinc++`, and + // stopped at `'bits/stdc++.h' file not found`. if (!tc.base.layers.cxx.empty()) { dm.cxxIncludes = tc.base.layers.cxxIncludes; + const auto& layer = tc.base.layers.cxx; + dm.stdlib = layer == "libc++" || layer == "libstdc++" ? layer : std::string{}; return dm; } + if (!dm.hasCfg) return dm; + // A TARGET WHOSE C LIBRARY A PLATFORM DESCRIBES TAKES NONE OF THE + // PAYLOAD'S OWN TARGET SIDE (E1): its libc++ headers and runtime + // directories are built for the payload's host. Measured on a FreeBSD row + // a platform defined (2026-11-10): the payload's `include/c++/v1` reached C + // units (`'__config_site' file not found`), and its `lib/` + // reached the link and the artifact's RPATH. if (auto t = mcpp::toolchain::triple::parse(tc.targetTriple); t && t->c_library_by_platform()) return dm; auto libcxxInclude = dm.llvmRoot / "include" / "c++" / "v1"; diff --git a/src/toolchain/hostflags.cppm b/src/toolchain/hostflags.cppm index 254eee9bc..81330964d 100644 --- a/src/toolchain/hostflags.cppm +++ b/src/toolchain/hostflags.cppm @@ -384,6 +384,13 @@ std::vector host_compile_tokens(const Toolchain& tc, // below), which is why this suppresses the two blocks rather than // returning early. const bool trustCfg = !bypassCfg && dm.hasCfg; + // WHETHER THIS COMMAND STATES THE CLANG DRIVER'S SEARCH ITSELF: it bypasses + // the cfg, or there is no cfg to trust (mcpp#965). Which libraries are the + // C and C++ layers is the target's fact, and a cfg beside the driver is a + // fact of one installation: the Windows LLVM payload carries none, and the + // isolation words below were withheld on every Windows host. GCC is not a + // clang driver and keeps its own search, as before. + const bool statesSearch = bypassCfg || (lm.clangDriver && !dm.hasCfg); // AND NOT WHEN THE TARGET SIDE COMES FROM THE GRAPH — the compile-side // counterpart of the replacement `flags.cppm` makes on the link line. @@ -441,6 +448,16 @@ std::vector host_compile_tokens(const Toolchain& tc, if (auto target = triple::parse(tc.targetTriple); target && target->arch == "aarch64") out.emplace_back("--rtlib=compiler-rt"); + } else if (statesSearch && !bypassCfg && !graphSuppliesTarget && cxxFromPayload + && !tc.base.layers.cxx.empty()) { + // A BASE'S C++ LAYER UNDER A DRIVER WITH NO CFG (mcpp#965): the same + // words as the branch above, less the bypass of a file that is not + // there. On a Windows host the std module of a libstdc++ base (the + // Hurd and Haiku rows of mcpp-platforms) was precompiled with neither + // the base's directories nor `-nostdinc++`, and stopped at + // `'bits/stdc++.h' file not found`. + for (auto& t : dm.cxx_tokens(esc, opt.clangStdlibSelect)) + out.push_back(t); } else if (bypassCfg) { // THE BYPASS IS NOT PART OF THE PAYLOAD'S HEADER SET, AND IT WAS // BEING SUPPRESSED WITH IT. @@ -490,14 +507,13 @@ std::vector host_compile_tokens(const Toolchain& tc, // GCC has no equivalent single flag — the shape would be `-nostdinc` // plus `-isystem ` and `<…/include-fixed>`, // re-adding exactly the two directories GCC's own C-library search - // already contributes beside the sysroot. Not implemented: `dm.hasCfg` - // is false for GCC (resolve_clang_driver only looks for a `.cfg` - // beside a Clang binary), so `bypassCfg` already withholds this whole - // block from that family — GCC stays exactly as it was before #662, - // unisolated, on every target row this codebase's own table pairs it - // with today (test_hostflags.cpp, + // already contributes beside the sysroot. Not implemented: `statesSearch` + // holds only for a clang driver (a cfg bypass, or a clang with no cfg), + // so this whole block is withheld from that family — GCC stays exactly as + // it was before #662, unisolated, on every target row this codebase's own + // table pairs it with today (test_hostflags.cpp, // GccEmitsNeitherIsolationTokenRegardlessOfGraphOrigin). - if (bypassCfg && graphSuppliesTarget) out.push_back("-nostdlibinc"); + if (statesSearch && graphSuppliesTarget) out.push_back("-nostdlibinc"); // THE CONDITION USED TO BE `!graphSuppliesTarget && !cxxFromPayload`, // WHICH IS RIGHT ABOUT `cxxFromPayload` AND WRONG TO ASK ABOUT THE C @@ -516,7 +532,7 @@ std::vector host_compile_tokens(const Toolchain& tc, // these headers", which is true for both `cxxFromGraph` (a package does) // and `appleSdkCxxHeaders` (the SDK does) — the two cases this branch // already told apart below by whether `opt.cxxFromGraph` is set. - if (bypassCfg && !cxxFromPayload) { + if (statesSearch && !cxxFromPayload) { // The driver's own C++ search contributes nothing: beside the compiler // it finds the payload's libc++, and clang's Darwin driver prefers that // copy to the SDK's whenever it exists. What replaces it is either the diff --git a/tests/unit/test_hostflags.cpp b/tests/unit/test_hostflags.cpp index efe6b39ba..d52dca926 100644 --- a/tests/unit/test_hostflags.cpp +++ b/tests/unit/test_hostflags.cpp @@ -750,6 +750,76 @@ UTest the_cxx_layer_decides_whose_libcxx_headers_are_emitted{"HostFlags/TheCxxLa }}; +// A base package's C++ layer reaches a clang driver with no cfg beside it +// (mcpp#965): the Windows LLVM payload carries none, and the std module of a +// libstdc++ base (the Hurd and Haiku rows of mcpp-platforms) was precompiled +// there with neither the base's directories nor `-nostdinc++`. The words are +// the layer's on every host; only the cfg bypass follows the file. +UTest a_bases_cxx_layer_reaches_a_driver_with_no_cfg{"HostFlags/ABasesCxxLayerReachesADriverWithNoCfg", [] { + namespace fs = std::filesystem; + const auto root = fs::temp_directory_path() / "mcpp_hostflags_nocfg_fixture"; + fs::remove_all(root); + fs::create_directories(root / "bin"); + { std::ofstream(root / "bin" / "clang++"); } + struct Cleanup { + fs::path p; + ~Cleanup() { std::error_code ec; fs::remove_all(p, ec); } + } cleanup{root}; + const auto has = [](const std::vector& v, std::string_view f) { + return std::ranges::find(v, f) != v.end(); + }; + + auto tc = tc_for(CompilerId::Clang); + tc.binaryPath = root / "bin" / "clang++"; + const auto inc = fs::path("/base/usr/include/c++/15"); + const auto machine = fs::path("/base/usr/include/x86_64-gnu/c++/15"); + tc.base.layers.cxx = "libstdc++"; + tc.base.layers.cxxIncludes = {inc, machine}; + expect(fatal(!mcpp::toolchain::resolve_clang_driver(tc).hasCfg)) << "the fixture has a cfg"; + + HostFlagOptions std_module; + std_module.cfgBypass = HostFlagOptions::CfgBypass::Always; + std_module.cAbiPrebuilt = true; + std_module.clangStdlibSelect = true; + const auto a = mcpp::toolchain::host_compile_tokens(tc, std_module, mcpp::toolchain::no_escape); + expect(bool(has(a, "-nostdinc++"))); + expect(bool(has(a, "-stdlib=libstdc++"))) << "the library selected is the layer's"; + expect(!has(a, "-stdlib=libc++")); + expect(bool(has(a, "-isystem" + inc.string()))); + expect(bool(has(a, "-isystem" + machine.string()))); + expect(!has(a, "--no-default-config")) << "a cfg that is not there is not bypassed"; + + // With a cfg, the same words after its bypass, as on Linux. + { std::ofstream(root / "bin" / "clang++.cfg"); } + const auto b = mcpp::toolchain::host_compile_tokens(tc, std_module, mcpp::toolchain::no_escape); + expect(bool(has(b, "--no-default-config"))); + expect(bool(has(b, "-nostdinc++"))); + expect(bool(has(b, "-stdlib=libstdc++"))); + expect(bool(has(b, "-isystem" + machine.string()))); + fs::remove(root / "bin" / "clang++.cfg"); + + // A graph-supplied layer closes the driver's own search with no cfg too: + // `-nostdinc++` for the C++ layer, `-nostdlibinc` for the C library. + auto graphTc = tc_for(CompilerId::Clang); + graphTc.binaryPath = root / "bin" / "clang++"; + HostFlagOptions graph; + graph.cfgBypass = HostFlagOptions::CfgBypass::Always; + graph.cAbiPrebuilt = false; + graph.cxxFromGraph = true; + const auto c = mcpp::toolchain::host_compile_tokens(graphTc, graph, mcpp::toolchain::no_escape); + expect(bool(has(c, "-nostdinc++"))); + expect(bool(has(c, "-nostdlibinc"))); + expect(!has(c, "--no-default-config")); + + // GCC keeps its own search, base or no base. + auto gcc = tc_for(CompilerId::GCC); + gcc.binaryPath = root / "bin" / "g++"; + gcc.base.layers = tc.base.layers; + const auto d = mcpp::toolchain::host_compile_tokens(gcc, std_module, mcpp::toolchain::no_escape); + expect(!has(d, "-nostdinc++")); + expect(!has(d, "-nostdlibinc")); +}}; + // The floating macros an Apple SDK leaves to once modules are on // (apple_float_macro_words). Stated for clang on an Apple target and for // nothing else, as the SDK's own spellings, and quotable by every reader. From fe724813c1217f52d3654b8fc74e231dd06af555 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Fri, 20 Nov 2026 00:14:41 +0900 Subject: [PATCH 22/30] A row the platform column misses is an index miss, refreshed as a dependency's is (mcpp#967) The platform column was read from the local copy of the index alone, so a row the index listed after that copy was taken was refused as an unknown target until `mcpp index update`. A miss for a row that is refused without an answer (one the engine does not know, or a planned one) now goes through decide_for_miss and apply -- `--offline`, `[index] auto_refresh`, the debounce, one sync per command -- and the column is scanned again after a refresh, the scan cache dropped. Decided once per row per process; the unknown-target and planned refusals say whether the index was refreshed, and why not. A row the table serves without a platform does not refresh. The column scan is split into helpers (one descriptor's claims, the roots, the cached scan), which also brings its loops to two levels. SPEC-013 v1.14 section 14 item 6, usage chapters 05 and 35 in both languages, e2e index::refresh/967; openkal::platforms/1106 leg D runs offline, where a refresh would bring the shared index whose openkal-platform lists the planned row. --- CHANGELOG.md | 6 + docs/05-dependencies.md | 2 +- docs/35-platforms.md | 9 ++ docs/specs/platforms.md | 8 +- docs/zh/05-dependencies.md | 2 +- docs/zh/35-platforms.md | 5 + src/build/prepare/platform.cpp | 260 +++++++++++++++++++++----------- src/build/prepare/state.cppm | 7 +- src/build/prepare/toolchain.cpp | 7 +- tests/e2e/index/refresh.cpp | 37 ++++- tests/e2e/openkal/platforms.cpp | 13 +- 11 files changed, 255 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 668b32275..583ef3f09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,12 @@ ### Fixed +- A target row the package index's platform column listed after the local copy of the index + was taken was refused as an unknown target until `mcpp index update`: the miss did not reach + the refresh policy. A row the local copy does not list, and that is refused without it (one + the engine does not know, or a `planned` one), now refreshes the index as a dependency's miss + does (`--offline`, `[index] auto_refresh`, the debounce, one sync per command) and is looked + up again; the refusal says whether the index was refreshed, and why not (mcpp#967). - On a Windows host the `std` module of a base package that supplies libstdc++ (the Hurd and Haiku rows of mcpp-platforms) was precompiled with neither the base's `include_dirs` nor `-nostdinc++`, and stopped at `'bits/stdc++.h' file not found`: the words that close the diff --git a/docs/05-dependencies.md b/docs/05-dependencies.md index b0cd68e24..9fe2520d3 100644 --- a/docs/05-dependencies.md +++ b/docs/05-dependencies.md @@ -464,7 +464,7 @@ Controls, in order of precedence: |---|---| | `--offline` (any command) | Never touch the network — no index refresh, no downloads, no toolchain auto-install, no `git ls-remote`/`clone`. Anything already installed still builds, including git deps whose commit is in `mcpp.lock` and whose clone is cached | | `MCPP_OFFLINE=1` | Same, for a whole shell session or CI job | -| `[index] auto_refresh = false` in `~/.mcpp/config.toml` | Never refresh an index implicitly: not on a dependency miss, not before installing a package the local index lacks, and not for the first sync of a project's custom index (that build stops and names `mcpp index update`). Downloads still work | +| `[index] auto_refresh = false` in `~/.mcpp/config.toml` | Never refresh an index implicitly: not on a dependency miss, not on a target row the index's platform column does not list (2026.11.19.1+), not before installing a package the local index lacks, and not for the first sync of a project's custom index (that build stops and names `mcpp index update`). Downloads still work | `MCPP_NO_AUTO_INSTALL=1` remains accepted as the older, narrower spelling of `--offline` (it gates only toolchain auto-install). diff --git a/docs/35-platforms.md b/docs/35-platforms.md index 05c401fce..b573c6981 100644 --- a/docs/35-platforms.md +++ b/docs/35-platforms.md @@ -248,6 +248,15 @@ project's own `[target.] platform` still comes first, and so do the project's own `[indices]` before the shared ones. The list is generated from the platform's manifest by the index's tools; engines before 2026.11.17.1 ignore it. +A row the local copy of the index does not list, and that would otherwise be +refused (a row the engine does not know, or a `planned` one), is an index miss +like a dependency's *(2026.11.19.1+, mcpp#967)*: the index is refreshed once +under the same controls (`--offline`, `[index] auto_refresh`, the debounce, at +most one sync per command, [05](05-dependencies.md)) and the column is read +again. A refusal that follows says whether the index was refreshed, and why not. +Before, a row listed after the local copy was taken was refused as an unknown +target until `mcpp index update`. + ## Writing a platform package A platform package is a package with a `[platform]` section. openkal-platform, diff --git a/docs/specs/platforms.md b/docs/specs/platforms.md index a3bbb72ab..a647513f0 100644 --- a/docs/specs/platforms.md +++ b/docs/specs/platforms.md @@ -4,7 +4,7 @@ |---|---| | 规范编号 | SPEC-013 | | 标题 | 平台:一类目标的可复用配置,其声明、选择、合并、对依赖图的补全、链接与封装、运行器与载体 | -| 状态 | 评审中 v1.13 | +| 状态 | 评审中 v1.14 | | 版本 | 1.13 | | 最后修改 | 2026-11-19 | | 对应实现 | mcpp >= 2026.10.27.1;§10 的执行宿主与 §11 的载体架构 mcpp >= 2026.10.29.1;`runner.hosts`、载体列表、`descriptor_keys` 与 `universal` 的弃用 mcpp >= 2026.10.30.1;§7.5 版本组参与解析、§14 平台系列与 §15.1 系列的锁 mcpp >= 2026.11.7.1;§14 内置平台为确切版本与提示 mcpp >= 2026.11.7.2;v1.7 条款 mcpp >= 2026.11.10.1;v1.10 条款 mcpp >= 2026.11.16.1;v1.11 条款(§14 索引的平台列) mcpp >= 2026.11.17.1;逐条标注 | @@ -493,6 +493,11 @@ package = { 一个平台包认领。列入的行:`[platform] targets`、`[[platform.define]] triple`、`[[platform.provider]] serves`, 其中的模式(`*-linux-musl`)代表它在平台自己的行与引擎内置表的行中匹配到的行;`[platform.hosted] targets` 中只列入引擎不认识或登记为 `planned` 的行,所以索引从不为宿主自己的行指定平台。早于本版本的引擎不读这个字段。 +6. 本地的索引副本没有认领一行、而没有答案时该行被拒绝(引擎不认识的行、`planned` 行),这是一次索引未命中: + 引擎**必须**按依赖未命中的同一策略决定是否刷新(`--offline`、`[index] auto_refresh`、去抖、每次运行至多一次), + 刷新后**必须**重新扫描,扫描的缓存随之作废;仍未命中时拒绝,消息写出索引是否已刷新及未刷新的原因。内置表持有、 + 没有平台也能构建的行(如以自己的 sysroot 构建的 `x86_64-freebsd`)未命中时不刷新。**已实现**(mcpp >= 2026.11.19.1, + mcpp#967;此前在本地副本取得之后才列入索引的行被拒绝为未知目标,直到 `mcpp index update`)。 每条命令至多扫描一次索引的条目,且只在内置表没有答案时扫描(约 400 个条目时整个拒绝过程 0.05 s)。**已实现** @@ -636,3 +641,4 @@ openkal-platform 的 `targets` 因此不含 WebAssembly。Web 经 `wasm32-emscri | 1.12 | 2026-11-17 | mcpp 2026.11.17.1:§3.1 尚无行定义的三元组按字面成分求值选择器(e2e 1040);§10 运行器的陈述与许可分属两层,覆盖层的 `runner = true`/`false`(e2e 1092)。 | | 1.12 | 2026-11-17 | mcpp 2026.11.17.2(设计 2026-11-17 §5):§3.1 平台选定之前可用的谓词,以层选择平台时的警告;§10 `hosts` 参与运行器的采用;§14 目标表的平台列为 0.29.1 | | 1.13 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):§7.5 版本组的约束进入合并,结果与请求到达的先后无关;拒绝列出每一个请求者。 | +| 1.14 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#967):§14 索引的平台列第 6 条,一行的未命中按依赖未命中的策略刷新索引并重新扫描,拒绝写出是否已刷新。 | diff --git a/docs/zh/05-dependencies.md b/docs/zh/05-dependencies.md index 619ae0d9a..9d2c7e8d4 100644 --- a/docs/zh/05-dependencies.md +++ b/docs/zh/05-dependencies.md @@ -389,7 +389,7 @@ mcpp index status # local state: state, age and revision |---|---| | `--offline`(任意命令) | 完全不碰网络 —— 不刷索引、不下载、不自动安装工具链,也不发出 `git ls-remote`/`clone`。已安装的东西照常构建,包括 commit 已在 `mcpp.lock`、克隆已在缓存里的 git 依赖 | | `MCPP_OFFLINE=1` | 同上,作用于整个 shell 会话或一次 CI job | -| `~/.mcpp/config.toml` 中的 `[index] auto_refresh = false` | 永不隐式刷新索引:依赖未命中时不刷新,安装本地索引没有的包之前不刷新,工程自定义索引的首次同步也不做(该次构建停止,并指出 `mcpp index update`)。下载仍然可用 | +| `~/.mcpp/config.toml` 中的 `[index] auto_refresh = false` | 永不隐式刷新索引:依赖未命中时不刷新,索引的平台列未列出目标行时不刷新(2026.11.19.1 起),安装本地索引没有的包之前不刷新,工程自定义索引的首次同步也不做(该次构建停止,并指出 `mcpp index update`)。下载仍然可用 | `MCPP_NO_AUTO_INSTALL=1` 作为 `--offline` 更早、更窄的拼法,仍然被接受 (它只约束工具链的自动安装)。 diff --git a/docs/zh/35-platforms.md b/docs/zh/35-platforms.md index 9591c3e84..e3c16f4d4 100644 --- a/docs/zh/35-platforms.md +++ b/docs/zh/35-platforms.md @@ -201,6 +201,11 @@ package = { 版本的选择、锁定与更新同上,报告的来源为 `index`。工程自己的 `[target.] platform` 仍然在前,工程自己的 `[indices]` 先于共享的索引。这份列表由索引的工具从平台的清单生成;2026.11.17.1 之前的引擎忽略它。 +本地索引副本没有列出、否则会被拒绝的行(引擎不认识的行,或 `planned` 行),与依赖一样是一次索引未命中 +*(2026.11.19.1 起,mcpp#967)*:按同样的控制(`--offline`、`[index] auto_refresh`、去抖、每条命令至多一次同步, +见 [05](05-dependencies.md))刷新一次索引,再读平台列。随后的拒绝写出索引是否已刷新及未刷新的原因。此前,在本地 +副本取得之后才列入的行被拒绝为未知目标,直到 `mcpp index update`。 + ## 编写平台包 平台包是带 `[platform]` 段的包。openkal-platform 的节选: diff --git a/src/build/prepare/platform.cpp b/src/build/prepare/platform.cpp index ed97a77a1..2482d3224 100644 --- a/src/build/prepare/platform.cpp +++ b/src/build/prepare/platform.cpp @@ -26,6 +26,7 @@ import mcpp.invocation; import mcpp.manifest; import mcpp.platform; import mcpp.provider; // the synced index's roots (the platform column, design 2026-11-16 §6.2) +import mcpp.pm.refresh_policy; // a miss of the platform column refreshes as a dependency's does (mcpp#967) import mcpp.pm.compat; import mcpp.project; import mcpp.targetside; @@ -351,7 +352,7 @@ void define_early_platform(PrepareState& state) { || mcpp::toolchain::triple::find_known_target(*parsed))) return; if (!chosen && chosenText.empty()) - if (auto column = index_platform_for(state, raw); !column.empty()) + if (auto column = index_platform_for(state, raw, true); !column.empty()) if (auto ref = mp::parse_ref_text(column)) chosen = *ref; if (!chosen) return; std::vector seg; @@ -376,23 +377,85 @@ void define_early_platform(PrepareState& state) { state.provisionalTarget = true; } -std::string index_platform_for(const PrepareState& state, std::string_view triple) { - // One scan of a set of index roots per process: the column is consulted - // only for a row the built-in table does not answer, and the answer does - // not change while one command runs. The project's own indices ([indices] - // of mcpp.toml) rank before the shared ones; one rank's claims are - // compared. The roots are found and the scan runs with no lock held: the - // configuration and the provider may wait on the home's lock, which a - // leg prepared beside this one may hold while it reaches this function. - struct Claim { - int rank; - std::string package; - }; - using Claims = std::map>; - auto cfg = state.get_cfg(false); - if (!cfg) return {}; - // (root, rank, the namespace of a descriptor that names none) - std::vector> roots; +namespace { + +// One claim of the platform column: the package whose index entry lists a row. +struct ColumnClaim { + int rank; + std::string package; +}; +using ColumnClaims = std::map>; +// (root, rank, the namespace of a descriptor that names none) +using ColumnRoot = std::tuple; + +// ` = ` at the start of a line or after `{` or `,`: the first such key is +// the package table's own (its keys precede its nested tables). +std::size_t lua_field(std::string_view lua, std::string_view key) { + for (std::size_t pos = 0; (pos = lua.find(key, pos)) != std::string_view::npos; pos += key.size()) { + std::size_t b = pos; + while (b > 0 && (lua[b - 1] == ' ' || lua[b - 1] == '\t')) --b; + std::size_t e = pos + key.size(); + while (e < lua.size() && (lua[e] == ' ' || lua[e] == '\t')) ++e; + if ((b == 0 || lua[b - 1] == '\n' || lua[b - 1] == ',' || lua[b - 1] == '{') && e < lua.size() + && lua[e] == '=' && (e + 1 >= lua.size() || lua[e + 1] != '=')) + return e + 1; + } + return std::string_view::npos; +} + +std::string lua_quoted(std::string_view lua, std::size_t at) { + std::string out; + const auto q1 = lua.find('"', at), nl = lua.find('\n', at); + if (q1 == std::string_view::npos || (nl != std::string_view::npos && q1 > nl)) return out; + const auto q2 = lua.find('"', q1 + 1); + if (q2 != std::string_view::npos) out = std::string(lua.substr(q1 + 1, q2 - q1 - 1)); + return out; +} + +// The rows one descriptor's `platform_rows` lists, claimed for its package. +void claim_rows(std::string_view lua, int rank, const std::string& defaultNs, ColumnClaims& out) { + const auto rows = lua_field(lua, "platform_rows"); + if (rows == std::string_view::npos) return; + const auto open = lua.find('{', rows), close = lua.find('}', rows); + if (open == std::string::npos || close == std::string::npos || close < open) return; + const auto nameAt = lua_field(lua, "name"), nsAt = lua_field(lua, "namespace"); + if (nameAt == std::string_view::npos) return; + const auto ns = nsAt == std::string_view::npos ? defaultNs : lua_quoted(lua, nsAt); + auto name = lua_quoted(lua, nameAt); + // A descriptor may spell its name qualified (`acme.plat`). + if (name.starts_with(ns + ".")) name.erase(0, ns.size() + 1); + if (ns.empty() || name.empty()) return; + const auto pkg = ns + "." + name; + for (std::size_t p = open; p < close; ++p) { + if (lua[p] != '"') continue; + const auto q = lua.find('"', p + 1); + if (q == std::string::npos || q > close) break; + auto& who = out[std::string(lua.substr(p + 1, q - p - 1))]; + if (std::ranges::none_of(who, [&](auto const& c) { return c.package == pkg; })) + who.push_back({rank, pkg}); + p = q; + } +} + +ColumnClaims scan_column(const std::vector& roots) { + ColumnClaims built; + for (auto const& [root, rank, defaultNs] : roots) { + std::error_code ec; + const auto pkgs = root / "pkgs"; + if (!fs::is_directory(pkgs, ec)) continue; + for (auto it = fs::recursive_directory_iterator(pkgs, ec); + !ec && it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (it->path().extension() != ".lua") continue; + std::ifstream is(it->path(), std::ios::binary); + claim_rows(std::string{std::istreambuf_iterator(is), {}}, rank, defaultNs, built); + } + } + return built; +} + +// The project's own indices ([indices] of mcpp.toml) rank before the shared ones. +std::vector column_roots(const PrepareState& state, const mcpp::config::GlobalConfig& cfg) { + std::vector roots; const auto add = [&](fs::path dir, int rank, std::string ns) { if (std::ranges::none_of(roots, [&](auto const& r) { return std::get<0>(r) == dir; })) roots.emplace_back(std::move(dir), rank, std::move(ns)); @@ -407,79 +470,89 @@ std::string index_platform_for(const PrepareState& state, std::string_view tripl for (auto const& dir : mcpp::provider::layout().index_roots_of(data.parent_path())) add(dir, 0, dir.filename().string()); } - for (auto const& dir : mcpp::provider::open(mcpp::config::make_xlings_env(**cfg))->index_roots()) + for (auto const& dir : mcpp::provider::open(mcpp::config::make_xlings_env(cfg))->index_roots()) add(dir, 1, std::string(mcpp::pm::kDefaultNamespace)); - std::string rootsKey; - for (auto const& [root, rank, defaultNs] : roots) rootsKey += std::format("{}\t{}\t{}\n", root.string(), rank, defaultNs); - static std::mutex lock; - static std::map> scanned; - std::shared_ptr claims; + return roots; +} + +std::mutex g_columnLock; +std::map> g_columnScans; +// A row the column missed, with what its refusal adds about the index. +std::map g_columnMisses; + +std::shared_ptr column_claims(const PrepareState& state, const mcpp::config::GlobalConfig& cfg) { + // The roots are found and the scan runs with no lock held: the + // configuration and the provider may wait on the home's lock, which a leg + // prepared beside this one may hold while it reaches this function. + const auto roots = column_roots(state, cfg); + std::string key; + for (auto const& [root, rank, defaultNs] : roots) key += std::format("{}\t{}\t{}\n", root.string(), rank, defaultNs); { - std::scoped_lock guard(lock); - if (auto found = scanned.find(rootsKey); found != scanned.end()) claims = found->second; + std::scoped_lock guard(g_columnLock); + if (auto found = g_columnScans.find(key); found != g_columnScans.end()) return found->second; } - if (!claims) { - // ` = ` at the start of a line or after `{` or `,`: the first such - // key is the package table's own (its keys precede its nested tables). - auto field = [](std::string_view lua, std::string_view key) -> std::size_t { - for (std::size_t pos = 0; (pos = lua.find(key, pos)) != std::string_view::npos; pos += key.size()) { - std::size_t b = pos; - while (b > 0 && (lua[b - 1] == ' ' || lua[b - 1] == '\t')) --b; - std::size_t e = pos + key.size(); - while (e < lua.size() && (lua[e] == ' ' || lua[e] == '\t')) ++e; - if ((b == 0 || lua[b - 1] == '\n' || lua[b - 1] == ',' || lua[b - 1] == '{') && e < lua.size() - && lua[e] == '=' && (e + 1 >= lua.size() || lua[e + 1] != '=')) - return e + 1; - } - return std::string_view::npos; - }; - auto quoted = [](std::string_view lua, std::size_t at) { - std::string out; - const auto q1 = lua.find('"', at), nl = lua.find('\n', at); - if (q1 == std::string_view::npos || (nl != std::string_view::npos && q1 > nl)) return out; - const auto q2 = lua.find('"', q1 + 1); - if (q2 != std::string_view::npos) out = std::string(lua.substr(q1 + 1, q2 - q1 - 1)); - return out; - }; - auto built = std::make_shared(); - for (auto const& [root, rank, defaultNs] : roots) { - std::error_code ec; - const auto pkgs = root / "pkgs"; - if (!fs::is_directory(pkgs, ec)) continue; - for (auto it = fs::recursive_directory_iterator(pkgs, ec); - !ec && it != fs::recursive_directory_iterator(); it.increment(ec)) { - if (it->path().extension() != ".lua") continue; - std::ifstream is(it->path(), std::ios::binary); - const std::string lua{std::istreambuf_iterator(is), {}}; - const auto rows = field(lua, "platform_rows"); - if (rows == std::string_view::npos) continue; - const auto open = lua.find('{', rows), close = lua.find('}', rows); - if (open == std::string::npos || close == std::string::npos || close < open) continue; - const auto nameAt = field(lua, "name"), nsAt = field(lua, "namespace"); - if (nameAt == std::string_view::npos) continue; - const auto ns = nsAt == std::string_view::npos ? defaultNs : quoted(lua, nsAt); - auto name = quoted(lua, nameAt); - // A descriptor may spell its name qualified (`acme.plat`). - if (name.starts_with(ns + ".")) name.erase(0, ns.size() + 1); - if (ns.empty() || name.empty()) continue; - const auto pkg = ns + "." + name; - for (std::size_t p = open; p < close; ++p) { - if (lua[p] != '"') continue; - const auto q = lua.find('"', p + 1); - if (q == std::string::npos || q > close) break; - auto& who = (*built)[lua.substr(p + 1, q - p - 1)]; - if (std::ranges::none_of(who, [&](auto const& c) { return c.package == pkg; })) - who.push_back({rank, pkg}); - p = q; - } + auto built = std::make_shared(scan_column(roots)); + std::scoped_lock guard(g_columnLock); + return g_columnScans.try_emplace(key, std::move(built)).first->second; +} + +} // namespace + +std::string index_platform_miss_note(std::string_view triple) { + std::scoped_lock guard(g_columnLock); + auto it = g_columnMisses.find(std::string(triple)); + return it == g_columnMisses.end() ? std::string{} : it->second; +} + +std::string index_platform_for(const PrepareState& state, std::string_view triple, bool refreshOnMiss) { + // One scan of a set of index roots per process: the column is consulted + // only for a row the built-in table does not answer, and the answer does + // not change while one command runs -- unless a miss refreshes the index. + // One rank's claims are compared. + auto cfg = state.get_cfg(false); + if (!cfg) return {}; + auto claims = column_claims(state, **cfg); + auto it = claims->find(std::string(triple)); + // A ROW THE LOCAL COPY DOES NOT LIST IS A MISS LIKE A DEPENDENCY'S + // (mcpp#967): the refresh policy decides (`--offline`, `[index] + // auto_refresh`, the debounce, one sync per run), and a refreshed index is + // scanned again. Before, a row the index listed after the local copy was + // taken was refused as unknown until `mcpp index update`. Decided once per + // row per process; what was decided is what the refusal says. A caller for + // which a miss is not a refusal (a row the table holds and serves without a + // platform) asks without it. + if (it == claims->end() || it->second.empty()) { + if (!refreshOnMiss) return {}; + { + std::scoped_lock guard(g_columnLock); + if (!g_columnMisses.try_emplace(std::string(triple)).second) return {}; + } + const auto env = mcpp::config::make_xlings_env(**cfg); + const auto decision = mcpp::pm::decide_for_miss( + mcpp::pm::policy_for(**cfg), env, std::format("the platform column for {}", triple)); + std::string note; + if (auto applied = mcpp::pm::apply(decision, env); !applied) { + note = std::format("the package index lists no platform for it, and its refresh failed: {}", + applied.error()); + } else if (!decision.shouldRefresh) { + note = std::format("the package index lists no platform for it, and was not refreshed ({})", + mcpp::pm::reason_text(decision.reason)); + } else { + { + std::scoped_lock guard(g_columnLock); + g_columnScans.clear(); } + claims = column_claims(state, **cfg); + it = claims->find(std::string(triple)); + note = "the package index, refreshed by this command, lists no platform for it"; } - std::scoped_lock guard(lock); - claims = scanned.try_emplace(rootsKey, std::move(built)).first->second; + const bool found = it != claims->end() && !it->second.empty(); + std::scoped_lock guard(g_columnLock); + if (found) g_columnMisses.erase(std::string(triple)); + else g_columnMisses[std::string(triple)] = note; + if (!found) return {}; } - auto it = claims->find(std::string(triple)); - if (it == claims->end() || it->second.empty()) return {}; - const int first = std::ranges::min(it->second, {}, &Claim::rank).rank; + const int first = std::ranges::min(it->second, {}, &ColumnClaim::rank).rank; std::vector who; for (auto const& c : it->second) if (c.rank == first) who.push_back(c.package); @@ -498,6 +571,14 @@ std::string platform_short_name(std::string_view ref) { return r ? known_short_name(*r) : std::string{}; } +// A row the built-in table holds as `planned`: nothing of the engine serves it, +// so a miss of the index's column for it ends in a refusal, and refreshes +// (mcpp#967). +static bool planned_row(const mcpp::toolchain::triple::Triple& t) { + const auto* known = mcpp::toolchain::triple::find_known_target(t); + return known && known->tier == "planned"; +} + // WHETHER THE BUILT-IN TABLE ANSWERS THE PLATFORM COLUMN FOR A ROW (design // 2026-11-16 §6.2). It does not for a row it does not hold (a platform defines // it), holds as `planned`, or holds with no platform in its column and no @@ -545,7 +626,7 @@ std::string named_provider(const PrepareState& state, const mcpp::toolchain::tri // a choice of the project would. A row nothing defines is answered by // `define_early_platform`. if (!mcpp::toolchain::triple::find_known_target(t) || table_answers(state, t)) return {}; - const auto column = index_platform_for(state, t.str()); + const auto column = index_platform_for(state, t.str(), planned_row(t)); if (column.empty()) return {}; const auto ref = mp::parse_ref_text(column); if (!ref) return {}; @@ -621,7 +702,7 @@ std::expected step1_select_platform(PrepareState& state) { { const bool tableAnswers = table_answers(state, t); if (!tableAnswers) - if (auto indexed = index_platform_for(state, t.str()); !indexed.empty()) { + if (auto indexed = index_platform_for(state, t.str(), planned_row(t)); !indexed.empty()) { column = std::move(indexed); fromIndex = true; } @@ -635,11 +716,12 @@ std::expected step1_select_platform(PrepareState& state) { state.platform = mcpp::platform::Resolved{}; state.platform.selection = std::move(*sel); if (state.platform.selection.none && !state.plannedRow.empty()) { + const auto column = index_platform_miss_note(t.str()); refusal::record(refusal::Code::TierPlanned); return std::unexpected(std::format( "target '{}' is registered but not yet supported (planned) — no toolchain is published for it yet.\n" - " A toolchain named for the build opts in early: [target.{}] toolchain, or --toolchain.", - state.plannedRow, state.plannedRow)); + " A toolchain named for the build opts in early: [target.{}] toolchain, or --toolchain.{}", + state.plannedRow, state.plannedRow, column.empty() ? std::string{} : "\n " + column)); } if (!state.platform.selection.none) { auto const& ref = state.platform.selection.ref; diff --git a/src/build/prepare/state.cppm b/src/build/prepare/state.cppm index 11484cc98..c49b67e68 100644 --- a/src/build/prepare/state.cppm +++ b/src/build/prepare/state.cppm @@ -1389,7 +1389,12 @@ std::string platform_short_name(std::string_view ref); // step 5): the platform package whose index entry lists `triple` among its // `platform_rows`, as `.@*` (the series of §14), when the // built-in table has no answer for it. Empty when no entry claims the row. -std::string index_platform_for(const PrepareState& state, std::string_view triple); +// `refreshOnMiss`: the caller refuses the row without an answer, so a miss goes +// through the index refresh policy as a dependency's does (mcpp#967). +std::string index_platform_for(const PrepareState& state, std::string_view triple, bool refreshOnMiss = false); +// What a refusal of `triple` adds about the index after a miss of its column: +// whether the index was refreshed, and why not. Empty when nothing missed. +std::string index_platform_miss_note(std::string_view triple); void step4b_seed_platform(PrepareState& state); std::expected step4b_apply_platform(PrepareState& state); diff --git a/src/build/prepare/toolchain.cpp b/src/build/prepare/toolchain.cpp index 0fd26f906..49df6bead 100644 --- a/src/build/prepare/toolchain.cpp +++ b/src/build/prepare/toolchain.cpp @@ -773,14 +773,17 @@ step1_validate_target_tier(PrepareState& state, TargetOverrideCtx& ctx) { ctx.requestedSpelling, rows)); } auto sug = triple::did_you_mean(ctx.requestedSpelling); + // Whether the index's platform column was looked at again (mcpp#967). + const auto column = index_platform_miss_note(ctx.requestedSpelling); refusal::record(refusal::Code::UnknownTarget); return std::unexpected(std::format( "unknown target '{}'{}\n" " known targets: `mcpp toolchain list`; a custom triple needs an\n" - " explicit [target.{}] section in mcpp.toml", + " explicit [target.{}] section in mcpp.toml{}", ctx.requestedSpelling, sug ? std::format(" — did you mean '{}'?", *sug) : "", - ctx.requestedSpelling)); + ctx.requestedSpelling, + column.empty() ? std::string{} : "\n " + column)); } // A TOOLCHAIN NAMED FOR THE BUILD IS AS EXPLICIT AS ONE THE SECTION NAMES. // The refusal says that no toolchain is published for the row; a build that diff --git a/tests/e2e/index/refresh.cpp b/tests/e2e/index/refresh.cpp index d70bb2a8f..0f73d1030 100644 --- a/tests/e2e/index/refresh.cpp +++ b/tests/e2e/index/refresh.cpp @@ -1,8 +1,8 @@ // When an index is refreshed and what a failed resolution says: refresh by local resolvability, // an index's floor, a bare compat name, the bounded and owned refresh, the envelope's network // record, auto_refresh, the offline codes, a recorded payload that is gone, a floor as a closing -// tip, and a version the index lacks (formerly tests/e2e/173, 185, 730, 732, 733, 734, 735, 803, -// 812, 949). +// tip, a version the index lacks, and a row the platform column misses (formerly tests/e2e/173, 185, +// 730, 732, 733, 734, 735, 803, 812, 949; 967). import std; import mcpplibs.mtest; import mcpp.test.support; @@ -439,6 +439,39 @@ exit 0 expect(fatal(s::no_line(a, "Downloading +compat") && s::no_line(b, "Downloading +compat"))) << "C: another namespace was shown as downloading: " << a << b; expect(fatal(s::has_line(a, "wire address tried: mcpplibs:laterpkg@1.0.1$"))) << "C: the refusal does not name the one address asked for: " << a; }; + + "967 a row the platform column misses is an index miss"_test[s::fresh_sandbox] = [] { + // A target row the local index's platform column does not list, and that is refused + // without it, reaches the refresh policy as a dependency's miss does (mcpp#967): inside + // the debounce window, with auto_refresh = false and offline, nothing is synced and the + // refusal says why. Before, the row was refused as unknown with no refresh in between. + const auto home = s::case_home(s::home::inherited); + const auto index = home / "indices/mcpplibs"; + // A present index (it has `pkgs/`) that lists no platform for the row, just synced. + s::project(index, {{".xlings-index-version", "abc1234\n"}, {"pkgs/.keep", ""}, {".mcpp-index-updated", ""}}); + s::project app(s::case_dir() / "app", {{"mcpp.toml", "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"src/main.cpp", "int main() { return 0; }\n"}}); + const auto refused = [&](std::string_view why, const s::options& o) { + auto r = app.mcpp({"build", "--target", "x86_64-toyos"}, o); + const auto out = s::text(r); + expect(fatal(r.started && r.status != 0)) << why << ": a row nothing lists was built: " << out; + expect(fatal(s::has_line(out, "unknown target 'x86_64-toyos'"))) << why << ": not refused as unknown: " << out; + expect(fatal(s::has_line(out, std::format(R"(skip refresh for the platform column for x86_64-toyos: {})", why)))) + << why << ": the miss did not reach the refresh policy: " << out; + expect(fatal(s::has_line(out, std::format(R"(lists no platform for it, and was not refreshed \({}\))", why)))) + << why << ": the refusal does not say why the index was not refreshed: " << out; + expect(fatal(s::no_line(out, "Refreshing +package index|Updating +package index"))) << why << ": a sync: " << out; + }; + s::options o = inherited; + o.env.emplace_back("MCPP_VERBOSE", "1"); + refused("index was just refreshed", o); + age(index / ".mcpp-index-updated", std::chrono::hours{24 * 30}); + s::options off = o; + off.env.emplace_back("MCPP_OFFLINE", "1"); + refused("offline mode", off); + s::project(home, {{"config.toml", s::read_file(home / "config.toml") + "\n[index]\nauto_refresh = false\n"}}); + refused(R"(\[index\] auto_refresh = false)", o); + }; }}; } // namespace diff --git a/tests/e2e/openkal/platforms.cpp b/tests/e2e/openkal/platforms.cpp index 9ca17be57..57c4af858 100644 --- a/tests/e2e/openkal/platforms.cpp +++ b/tests/e2e/openkal/platforms.cpp @@ -591,7 +591,8 @@ flags = ["-e", "_start"] // F so does the row no built-in provider serves; G the host's row, which the table // answers, is not taken from the index; B a statement of the project stands over the // index; C two descriptors of one index listing the row are named, and the first is - // taken; D without the column the row is unknown and the planned row is refused. + // taken; D without the column, offline, the row is unknown and the planned row is + // refused, and the refusal says the index was not refreshed (mcpp#967). const auto platform = [](std::string_view name) { return std::format(R"([package] namespace = "acme" @@ -685,12 +686,16 @@ main = "src/main.c" expect(fatal(e2e::succeeded(r))) << "C: the build with two claims failed: " << s::text(r); expect(fatal(e2e::contains(s::text(r), "lists acme.toy, acme.toy2 for x86_64-toyos"))) << "C: the two claims are not named: " << s::text(r); expect(fatal(s::has_line(s::text(r), R"(Platform +toy 0\.1\.0)"))) << "C: the first claim was not taken: " << s::text(r); - // D + // D, offline: a miss of the column is an index miss (mcpp#967), and a refresh would bring + // the shared index, whose openkal-platform lists the planned row. app.write("local-index/pkgs/t/toy.lua", descriptor("toy", "")); app.write("local-index/pkgs/t/toy2.lua", descriptor("toy2", "")); - r = app.mcpp({"build", "--target", "x86_64-toyos"}, inherited); + s::options offline = inherited; + offline.env.emplace_back("MCPP_OFFLINE", "1"); + r = app.mcpp({"build", "--target", "x86_64-toyos"}, offline); expect(fatal(e2e::contains(s::text(r), "unknown target 'x86_64-toyos'"))) << "D: without the column the row was known: " << s::text(r); - r = app.mcpp({"build", "--target", "i686-linux-musl"}, inherited); + expect(fatal(e2e::contains(s::text(r), "lists no platform for it, and was not refreshed (offline mode)"))) << "D: the refusal does not say the index was not refreshed: " << s::text(r); + r = app.mcpp({"build", "--target", "i686-linux-musl"}, offline); expect(fatal(e2e::contains(s::text(r), "registered but not yet supported (planned)"))) << "D: without the column the planned row was built: " << s::text(r); }; }}; From 3525899175bfc612e5f672a8357365d4416ff55b Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Fri, 20 Nov 2026 01:41:12 +0900 Subject: [PATCH 23/30] Each build program resolves its own build dependencies; a platform series takes the newest release the build can use (mcpp#972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The target is one resolution and each build program -- a package's build.mcpp or its rules' program, a platform's plugin -- is another (design 2026-11-19 §21 A1-A6). A program's request takes the newest version the target or another program holds that admits it, the target being resolved first, so a build without a conflict keeps its versions and its build programs' identities. Programs that hold one package at two versions build in two build programs' graphs named by those versions, and mcpp.lock records the second version as an entry `+b` with `programs`. A conflict inside one program is refused naming the program. An entry the plugin-imports compat reading turns into a library's dependency belongs to each resolution that holds the library. A platform selected as a series skips a release whose version group a request of the build does not admit, naming the release and the request, and is refused listing each release when none is left. The pack launchers build on mcpp's own mcpplibs.os (0.4.6), which check_version_pins.sh holds equal to mcpp.toml's. The global dependency cache moves from plan.cpp to dependency_cache.cpp, divided at its own banners; graph.cpp's rename staging and provisioning closures move to graph_stage.cpp and graph_provisioning.cpp, the resolutions' helpers to graph_requests.cpp, and step13_lockfile's version entries to a function of their own: every file and step stays under its limit. --- ...11-19-ecosystem-rule-convergence-design.md | 32 +- .github/tools/check_version_pins.sh | 22 +- CHANGELOG.md | 30 +- docs/05-dependencies.md | 42 +- docs/30-build-mcpp.md | 5 + docs/35-platforms.md | 39 +- docs/50-machine-output.md | 2 +- docs/specs/README.md | 6 +- docs/specs/build-plugins.md | 17 +- docs/specs/manifest-semantics.md | 20 +- docs/specs/platforms.md | 23 +- docs/zh/05-dependencies.md | 26 +- docs/zh/30-build-mcpp.md | 5 +- docs/zh/35-platforms.md | 25 +- docs/zh/50-machine-output.md | 2 +- src/build/prepare.cppm | 6 + src/build/prepare/dependency_cache.cpp | 559 ++++++++++++++++++ src/build/prepare/driver.cpp | 35 +- src/build/prepare/features.cpp | 49 +- src/build/prepare/graph.cpp | 462 +++++---------- src/build/prepare/graph_edges.cpp | 10 +- src/build/prepare/graph_provisioning.cpp | 190 ++++++ src/build/prepare/graph_requests.cpp | 140 ++++- src/build/prepare/graph_stage.cpp | 154 +++++ src/build/prepare/host_graph.cpp | 81 ++- src/build/prepare/plan.cpp | 528 +---------------- src/build/prepare/platform.cpp | 64 +- src/build/prepare/records.cpp | 216 ++++--- src/build/prepare/state.cppm | 78 ++- src/doctor.cppm | 4 +- src/pack/distribution.cppm | 16 +- src/pm/lock_io.cppm | 14 + tests/e2e/deps/graph.cpp | 55 +- tests/e2e/openkal/platforms.cpp | 119 +++- tests/unit/test_pm_lock_io.cpp | 23 + 35 files changed, 2027 insertions(+), 1072 deletions(-) create mode 100644 src/build/prepare/dependency_cache.cpp create mode 100644 src/build/prepare/graph_provisioning.cpp create mode 100644 src/build/prepare/graph_stage.cpp diff --git a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md index 70d305a39..c53efa2c9 100644 --- a/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md +++ b/.agents/docs/2026-11-19-ecosystem-rule-convergence-design.md @@ -1030,13 +1030,17 @@ windows-2022,探针 mcpp#971(运行 44508574496),以 `CreateProcessW` - **共享。** 宿主范围的请求按约束取版本时,若另一个范围已取得满足该约束的版本,取其中最新者,否则取索引中最新者。没有冲突的 构建因此与此前取得相同的版本,构建程序的身份不变,升级不引起重编。 -**A2 执行:构建程序图按版本划分。** R10.2 的图仍按宿主工具链与标准分组;组内的程序按闭包中的版本划分:一个程序加入第一个 -在共有的每个包上版本相同的划分,否则另起一个。第一个划分在原目录,其余在 `-<摘要>`(摘要取该划分与第一个划分不同的 -包与版本)。每个划分一个子构建,`resolution_pins` 取该划分的版本;划分内版本相同的包只编译一次,划分之间由全局依赖缓存按身份 -共享。R10.3 的程序身份本来覆盖闭包中每个包的版本。 - -**A3 锁。** 一个身份一个条目;同一身份在另一个范围取得不同的版本时,该版本另成一个条目,名字带 `+b<范围摘要>`,`program` -写出该范围。合并与 `--locked` 按身份比较,不变。 +**A2 执行:构建程序图按版本划分。** R10.2 的图仍按宿主工具链与标准分组;组内的程序按**有争议的包**划分:两个程序取得不同 +版本的包。不取得任何有争议的包的程序留在原目录 ``;其余按它们取得的有争议的包与版本分组,目录为 `-<摘要>`, +摘要取这些包与版本。每个划分一个子构建,`resolution_pins` 取目标的版本、再以划分内程序自己的版本覆盖(划分内一致);划分 +内版本相同的包只编译一次,划分之间由全局依赖缓存按身份共享。R10.3 的程序身份本来覆盖闭包中每个包的版本。(实现时由 +"加入第一个相容的划分、第一个划分在原目录"改为按内容命名:前者的目录取决于一条命令构建哪些程序、以什么顺序,`mcpp build` +与 `mcpp test` 会把同一个程序放进不同的目录而互相使其失效;按内容命名时,一个程序在任何命令下都找到同一个目录,没有冲突的 +构建仍在原目录。) + +**A3 锁。** 一个身份一个条目;同一身份在另一个范围取得不同的版本时,该版本另成一个条目,名字带 `+b<范围摘要>`,`programs` +写出持有该版本的全部范围(排序;几个程序取得同一个版本时是一个条目)。合并与 `--locked` 按条目名比较,不变。`hash` 按包名 +与版本计算,不含 `+b`:它描述包的字节,不描述持有它的程序。 **A4 诊断。** 冲突只在一个范围内发生:消息写出该构建程序(`the build program of <包>`)与每个请求者。 @@ -1088,6 +1092,20 @@ windows-2022,探针 mcpp#971(运行 44508574496),以 `CreateProcessW` - **openkal-image。** `examples/starts` 在四台机器上:启动自己(argv[0] 如启动者所给)、`exits.oke` 与一个系统程序;签名要求下 未签名的映像被拒绝为 `kal_err_permission`;不指定加载器时被拒绝为 `kal_err_not_program`。 +### 21.7 实现记录 + +- **目标先解析。** 构建程序范围的工作项在目标的工作列表排空之后处理;A1 的"取已取得的版本"因此与声明的先后无关。 + e2e 956 B(构建程序的请求先声明)由"合并并替换"变为"取目标的 1.1.0,不出现 `Merged`"。 +- **一个 (包, 版本) 一个槽位。** 两个范围取得同一版本时共用一个包槽位与依赖边,一个库只编译一次;取得不同版本时各有槽位。 + `package_instances` 按边求实例,只被构建程序范围到达的槽位只有构建程序实例。 +- **划分的度量。** e2e 1088:工程的程序取 mcpplibs.os 0.4.6、依赖的程序取 0.4.1(连带 openkal 0.24.0、openkal-linux + 0.25.0),两张构建程序图各一个版本,第二次构建两个程序都不重编;锁中 0.4.1 一组为 `+b` 条目。e2e 972:平台插件 + `=1.0.0` 与工程 `^1.1.0` 各取一个版本,各自运行时只认自己版本的值。 +- **A5 的重启。** 拒绝点(`platform-version-group` 的两处)在本次的平台版本由系列取得时登记 (版本, 请求);`prepare_build` + 以 `platform_series_skipped` 重新开始,系列加上 `<版本`,上一次取得的平台目录不再沿用;`resolve_graph` 同样处理。上限 32 次。 + 报告每次只写新跳过的那个版本(此前各次已写出)。 +- **A6。** `kLauncherOs` 为 0.4.6,`check_version_pins.sh` 的第三项检查它与 `mcpp.toml` 相同。 + ## 22. 源码的名字与路径依赖的缓存(#974) **来源。** mcpp#974:经 `path` 到达的库从不进入全局构建缓存,依赖它的每个工程、每次干净构建都重新编译它;HuxerUI 的一致性 CI diff --git a/.github/tools/check_version_pins.sh b/.github/tools/check_version_pins.sh index 7ec089b66..2238ba297 100755 --- a/.github/tools/check_version_pins.sh +++ b/.github/tools/check_version_pins.sh @@ -2,7 +2,7 @@ # # Version / pin drift guard. # -# Two invariants that used to live only in a comment: +# Three invariants that used to live only in a comment: # # 1. Every xlings version pinned anywhere in .github/ equals # `pinned::kXlingsVersion` in src/provider/xlings/xlings.cppm — which is the version @@ -11,6 +11,8 @@ # # 2. mcpp's own version is identical in all four places that carry it. # +# 3. The launchers a pack builds use mcpp's own mcpplibs.os release. +# # Why this exists: src/provider/xlings/xlings.cppm used to say "keep in lock-step with the # XLINGS_VERSION pins in release.yml / cross-build-test.yml / ci-linux-e2e.yml" # and that list was ALREADY incomplete — it omitted both composite actions, @@ -173,7 +175,23 @@ if [ -n "$v_xl" ] && [ -n "$v_toml" ] && [ "$v_xl" != "$v_toml" ]; then || bad "bootstrap pin '$v_xl' is NEWER than the version being built ('$v_toml') — CI would try to install an unreleased mcpp" fi +# ── 3. The launchers' OS library is mcpp's own ──────────────────────────── +# +# A pack's launchers build on one release of mcpplibs.os (src/pack/ +# distribution.cppm, kLauncherOs), and mcpp builds on one (mcpp.toml). The +# platform's plugin resolves its build dependencies apart from a launcher +# (design 2026-11-19 §21 A6), so nothing outside mcpp chooses the launchers' +# release, and two releases of one library in one repository is drift. +os_toml=$(awk -F '"' '/^mcpplibs\.os[[:space:]]*=/{print $2; exit}' mcpp.toml) +os_launcher=$(grep -oE 'kLauncherOs[[:space:]]*=[[:space:]]*"[^"]+"' src/pack/distribution.cppm \ + | grep -oE '"[^"]+"' | tr -d '"' | head -1) +if [ -z "$os_toml" ] || [ -z "$os_launcher" ]; then + bad "could not read mcpplibs.os from mcpp.toml ('$os_toml') or kLauncherOs from src/pack/distribution.cppm ('$os_launcher')" +elif [ "$os_toml" != "$os_launcher" ]; then + bad "src/pack/distribution.cppm builds the launchers on mcpplibs.os '$os_launcher', mcpp.toml builds mcpp on '$os_toml'" +fi + if [ "$fail" = 0 ]; then - echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl" >&2 + echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl; launchers on mcpplibs.os $os_launcher" >&2 fi exit "$fail" diff --git a/CHANGELOG.md b/CHANGELOG.md index 583ef3f09..71e7aab80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,13 +72,31 @@ - mcpp does not set `VSLANG` for cl.exe: the compiler speaks the language of the user's environment (`VSLANG`, else Visual Studio's), where it spoke English whenever the English resources were installed. `VSLANG=1033` asks for English (mcpp#970). -- A build program that pins `mcpp.plugins` below 0.26 shares no host graph with - `openkal-platform@0.31.0`, whose build program links `mcpp.plugins` 0.26.1: such a pin is - refused as irreconcilable on the rows openkal serves, and moves to 0.26.1. +- Each build program resolves its own `[build-dependencies]` (mcpp#972, SPEC-004 §12.3, + SPEC-007 R10.7). The target is one resolution, and each build program -- a package's + `build.mcpp` or the program its rules synthesise, a platform's plugin -- is another: a + program loads one version of a package, and two programs do not constrain each other. A + project's `build.mcpp` may use `mcpp.plugins` 0.22.1 beside a platform plugin that uses + 0.26.1, where the two were refused as irreconcilable and the project had to copy the + version the platform used inside. A build program's request takes the newest version the + target or another program already holds that admits it, the target being resolved first, + so a build without a conflict keeps its versions and its build programs' identities. + Programs that hold one package at two versions are compiled in two build programs' + graphs, `-`, named by those versions. `mcpp.lock` records a version only + build programs hold as an entry of its own, `+b`, whose `programs` lists + them; a conflict inside one program is refused naming the program and every requester. +- A platform selected as a series skips a release whose version group a request of the build + does not admit: the build is planned again with the series below it, and the report names + the release and the request (SPEC-013 §14). With no release of the series left the build is + refused (`platform-version-group`), listing each release and the request that refused it. A + release written exactly, or recorded in `mcpp.lock`, is taken as written. Before, the + series' newest release decided, and a platform release that moved its group stopped every + project that selected the series. - The launchers `mcpp pack` writes (`bin/.exe`, the single-file launcher) build on - `mcpplibs.os` 0.4.5, the release the build program of openkal-platform 0.31 links through - `mcpp.plugins` 0.26.1: a launcher is built over that platform, and the platform's build - dependencies are resolved with the launcher's own (mcpp#972). + `mcpplibs.os` 0.4.6, mcpp's own release (0.4.4 in 2026.11.18.1); + `.github/tools/check_version_pins.sh` holds the two equal. A launcher is built over the + platform of the rows openkal serves, whose plugin now resolves its build dependencies + apart from the launcher (mcpp#972). ### Fixed diff --git a/docs/05-dependencies.md b/docs/05-dependencies.md index 9fe2520d3..b561b95c9 100644 --- a/docs/05-dependencies.md +++ b/docs/05-dependencies.md @@ -265,9 +265,10 @@ mcpplibs.os = { version = "0.4.4", override = true } the defaults, and the project may choose another. - **When it is needed.** Two version requests that no SemVer merge reconciles are refused naming both, and the refusal ends with the line that overrides - the package. Two requests from `[build-dependencies]` tables are refused the - same way and are never renamed apart: a build dependency is a module of a - build program, and one build loads one version of it (2026.11.16.1). A library that pins one release of a package every consumer + the package. Two requests of one build program's resolution are refused the + same way, naming the program, and are never renamed apart: a build dependency + is a module of a build program, and a program loads one version of it + (2026.11.16.1; per program since 2026.11.19.1, below). A library that pins one release of a package every consumer shares (one that provides a target-side layer or a kernel ABI) is noted to the projects that use it, and `mcpp publish` refuses the pin in a library: a range leaves the choice to the project. @@ -538,9 +539,34 @@ and the build program is compiled by the same plan (30 — `build.mcpp`). **One package in both tables.** A package written in `[dependencies]` and in `[build-dependencies]` of one manifest has two instances, one compiled for the -target and one for the build program. Both have the version the lock records. -Their features are activated separately: a feature requested only by a -`[build-dependencies]` edge does not reach the target instance. +target and one for the build program. Both have one version unless the two +requests admit none in common (below). Their features are activated +separately: a feature requested only by a `[build-dependencies]` edge does not +reach the target instance. + +**Each build program resolves its own build dependencies *(mcpp +2026.11.19.1+)*.** A C++ program loads one version of a package; two programs do +not constrain each other. The target (the selected members, what they reach +through `[dependencies]` and what the platform adds) is one resolution, and +each build program is another: a package's `build.mcpp` (or the program its +rules synthesise) with its `[build-dependencies]` and their closure, and a +platform's plugin with the platform's. + +- Within one resolution, requests merge as before, and two requests no merge + reconciles are refused, naming the program (`in the build program of + app@0.1.0`) and every requester. +- A build program's request takes the newest version the target or another + program already holds that the request admits, and the newest the index holds + only when none does; the target is resolved first. A build without a conflict + keeps the versions, and the build programs' identities, it had before. +- Two versions of one package coexist only between programs: the project's + `build.mcpp` may use `mcpp.plugins` 0.22.1 while its platform's plugin uses + 0.26.1. `mcpp.lock` records the second version as an entry of its own, + `+b`, whose `programs` lists the programs that hold it. +- The root's `override = true` holds for every resolution. + +Before, one table held every resolution of a build, and a project's build +program could only use the version the platform's plugin used (mcpp#972). **Which row reads a package's conditional tables *(mcpp 2026.11.19.1+)*.** A package's `[target..dependencies]` are read by the row the package is @@ -573,8 +599,8 @@ A cycle among packages is refused where the graph is resolved, naming its edges, under every cache mode; a tool whose own sub-build requests it again is refused at that first repetition, naming the chain. -Unlike `[dev-dependencies]`, build dependencies **are** walked transitively and -resolved with the project, one version per package: a build dependency's own +Unlike `[dev-dependencies]`, build dependencies **are** walked transitively, +one version per package in each build program: a build dependency's own dependencies are what make it work, and they serve the build program with it. A feature scopes build-time requests without a second declaration site. A diff --git a/docs/30-build-mcpp.md b/docs/30-build-mcpp.md index ddf9fc741..61800500a 100644 --- a/docs/30-build-mcpp.md +++ b/docs/30-build-mcpp.md @@ -1888,6 +1888,11 @@ dependencies and its implementation units — and `build.mcpp` is compiled by the same plan, so a BMI and the program importing it agree on standard, dialect and compiler by construction. A library several programs use is compiled once for them, and a program of another standard is in another graph. +Each build program resolves its own `[build-dependencies]` (05 — Dependencies, +*Each build program resolves its own build dependencies*); programs that hold +one package at different versions are in different graphs, +`-`, named by those versions (mcpp 2026.11.19.1+), and a +program that holds none of them stays in ``. `mcpp build` in that directory reproduces the compile of the build programs. The compiled program is also placed beside its package's run record, as `target/.build-mcpp/build.mcpp.bin` (`build.mcpp.exe` on Windows), to be run by diff --git a/docs/35-platforms.md b/docs/35-platforms.md index b573c6981..639ded628 100644 --- a/docs/35-platforms.md +++ b/docs/35-platforms.md @@ -192,13 +192,21 @@ directory or a repository are not bound by the group. An mcpp before index change resolves the same either way. The group's version is a constraint of the request it binds and takes part in -every merge *(mcpp 2026.11.19.1+)*. A package that a build program and the target -both ask for has one version: a build program's `>=1.0, <2.0` and the group's -`=1.1.0` give 1.1.0, in whichever order the two requests arrive, and a request -that had already taken a newer version is replaced (`Merged 1.2.0 ⨯ -1.1.0 → v1.1.0`). Where no version satisfies every request the refusal lists -each request and its requester, the group's under the platform's name. A -package only a build program uses is still not bound. +every merge of the target *(mcpp 2026.11.19.1+)*. Where no version satisfies +every request the refusal lists each request and its requester, the group's +under the platform's name. A build program's request is resolved after the +target's and takes the version the target holds when it admits it: a build +program's `>=1.0, <2.0` beside the group's `=1.1.0` gives 1.1.0 to both, in +whichever order the two are declared. A build program resolves its own build +dependencies (05 — Dependencies), so a package only build programs use, or a +version the target's does not admit, is not bound by the group. + +**The plugin and the project's `build.mcpp` are two programs *(mcpp +2026.11.19.1+)*.** Each resolves its own build dependencies: the platform's +plugin may use `mcpp.plugins` 0.26.1 while the project's `build.mcpp` uses +0.22.1, and a project no longer copies the versions of the platform's internal +build tools. `mcpp.lock` records the second version as an entry +`+b` with `programs = ["build:@"]`. ## The built-in platform is one release @@ -224,6 +232,21 @@ next build takes the recorded release while the series admits it. `mcpp update` (or `mcpp update openkal-platform`) drops the record, and the build after it takes the newest release again. +A series also skips a release whose version group a request of the build does +not admit *(mcpp 2026.11.19.1+)*: the build is planned again with the series +below it, and the report names the release and the request, + +``` + Platform plat v0.2.0 is not usable by this build: its version group names fam 1.2.0, and proj requests '~1.1.0' + Platform plat ^0 → v0.1.0 (the newest this build can use) +``` + +so a platform release that moves its group does not stop a project that selects +the series. With no release left the build is refused +(`platform-version-group`), listing each release and the request that refused +it. A release written exactly, or recorded in `mcpp.lock`, is taken as written, +and its group's refusal is the build's. + ## A row the index names (2026.11.17.1+) A row the built-in table has no platform for — a row the engine does not know, @@ -541,7 +564,7 @@ event and in `result.json` ([50](50-machine-output.md)). | `platform-not-a-platform` | the selected package has no `[platform]` | select a platform package, or `none` | | `platform-target-unserved` | the platform does not serve the triple | build for one of its `targets`, or for the host row it names | | `platform-tool-absent` | a program of the link or the seal is neither built nor found | name its package under `[target.'cfg(platform = "…")'.tools]` | -| `platform-version-group` | another version of a family package is in the graph | select the platform release of that version, or pin the group's version | +| `platform-version-group` | another version of a family package is in the graph, or no release of a series admits the build's requests | select the platform release of that version, or pin the group's version | | `platform-phase-directive` | the plugin stated a directive the platform phase does not take | state it from the package's own build program | | `lld-required-absent` | the platform links with `ld.lld` and the toolchain has none | use the platform's toolchain | diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index b041f6751..57b3e737d 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -673,7 +673,7 @@ a program classifying the outcome reads `reason`: | `platform-not-a-platform` | the package selected as the target's platform has no `[platform]` section *(2026.10.27.1+)* | | `platform-target-unserved` | the selected platform does not serve the target: the triple is not among its `targets`, and a host triple is not among its `[platform.hosted] targets` *(2026.10.27.1+)* | | `platform-tool-absent` | a program the platform's link or seal runs (`okld` for an ok image) is neither built by the graph nor found on this machine *(2026.10.27.1+; `image-tool-absent` in 2026.10.26.2)* | -| `platform-version-group` | a package of the platform's version group is in the graph at another version *(2026.10.27.1+)* | +| `platform-version-group` | a package of the platform's version group is in the graph at another version *(2026.10.27.1+)*, or no release of a platform series admits the build's requests *(2026.11.19.1+)* | | `platform-phase-directive` | a platform's plugin stated a directive its phase does not take *(2026.10.27.1+)* | | `other` | a refusal whose branch has not been given a token yet | diff --git a/docs/specs/README.md b/docs/specs/README.md index f243f2b81..82a448f87 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -33,16 +33,16 @@ | [SPEC-001](package-identity.md) | 包身份(`package.namespace` / `package.name`)、`[dependencies]` 选择器与匹配机制 | 评审中 v1.1 | 2026-08-03 | mcpp >= 0.0.106 | | [SPEC-002](target-side.md) | 目标侧模型与能力声明(`mcpp:` 保留命名空间、五层、三条规则、目标侧执行、一个进程一套 C++ 运行时) | 评审中 v1.6 | 2026-11-10 | mcpp >= 2026.8.24.2;§2.4 与 §6 mcpp >= 2026.10.24.1;§2.2 的对象形式与 §7 mcpp >= 2026.10.26.1;§2.5 与 §3.8 mcpp >= 2026.11.10.1 | | [SPEC-003](exit-codes.md) | 退出码契约(分类、语义、稳定性承诺) | 评审中 v1.2 | 2026-10-26 | mcpp >= 2026.9.1.1;`5` 与信号 mcpp >= 2026.10.26.1 | -| [SPEC-004](manifest-semantics.md) | `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约 | 草案 v1.17 | 2026-10-23 | 条件化形状 mcpp >= 2026.8.29.1;目标轴 mcpp >= 2026.9.6.4;`linkage` 默认值 mcpp >= 2026.9.15.2;链接 flag 的词读法 mcpp >= 2026.9.26.2;条件化的 `dialect_cxxflags` 与 `-p` 的包身份 mcpp >= 2026.9.28.1;条件表按具体程度生效 mcpp >= 2026.9.28.2;平台前缀规约 mcpp >= 2026.10.5.1;根包的继承、根位置的 profile、链接 flag 的作用域、链接期优化与导出发现 mcpp >= 2026.10.5.2;架构段的 `x86` 即 `i686` mcpp >= 2026.10.5.3;前缀即语义与工作空间层 mcpp >= 2026.10.10.1;依赖的库与配置(§12) mcpp >= 2026.10.11.1;值的四档层次与 `run_under` mcpp >= 2026.10.24.1 | +| [SPEC-004](manifest-semantics.md) | `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约 | 草案 v1.23 | 2026-11-20 | 条件化形状 mcpp >= 2026.8.29.1;目标轴 mcpp >= 2026.9.6.4;`linkage` 默认值 mcpp >= 2026.9.15.2;链接 flag 的词读法 mcpp >= 2026.9.26.2;条件化的 `dialect_cxxflags` 与 `-p` 的包身份 mcpp >= 2026.9.28.1;条件表按具体程度生效 mcpp >= 2026.9.28.2;平台前缀规约 mcpp >= 2026.10.5.1;根包的继承、根位置的 profile、链接 flag 的作用域、链接期优化与导出发现 mcpp >= 2026.10.5.2;架构段的 `x86` 即 `i686` mcpp >= 2026.10.5.3;前缀即语义与工作空间层 mcpp >= 2026.10.10.1;依赖的库与配置(§12) mcpp >= 2026.10.11.1;值的四档层次与 `run_under` mcpp >= 2026.10.24.1;§12 第 3 条一个程序一个版本 mcpp >= 2026.11.19.1 | | [SPEC-005](build-database.md) | 构建数据库:`mcpp emit build-database` 的内容、取值规则与不写工程目录的保证 | 评审中 v1.8 | 2026-10-11 | mcpp >= 2026.9.15.1;v1.3 条款 mcpp >= 2026.9.26.2;v1.4 条款 mcpp >= 2026.9.27.1;v1.5 条款 mcpp >= 2026.9.28.1;v1.6 条款 mcpp >= 2026.9.29.5;v1.7 条款 mcpp >= 2026.9.30.2;v1.8 条款 mcpp >= 2026.10.11.1 | | [SPEC-006](toolchain-management.md) | 工具链管理:身份、来源、选择与载荷契约 | 草案 v0.11 | 2026-11-10 | 逐条标注;已实现条款 mcpp >= 2026.9.24.1;§3.7 mcpp >= 2026.9.28.1;§3.7.1 mcpp >= 2026.9.28.2;§2.2.1 与 §3.3 的非缺省来源 mcpp >= 2026.10.1.3;§3.8 mcpp >= 2026.10.8.1;§3.9 至 §3.11 mcpp >= 2026.10.24.1;v0.11 条款 mcpp >= 2026.11.10.1 | -| [SPEC-007](build-plugins.md) | 构建插件:配置、施工与校验的分工,运行时与规划期的义务 | 草案 v0.12 | 2026-11-05 | 逐条标注;mcpp >= 2026.9.26.2;v0.3 条款 mcpp >= 2026.9.27.1;v0.4 条款 mcpp >= 2026.9.28.1;v0.5 条款 mcpp >= 2026.9.28.2;v0.6(§9)mcpp >= 2026.9.28.3;R3.9、R3.10 mcpp >= 2026.10.10.1;§10 mcpp >= 2026.10.11.1;§3.1 协议 18 mcpp >= 2026.11.5.1 | +| [SPEC-007](build-plugins.md) | 构建插件:配置、施工与校验的分工,运行时与规划期的义务 | 草案 v0.15 | 2026-11-20 | 逐条标注;mcpp >= 2026.9.26.2;v0.3 条款 mcpp >= 2026.9.27.1;v0.4 条款 mcpp >= 2026.9.28.1;v0.5 条款 mcpp >= 2026.9.28.2;v0.6(§9)mcpp >= 2026.9.28.3;R3.9、R3.10 mcpp >= 2026.10.10.1;§10 mcpp >= 2026.10.11.1;§3.1 协议 18 mcpp >= 2026.11.5.1;协议 19 mcpp >= 2026.11.16.1;协议 20、R10.7 mcpp >= 2026.11.19.1 | | [SPEC-008](library-interface.md) | 库的接口:公开模块、发布闭包与两种形态的一致 | 草案 v0.2 | 2026-10-11 | 第一阶段(只警告)mcpp >= 2026.9.28.3;v0.2 mcpp >= 2026.10.11.1 | | [SPEC-009](toolchain-maintenance.md) | 工具链的支持与维护:版本线、默认值、来源、移动与退役 | 草案 v0.6 | 2026-10-09 | 逐条标注;本版只有规范,多数条款未实现;§6.4 mcpp >= 2026.10.8.1 | | [SPEC-010](home-initialization.md) | home 选择、单跳入口、初始化、网络边界与单次运行的 `--home` | 草案 v0.2 | 2026-10-26 | mcpp >= 2026.10.19.1;网络能力逐条标注;§6 mcpp >= 2026.10.26.1、xlings >= 2026.10.26.1 | | [SPEC-011](concurrency.md) | 并发运行:三段、构建目录锁、原子发布、`mcpp.lock` 的乐观写入、取消;编译执行器的恢复、发布、记录与文本 | 评审中 v1.2 | 2026-11-19 | mcpp >= 2026.10.26.1;§5.1 mcpp >= 2026.11.7.1;§5.1 第 6 条的不可观察单元与第 8 至 12 条 mcpp >= 2026.11.19.1 | | [SPEC-012](human-output.md) | 面向人的输出:两条流、两种介质、行的种类、颜色、程序的文本 | 评审中 v1.1 | 2026-11-19 | mcpp >= 2026.10.26.1;§5 mcpp >= 2026.11.19.1 | -| [SPEC-013](platforms.md) | 平台:`[platform]` 的七组键、版本组与 `hosted`,三种写法与层次,目标图与宿主图的选择,`cfg(platform)`,依赖图的补全,链接、命名与封装,运行器与载体,平台阶段与协议 17 | 评审中 v1.7 | 2026-11-10 | mcpp >= 2026.10.27.1;§10 的执行宿主与 §11 的载体架构 mcpp >= 2026.10.29.1;§7.5 版本组参与解析与 §14 平台系列 mcpp >= 2026.11.7.1;§14 内置平台为确切版本 mcpp >= 2026.11.7.2;v1.7 条款 mcpp >= 2026.11.10.1 | +| [SPEC-013](platforms.md) | 平台:`[platform]` 的七组键、版本组与 `hosted`,三种写法与层次,目标图与宿主图的选择,`cfg(platform)`,依赖图的补全,链接、命名与封装,运行器与载体,平台阶段与协议 17 | 评审中 v1.15 | 2026-11-20 | mcpp >= 2026.10.27.1;§10 的执行宿主与 §11 的载体架构 mcpp >= 2026.10.29.1;§7.5 版本组参与解析与 §14 平台系列 mcpp >= 2026.11.7.1;§14 内置平台为确切版本 mcpp >= 2026.11.7.2;v1.7 条款 mcpp >= 2026.11.10.1;§14 索引的平台列 mcpp >= 2026.11.17.1;§14 系列取可用的最新版本与 §15.1 的 `+b` 条目 mcpp >= 2026.11.19.1 | | [SPEC-014](packaging.md) | 打包与分发:链接、闭包、容器三个维度与构建腿,腿的合并,暂存树与 `mcpp-package.json`,平台的树、运行覆盖、单文件、polyglot、容器列表、签名、压缩、合并与分发描述 | 评审中 v2.2 | 2026-11-10 | mcpp >= 2026.10.31.1;逐条标注;v2.2 条款 mcpp >= 2026.11.10.1 | | [SPEC-015](target-sets.md) | 目标集合与腿:集合的写法与展开,命令对多条腿的语义,串行规划与一起构建,测试归档,机器输出 | 评审中 v1.0 | 2026-10-30 | mcpp >= 2026.10.30.1 | | [SPEC-016](targets-and-providers.md) | 目标与提供者:行、提供者、选择规则、平台定义的行、C 环境与执行、报告、机器输出与锁 | 评审中 v1.1 | 2026-11-10 | mcpp >= 2026.11.9.1;v1.1 条款 mcpp >= 2026.11.10.1 | diff --git a/docs/specs/build-plugins.md b/docs/specs/build-plugins.md index 743011079..ec7cbba40 100644 --- a/docs/specs/build-plugins.md +++ b/docs/specs/build-plugins.md @@ -4,10 +4,10 @@ |---|---| | 规范编号 | SPEC-007 | | 标题 | 构建插件:配置、施工与校验的分工,运行时与规划期的义务 | -| 状态 | 草案 v0.14 | -| 版本 | 0.14 | -| 最后修改 | 2026-11-19 | -| 对应实现 | 逐条标注。未注明版本的「已实现」条款对应 mcpp >= 2026.9.26.1;注明 mcpp#702 的条款对应 mcpp >= 2026.9.26.2;注明 mcpp#707、#708、#709、#711 的条款对应 mcpp >= 2026.9.27.1;注明 mcpp#723 的条款对应 mcpp >= 2026.9.28.1;注明 mcpp 2026.9.28.2 的条款对应该版本;§9 与注明 mcpp#734 的条款对应 mcpp >= 2026.9.28.3;注明 mcpp#778 的条款对应 mcpp >= 2026.10.10.1;§10 对应 mcpp >= 2026.10.11.1;R9.10 与协议 17 对应 mcpp >= 2026.10.27.1;§3.1 与协议 18 对应 mcpp >= 2026.11.5.1;协议 19 对应 mcpp >= 2026.11.16.1;协议 20 与 R3.16、R3.17、R6.7 对应 mcpp >= 2026.11.19.1 | +| 状态 | 草案 v0.15 | +| 版本 | 0.15 | +| 最后修改 | 2026-11-20 | +| 对应实现 | 逐条标注。未注明版本的「已实现」条款对应 mcpp >= 2026.9.26.1;注明 mcpp#702 的条款对应 mcpp >= 2026.9.26.2;注明 mcpp#707、#708、#709、#711 的条款对应 mcpp >= 2026.9.27.1;注明 mcpp#723 的条款对应 mcpp >= 2026.9.28.1;注明 mcpp 2026.9.28.2 的条款对应该版本;§9 与注明 mcpp#734 的条款对应 mcpp >= 2026.9.28.3;注明 mcpp#778 的条款对应 mcpp >= 2026.10.10.1;§10 对应 mcpp >= 2026.10.11.1;R9.10 与协议 17 对应 mcpp >= 2026.10.27.1;§3.1 与协议 18 对应 mcpp >= 2026.11.5.1;协议 19 对应 mcpp >= 2026.11.16.1;协议 20 与 R3.16、R3.17、R6.7、R10.7 对应 mcpp >= 2026.11.19.1 | | 相关设计文档 | `.agents/docs/2026-09-26-compile-database-and-issue-699-design.md`(§5)、`.agents/docs/2026-09-28-ecosystem-design-and-optimisation-plan.md`(WS1、WS3)、`.agents/docs/2026-09-28-build-cost-foreign-toolsets-and-library-surface-design.md`(§3、§4)、`.agents/docs/2026-10-10-one-source-one-meaning-design.md`(§4)、`.agents/docs/2026-10-11-build-dependencies-one-pipeline-design.md` | | 相关 issue | mcpp#699、mcpp#701、mcpp#702、mcpp#703、mcpp#707、mcpp#708、mcpp#709、mcpp#711、mcpp#734、mcpp#755、mcpp#778、mcpp#797 | | 使用文档 | [docs/30 - build.mcpp](../30-build-mcpp.md)、[docs/31 - 编写规则包](../31-authoring-a-rule-package.md) | @@ -342,6 +342,9 @@ - **R10.2** 一次构建中宿主工具链与 C++ 标准相同的构建程序是同一张构建程序图的成员;图由引擎写到 `/target/.build-mcpp/host//`,由子构建规划,由 ninja 构建,`build.mcpp` 由同一个 计划编译。在该目录执行 `mcpp build` 即复现构建程序的编译。多个程序使用的库只编译一次。(**已实现**) + 两个程序取得同一个包的不同版本时(R10.7),图按这些包的版本划分:不涉及它们的程序仍在 ``, + 涉及的程序在 `-<摘要>`,摘要取自程序取得的这些包与版本;子构建按该图的版本取每个包。 + (**已实现**,mcpp 2026.11.19.1) - **R10.3** 程序在其身份变化时编译;身份覆盖引擎版本、工具链、标准、程序文本,以及图中每个包的 名字、版本、来源、构建程序实例的特性与可原地变化的源码树的内容。程序的运行缓存以身份为键。 所有程序身份未变时,不规划构建程序图。(**已实现**) @@ -360,6 +363,11 @@ - **P4** 程序目标的单元导入引擎接口时拒绝,并指向 `build.mcpp`。 - **P5** `mcpp pack` 拒绝插件:插件以源码分发。 (**已实现**) +- **R10.7** 每个构建程序的库按自己的范围解析(SPEC-004 §12 第 3 条):一个程序内一个包一个版本,两个 + 程序之间互不约束。程序的请求先取目标或另一个程序已取得、且满足它的最新版本,所以没有冲突的构建与 + 共用一次解析时取得相同的版本,程序的身份(R10.3)不变。一个程序内不能合并的两个请求在规划期拒绝, + 消息写出该程序与每个请求者;构建依赖不改名(设计 2026-11-15 N29)。另一个版本在 `mcpp.lock` 中为 + `+b` 条目(SPEC-013 §15.1)。(**已实现**,mcpp 2026.11.19.1,mcpp#972) ## 11. 变更记录 @@ -379,3 +387,4 @@ | 0.12 | 2026-11-05 | 随 mcpp 2026.11.5.1:新增 §3.1 可选 action 进度(协议 18),规定通道、限额、序号、关闭与失败/取消行为。 | | 0.13 | 2026-11-16 | 随 mcpp 2026.11.16.1(设计 2026-11-15 N30):R9.2 的协议表增加协议 19(`pack_program`)。;协议 19 增加 `runner_spec`(设计 2026-11-15 R2)。 | | 0.14 | 2026-11-19 | 随 mcpp 2026.11.19.1(设计 2026-11-19 §8):新增 R3.16(链接单元的对象清单,协议 20,mcpp#959)、R3.17(引擎自身命令的判据与 `${mcpp.self} write`)、R6.7(未声明的工具);R6.3 补充特性的工具到达链接该包库的每个构建程序(mcpp#955);R9.2 的协议表增加协议 20。 | +| 0.15 | 2026-11-20 | 随 mcpp 2026.11.19.1(mcpp#972,设计 2026-11-19 §21 A1、A2):新增 R10.7 构建程序的库按程序解析;R10.2 构建程序图按版本划分。 | diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index 277599ad6..3851f46a3 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -5,8 +5,8 @@ | **规范编号** | SPEC-004 | | **标题** | `mcpp.toml` 的平面划分、条件化形状、解析轴与命名规约 | | **状态** | **草案(Draft)** | -| **版本** | 1.22 | -| **最后修改** | 2026-11-19 | +| **版本** | 1.23 | +| **最后修改** | 2026-11-20 | | **最低实现版本** | 条件化形状:mcpp **2026.8.29.1**(`[target..build-dependencies]` 起齐备);目标轴:mcpp **2026.9.6.4** | | **作者/维护** | mcpp-community | | **相关设计文档** | `.agents/docs/2026-09-07-mcpp-toml-unified-semantics-design.md`
`.agents/docs/2026-06-04-manifest-schema-ownership.md`
`.agents/docs/2026-09-03-xlings-workspace-as-the-one-table.md`
`.agents/docs/2026-09-25-issue-690-workspace-build-inheritance-consistency.md`
`.agents/docs/2026-09-27-eight-reports-by-home-and-one-optimisation-plan.md`
`.agents/docs/2026-10-05-std-module-pair-msvc-lto-and-export-discovery-design.md`
`.agents/docs/2026-10-06-windows-x86-arch-vocabulary-and-rc-follow-ups-design.md`
`.agents/docs/2026-10-10-one-source-one-meaning-design.md`
`.agents/docs/2026-10-11-build-dependencies-one-pipeline-design.md`
`.agents/docs/2026-10-23-target-base-and-execution-host-design.md`
`.agents/docs/2026-10-26-control-contract-runtime-and-ide-ecosystem-design.md`(§24) | @@ -757,8 +757,16 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m 某配置中是否有实例,由从根出发、经过该配置的边的可达性决定。 2. **实例。** 一个包在一次构建中可以有目标实例与构建程序实例。一份清单在两张表中写同一个包, 得到两条边、两个实例,实现**禁止**把它们合并为一条边。 -3. **一个包一个版本。** 两个实例的版本相同,是锁文件记录的版本;构建程序图**必须**以主图的 - 解析为准。 +3. **一个程序一个版本。** 解析的范围是程序:目标配置(命令选择的成员、它们经 `[dependencies]` + 到达的包与平台加入的包)是一个范围;每个构建程序(一个包的 `build.mcpp` 或其规则合成的程序、 + 平台的插件)以该包的 `[build-dependencies]` 及其闭包为一个范围,名为 `build:<包>@<版本或来源>`。 + 一个范围内的请求按 §4、§7 合并,一个包一个版本;不同范围的同一个包各有记录,互不约束。根的 + `override = true` 对所有范围生效。构建程序范围的请求按约束取版本时,**必须**先取目标或另一个 + 程序已取得、且满足该约束的最新版本,没有时取索引中的最新版本;目标的请求先于构建程序的请求 + 解析。没有冲突的构建因此在目标实例与构建程序实例上取同一版本;两个版本只在两个程序之间并存, + 锁以 `+b` 条目记录(SPEC-013 §15.1)。一个范围内不能合并的两个请求被拒绝,消息**必须**写出 + 该构建程序(`the build program of <包>`)与每个请求者。**状态:已实现**(mcpp 2026.11.19.1, + mcpp#972;此前一次构建的所有范围共用一张解析表,两个构建程序的不同版本被拒绝)。 4. **按配置激活特性。** 目标实例的特性只由服务目标的边请求;构建程序实例的特性只由服务构建程序 的边请求。特性声明的载荷(`feature-xlings`)服务这次构建,取两者的并集。构建程序实例的 公开 include 目录、接口宏定义与链接 flag **禁止**进入目标的编译与链接。 @@ -767,7 +775,8 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m 的特性即请求库。`host-module` 的名字来自它只表示接口根模块的时期,按 §5.2 不改名;它的含义 是"构建程序使用这个依赖的库":公开模块可导入、公开头文件可见、对象参与链接。在 `[dependencies]` 中,库就是这条边的含义。 -6. **构建程序图。** 一次构建中宿主工具链与 C++ 标准相同的构建程序,是同一张图的成员;图中每个库 +6. **构建程序图。** 一次构建中宿主工具链与 C++ 标准相同、且在共有的包上版本相同的构建程序,是同一张 + 图的成员;两个程序取得同一个包的不同版本时,各自所在的图以这些版本命名(第 3 条)。图中每个库 按完整的包编译(包含目录、宏定义、特性、依赖、实现单元),构建程序本身由同一个计划编译。程序 在其身份变化时编译;身份**必须**覆盖引擎版本、工具链、标准、程序文本,以及图中每个包的名字、 版本、来源、构建程序实例的特性与可原地变化的源码树的内容。 @@ -871,3 +880,4 @@ mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 m | 1.20 | 2026-11-17 | mcpp 2026.11.17.1(设计 2026-11-16 §4):新增 §15 运行时文件与发布文件:`deploy` 的条目语法(字符串、通配符、`name`)、固定目录规则与过期副本的移除;`license_files` 与其缺省值。 | | 1.21 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):新增 §4.8 包的依赖表由构建它的那一行读取;两侧各自发出请求,锁记录两侧。 | | 1.22 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#931):新增 §8.1 链接的输入:链接词点名的文件(选项的值、`,` 分隔的段、MSVC `/` 选项的值、动作声明的输出)与链接器自己的报告;§10.2 特性的 `tools` 到达链接本包库的每个构建程序(mcpp#955);§9 新增第 17 条包实例的目录(mcpp#953)。 | +| 1.23 | 2026-11-20 | mcpp 2026.11.19.1(mcpp#972,设计 2026-11-19 §21 A1、A2、A4):§12 第 3 条改为一个程序一个版本,解析的范围是程序,构建程序先取已取得的版本,冲突的消息写出程序;第 6 条构建程序图按版本划分。 | diff --git a/docs/specs/platforms.md b/docs/specs/platforms.md index a647513f0..aefed2a9e 100644 --- a/docs/specs/platforms.md +++ b/docs/specs/platforms.md @@ -4,10 +4,10 @@ |---|---| | 规范编号 | SPEC-013 | | 标题 | 平台:一类目标的可复用配置,其声明、选择、合并、对依赖图的补全、链接与封装、运行器与载体 | -| 状态 | 评审中 v1.14 | -| 版本 | 1.13 | -| 最后修改 | 2026-11-19 | -| 对应实现 | mcpp >= 2026.10.27.1;§10 的执行宿主与 §11 的载体架构 mcpp >= 2026.10.29.1;`runner.hosts`、载体列表、`descriptor_keys` 与 `universal` 的弃用 mcpp >= 2026.10.30.1;§7.5 版本组参与解析、§14 平台系列与 §15.1 系列的锁 mcpp >= 2026.11.7.1;§14 内置平台为确切版本与提示 mcpp >= 2026.11.7.2;v1.7 条款 mcpp >= 2026.11.10.1;v1.10 条款 mcpp >= 2026.11.16.1;v1.11 条款(§14 索引的平台列) mcpp >= 2026.11.17.1;逐条标注 | +| 状态 | 评审中 v1.15 | +| 版本 | 1.15 | +| 最后修改 | 2026-11-20 | +| 对应实现 | mcpp >= 2026.10.27.1;§10 的执行宿主与 §11 的载体架构 mcpp >= 2026.10.29.1;`runner.hosts`、载体列表、`descriptor_keys` 与 `universal` 的弃用 mcpp >= 2026.10.30.1;§7.5 版本组参与解析、§14 平台系列与 §15.1 系列的锁 mcpp >= 2026.11.7.1;§14 内置平台为确切版本与提示 mcpp >= 2026.11.7.2;v1.7 条款 mcpp >= 2026.11.10.1;v1.10 条款 mcpp >= 2026.11.16.1;v1.11 条款(§14 索引的平台列) mcpp >= 2026.11.17.1;v1.15 条款(§14 系列取本次构建可用的最新版本,§15.1 构建程序的版本条目) mcpp >= 2026.11.19.1;逐条标注 | | 相关设计文档 | `.agents/docs/2026-11-16-ecosystem-gap-supplement-part2-design.md`、`.agents/docs/2026-11-15-ecosystem-gap-supplement-design.md`、`.agents/docs/2026-10-28-platforms-design.md`、`.agents/docs/2026-10-28-platforms-part2-packaging-and-distribution-design.md`、`.agents/docs/2026-10-29-openkal-run-test-pack-design.md` | | 相关 issue | mcpp#846、mcpp#847、mcpp#848 | | 相关规范 | [SPEC-002](target-side.md)(目标侧的层)、[SPEC-004](manifest-semantics.md)(条件化形状)、[SPEC-006](toolchain-management.md)(工具链选择)、[SPEC-007](build-plugins.md)(构建程序与协议)、[SPEC-011](concurrency.md)(原子发布)、[SPEC-014](packaging.md)(打包) | @@ -467,9 +467,14 @@ run_under = ["qemu-aarch64"] 1. 锁中记录了这一平台包(`origin = "platform"`)且记录的版本满足范围时,取记录的版本(§15.1); 2. 否则取范围内最新、且其 `[package] mcpp` 下限不高于当前引擎的版本;跳过的版本连同所需的引擎版本写入报告; -3. 结果写入 `mcpp.lock`;`mcpp update`(或 `mcpp update <平台包>`)删除记录,下一次构建按第 2 条重新选择。 +3. 按第 2 条取得的版本,其版本组(§7.5)不容许本次构建的某个请求而被拒绝时,该版本被跳过:准备以范围内更早、 + 引擎下界满足的版本重新开始,跳过的版本连同拒绝它的请求写入报告,与第 2 条同一形式。范围内没有可用的版本时 + 拒绝(`platform-version-group`),消息列出每个跳过的版本与拒绝它的请求。**已实现**(mcpp >= 2026.11.19.1, + 设计 2026-11-19 §21 A5;此前范围内最新的版本被版本组拒绝时,整个构建被拒绝); +4. 结果写入 `mcpp.lock`;`mcpp update`(或 `mcpp update <平台包>`)删除记录,下一次构建按第 2、3 条重新选择。 -平台发布补丁版本因此不需要发布 mcpp。精确版本(`@0.24.1`、`@=0.24.1`)按写出的取。**已实现** +平台发布补丁版本因此不需要发布 mcpp,平台的新版本改变版本组也不使以系列选择它的工程停止构建。精确版本 +(`@0.24.1`、`@=0.24.1`)与第 1 条取得的版本按写出的取:其版本组的拒绝即构建的拒绝。**已实现** **索引的平台列(mcpp >= 2026.11.17.1)。** 平台包的索引条目可以列出它提供的行: @@ -533,6 +538,11 @@ openkal-platform 的 `targets` 因此不含 WebAssembly。Web 经 `wasm32-emscri 以系列选择的平台包(§14)从锁中读回:记录的版本满足系列时被采用,这是锁对版本依赖的唯一一处约束;其他版本依赖 每次按约束重新解析,锁只作断言(`--locked`)。**已实现**(mcpp >= 2026.11.7.1) +一个包在构建程序的解析中取得另一个版本时(SPEC-004 §12.3,如平台的插件与工程的 `build.mcpp` 各用一个版本),该版本 +另成一个条目,名字为 `<名字>+b<摘要>`,`programs` 写出取得它的构建程序的解析范围(`build:<包>@<版本或来源>`,排序), +摘要取自这些范围;目标与第一个程序取得的版本仍为 `<名字>` 的条目。`--locked` 按条目的名字比较。**已实现** +(mcpp >= 2026.11.19.1,设计 2026-11-19 §21 A3) + ### 15.2 身份 平台包与它加入的包是依赖图的成员,进入配置指纹的依赖部分;换平台或平台版本即换构建目录。平台的身份为 @@ -642,3 +652,4 @@ openkal-platform 的 `targets` 因此不含 WebAssembly。Web 经 `wasm32-emscri | 1.12 | 2026-11-17 | mcpp 2026.11.17.2(设计 2026-11-17 §5):§3.1 平台选定之前可用的谓词,以层选择平台时的警告;§10 `hosts` 参与运行器的采用;§14 目标表的平台列为 0.29.1 | | 1.13 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#956):§7.5 版本组的约束进入合并,结果与请求到达的先后无关;拒绝列出每一个请求者。 | | 1.14 | 2026-11-19 | mcpp 2026.11.19.1(mcpp#967):§14 索引的平台列第 6 条,一行的未命中按依赖未命中的策略刷新索引并重新扫描,拒绝写出是否已刷新。 | +| 1.15 | 2026-11-20 | mcpp 2026.11.19.1(mcpp#972,设计 2026-11-19 §21 A3、A5):§14 平台系列第 3 条,版本组不容许本次构建的请求时取范围内更早的版本并写出跳过的版本与请求,全部不可用时拒绝;§15.1 构建程序取得的另一版本为 `+b` 条目并写出 `programs`。 | diff --git a/docs/zh/05-dependencies.md b/docs/zh/05-dependencies.md index 9d2c7e8d4..cad8c14eb 100644 --- a/docs/zh/05-dependencies.md +++ b/docs/zh/05-dependencies.md @@ -215,7 +215,7 @@ mcpplibs.os = { version = "0.4.4", override = true } - **生效位置。** 根的依赖表与 `[workspace.dependencies]`;后者管辖工作空间所有成员的计划(在那里声明一个包不会让成员依赖它)。`[dependencies]` 管辖构建的图与测试的图;`[dev-dependencies]` 只管辖测试的图;`[build-dependencies]` 只管辖构建程序的图。库写出它时被告知已忽略:这是工程的决定。 - **陈述。** 被替换且本不接受覆盖值的每条请求打印一次:`Overrode <包> <请求> (requested by <请求方>) -> <覆盖> (<表>)`。`mcpp.lock` 为该包记 `override = true`,`--locked` 把该标记视为记录的解析结果的一部分。 - **平台的版本组。** 覆盖版本组所列的包时照样生效,并给出警告,写出版本组的版本:平台给出默认值,工程可以选择另一个。 -- **何时需要。** 任何 SemVer 合并都无法调和的两个版本请求被拒绝时,会写出两者,并在末尾给出覆盖该包的那一行。两个来自 `[build-dependencies]` 的请求同样被拒绝,且不以重命名分开:构建依赖是构建程序的模块,一次构建只加载它的一个版本(2026.11.16.1)。库精确固定一个所有使用者共享的包(提供目标侧层或内核 ABI 的包)时,使用它的工程会得到提示,`mcpp publish` 拒绝库中的这种固定:范围把选择留给工程。 +- **何时需要。** 任何 SemVer 合并都无法调和的两个版本请求被拒绝时,会写出两者,并在末尾给出覆盖该包的那一行。同一个构建程序的解析中的两个请求同样被拒绝,消息写出该程序,且不以重命名分开:构建依赖是构建程序的模块,一个程序只加载它的一个版本(2026.11.16.1;自 2026.11.19.1 按程序,见下文)。库精确固定一个所有使用者共享的包(提供目标侧层或内核 ABI 的包)时,使用它的工程会得到提示,`mcpp publish` 拒绝库中的这种固定:范围把选择留给工程。 ### `path` 或 `git` 依赖的身份(mcpp 2026.9.14.2+) @@ -459,9 +459,25 @@ protobuf = { version = "35.1", tools = ["protoc"] } # 构建期间运行 (30 —— `build.mcpp`)。 **同一个包写在两张表中。** 一份清单在 `[dependencies]` 与 `[build-dependencies]` -中写同一个包时,这个包有两个实例,一个为目标编译,一个为构建程序编译。两者的版本都是 -锁文件记录的版本。它们的特性分别激活:只由 `[build-dependencies]` 边请求的特性 -不进入目标实例。 +中写同一个包时,这个包有两个实例,一个为目标编译,一个为构建程序编译。两个请求有共同 +容许的版本时,两者是同一个版本(见下文)。它们的特性分别激活:只由 `[build-dependencies]` +边请求的特性不进入目标实例。 + +**每个构建程序解析自己的构建依赖 *(mcpp 2026.11.19.1+)*。** 一个 C++ 程序只加载一个包的 +一个版本;两个程序之间没有这个约束。目标(选择的成员、它们经 `[dependencies]` 到达的包与平台 +加入的包)是一次解析,每个构建程序是另一次:一个包的 `build.mcpp`(或其规则合成的程序)连同它的 +`[build-dependencies]` 及其闭包,平台的插件连同平台的构建依赖。 + +- 一次解析内的请求照旧合并;不能合并的两个请求被拒绝,消息写出该程序(`in the build program of + app@0.1.0`)与每个请求者。 +- 构建程序的请求取目标或另一个程序已取得、且满足该请求的最新版本,都没有时才取索引中的最新版本; + 目标先解析。没有冲突的构建因此保持此前的版本与构建程序的身份。 +- 同一个包的两个版本只在程序之间并存:工程的 `build.mcpp` 可以用 `mcpp.plugins` 0.22.1,而平台的 + 插件用 0.26.1。`mcpp.lock` 把第二个版本记为单独的条目 `<名字>+b<摘要>`,其 `programs` 列出取得 + 它的程序。 +- 根的 `override = true` 对每一次解析生效。 + +此前一次构建的所有解析共用一张表,工程的构建程序只能使用平台插件所用的版本(mcpp#972)。 **包的条件表由哪一行读取 *(mcpp 2026.11.19.1+)*。** 一个包的 `[target..dependencies]` 由构建它的那一行读取(SPEC-004 §4.8)。目标的包由构建的目标读取。只经 `[build-dependencies]` @@ -487,7 +503,7 @@ protobuf = { version = "35.1", tools = ["protoc"] } # 构建期间运行 一个工具子构建再次请求正在构建的同一个工具时,在第一次重复处被拒绝,并给出 请求链。 -与 `[dev-dependencies]` 不同,构建期依赖**会**被传递遍历,并与工程一起解析, +与 `[dev-dependencies]` 不同,构建期依赖**会**被传递遍历,每个构建程序内 一个包一个版本:构建期依赖自己的依赖正是让它能工作的东西,它们与它一起服务构建程序。 一个 feature 可以为构建期请求划定范围,而无需另开一个声明处。 diff --git a/docs/zh/30-build-mcpp.md b/docs/zh/30-build-mcpp.md index ef02ec53f..8b7ebe560 100644 --- a/docs/zh/30-build-mcpp.md +++ b/docs/zh/30-build-mcpp.md @@ -1573,7 +1573,10 @@ int main() { 并像任何工作空间一样由 mcpp 规划与构建:每个库按完整的包编译——它的包含目录、宏定义、 特性、自己的依赖与实现单元都参与——`build.mcpp` 由同一个计划编译,因此 BMI 与导入它的 程序在标准、方言与编译器上的一致由构造保证。多个程序使用的库只为它们编译一次,标准不同的 -程序在另一张图中。在该目录中执行 `mcpp build` 即可复现构建程序的编译。编译出的程序另放一份在其包的 +程序在另一张图中。每个构建程序解析自己的 `[build-dependencies]`(05 —— 依赖,*每个构建程序解析 +自己的构建依赖*);取得同一个包的不同版本的程序在不同的图中,即以这些版本命名的 +`-<摘要>`(mcpp 2026.11.19.1+),不涉及这些包的程序留在 ``。 +在该目录中执行 `mcpp build` 即可复现构建程序的编译。编译出的程序另放一份在其包的 运行记录旁,即 `target/.build-mcpp/build.mcpp.bin`(Windows 上为 `build.mcpp.exe`),供以自选的环境手动运行。 程序在其**身份**变化时编译:引擎版本、宿主工具链、标准、程序文本,以及其图中的每个包—— diff --git a/docs/zh/35-platforms.md b/docs/zh/35-platforms.md index e3c16f4d4..91da480ad 100644 --- a/docs/zh/35-platforms.md +++ b/docs/zh/35-platforms.md @@ -160,10 +160,14 @@ error: openkal-musl is requested as '^0.27.0' by app, and the platform openkal-p 只为本机构建的包(工具的)以及从目录或仓库取得的包不受版本组约束。早于 2026.11.7.1 的 mcpp 改为在解析之后检查 版本组;一个家族以一次索引改动整组发布时,两者结果相同。 -版本组的版本是它所约束的请求的约束,参与每一次合并*(mcpp 2026.11.19.1+)*。构建程序与目标都请求的包只有一个版本: -构建程序的 `>=1.0, <2.0` 与版本组的 `=1.1.0` 给出 1.1.0,与两个请求到达的先后无关;已经取得更新版本的请求被替换 -(`Merged <包> 1.2.0 ⨯ 1.1.0 → v1.1.0`)。没有版本满足全部请求时,拒绝列出每一个请求及其请求者,版本组的约束以 -平台的名字列出。只有构建程序使用的包仍不受约束。 +版本组的版本是它所约束的请求的约束,参与目标的每一次合并*(mcpp 2026.11.19.1+)*。没有版本满足全部请求时,拒绝列出 +每一个请求及其请求者,版本组的约束以平台的名字列出。构建程序的请求在目标的请求之后解析,目标已取得的版本满足它时取该 +版本:构建程序的 `>=1.0, <2.0` 与版本组的 `=1.1.0` 并存时两者都取 1.1.0,与两者声明的先后无关。构建程序解析自己的 +构建依赖(05 —— 依赖),所以只有构建程序使用的包、或目标的版本不满足的请求,不受版本组约束。 + +**插件与工程的 `build.mcpp` 是两个程序 *(mcpp 2026.11.19.1+)*。** 两者各自解析构建依赖:平台的插件可以用 +`mcpp.plugins` 0.26.1,而工程的 `build.mcpp` 用 0.22.1,工程不再抄写平台内部构建工具的版本。`mcpp.lock` 把第二个 +版本记为条目 `<名字>+b<摘要>`,并写出 `programs = ["build:<包>@<版本>"]`。 ## 内置平台是一个版本 @@ -182,6 +186,17 @@ platform = "openkal-platform@0.28.0" # 一个版本,或一个系列:"openkal 之后的构建在系列容许时采用记录的版本。`mcpp update`(或 `mcpp update openkal-platform`)删除这条记录,下一次构建重新 取最新版本。 +系列也跳过版本组不容许本次构建某个请求的版本*(mcpp 2026.11.19.1+)*:构建以低于它的系列重新规划,报告写出该版本与 +拒绝它的请求, + +``` + Platform plat v0.2.0 is not usable by this build: its version group names fam 1.2.0, and proj requests '~1.1.0' + Platform plat ^0 → v0.1.0 (the newest this build can use) +``` + +所以平台的新版本改变版本组,不使以系列选择它的工程停止构建。没有可用的版本时构建被拒绝(`platform-version-group`), +消息列出每个版本与拒绝它的请求。精确写出的版本与 `mcpp.lock` 记录的版本按写出的取,其版本组的拒绝即构建的拒绝。 + ## 索引列出的行(2026.11.17.1+) 内置表中没有平台的行,即引擎不认识的行(如 `x86_64-uefi`)、`planned` 行(如 `i686-linux-musl`),以及内置表持有、 @@ -429,7 +444,7 @@ mcpp toolchain install --target x86_64-openkal # fetch and build what the | `platform-not-a-platform` | 所选的包没有 `[platform]` | 选择一个平台包,或 `none` | | `platform-target-unserved` | 平台不服务该三元组 | 为它的某个 `targets` 构建,或为消息给出的宿主行构建 | | `platform-tool-absent` | 链接或封装要运行的程序既未构建、也未找到 | 在 `[target.'cfg(platform = "…")'.tools]` 下写出构建它的包 | -| `platform-version-group` | 家族中某个包的另一个版本在图中 | 选择携带该版本的平台版本,或钉住版本组中的版本 | +| `platform-version-group` | 家族中某个包的另一个版本在图中,或系列中没有版本容许本次构建的请求 | 选择携带该版本的平台版本,或钉住版本组中的版本 | | `platform-phase-directive` | 插件陈述了平台阶段不接受的指令 | 改由包自己的构建程序陈述 | | `lld-required-absent` | 平台以 `ld.lld` 链接,而工具链没有它 | 使用平台的工具链 | diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index d75ca8562..e0ae5b600 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -601,7 +601,7 @@ replaced}` —— `origin` 与构建的状态行使用的是同一句话 | `platform-not-a-platform` | 被选为该目标平台的包没有 `[platform]` 段 *(2026.10.27.1+)* | | `platform-target-unserved` | 所选平台不服务该目标:三元组不在其 `targets` 中,宿主三元组也不在其 `[platform.hosted] targets` 中 *(2026.10.27.1+)* | | `platform-tool-absent` | 平台的链接或封装要运行的程序(ok 映像的 `okld`)既不由图构建,也不在本机 *(2026.10.27.1+;2026.10.26.2 中为 `image-tool-absent`)* | -| `platform-version-group` | 平台版本组中的一个包以另一个版本出现在图中 *(2026.10.27.1+)* | +| `platform-version-group` | 平台版本组中的一个包以另一个版本出现在图中 *(2026.10.27.1+)*,或平台系列中没有版本容许本次构建的请求 *(2026.11.19.1+)* | | `platform-phase-directive` | 平台插件陈述了其阶段不接受的指令 *(2026.10.27.1+)* | | `other` | 一个尚未被赋予记号的拒绝分支 | diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index d21302448..a931442c8 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -1086,6 +1086,12 @@ export struct BuildOverrides { // package and the second reads its `[platform]` from here at once // (SPEC-013 §7). Set by `prepare_build` itself, never by a caller. std::filesystem::path platform_root; + // THE RELEASES OF A PLATFORM SERIES THIS BUILD CANNOT USE (design + // 2026-11-19 §21 A5, SPEC-013 §14): each release an earlier pass took and + // the platform's version group refused, with the request that refused it. + // The series is taken below them. Set by `prepare_build` itself, never by + // a caller. + std::vector> platform_series_skipped; // THE PLATFORM THIS PREPARE'S TARGET USES, stated by the caller: the // host graph's platform, given to the build programs' sub-build (SPEC-013 // §6). Read as the flag tier; empty on every other prepare. diff --git a/src/build/prepare/dependency_cache.cpp b/src/build/prepare/dependency_cache.cpp new file mode 100644 index 000000000..9af3fd312 --- /dev/null +++ b/src/build/prepare/dependency_cache.cpp @@ -0,0 +1,559 @@ +// dependency_cache.cpp -- P13: the global dependency cache: whose sources are +// named, each package's key folded bottom-up, the units an entry serves and the +// entries a build writes. Split from plan.cpp at its own banner ("Global +// dependency cache") when naming a path dependency's sources (mcpp#974) took +// the file and the step past their limits. + +module mcpp.build.prepare; +import :state; + +import std; +import mcpp.diag; +import mcpp.home; +import mcpp.build.source_digest; +import nlohmann.json; +import mcpp.manifest; +import mcpp.toolchain.cppfly; +import mcpp.toolchain.model; +import mcpp.toolchain.triple; +import mcpp.build.plan; +import mcpp.build.cache_key; +import mcpp.graph; +import mcpp.ui; +import mcpp.bmi_cache; + +namespace mcpp::build { + +namespace ck = mcpp::build::cache_key; + +// WHETHER A PACKAGE'S SOURCES ARE NAMED, so that its build is shared through the +// global cache by that name (mcpp#974): an index package by its version, a git +// dependency by its resolved commit, a path dependency by the digest of its +// tree's content. The project's own trees -- the root, a selected member, any +// package below this build's root or the root of a build above it -- are named +// by nothing and built in place: they are what is being edited. +bool sources_named(const PrepareState& state, std::size_t i) { + if (i == 0 || i - 1 >= state.dep_cache_identities.size()) return false; + const auto& id = state.dep_cache_identities[i - 1]; + if (id.sourceKind == "version") return true; + if (id.sourceKind == "git") return !id.sourceRef.empty(); + if (id.sourceKind != "path" || state.packages[i].selectedMember) return false; + std::vector own = state.overrides.own_roots; + own.push_back(state.workRoot); + return !mcpp::build::path_is_under_any(state.packages[i].root, own); +} + +// The root's own `[dev-dependencies]` serve its tests, not its library, which a +// framework that uses the package under test reaches (names_the_root, +// graph_edges.cpp): the root's key does not fold them in. +static bool serves_root_tests(const DependencyEdge& e) { + return e.consumerPackageIndex == 0 && e.devTable; +} + +// The package graph the keys fold along: the declared edges, and the imports no +// declared edge covers. An import a declared edge already covers adds nothing. +// One that a declared path runs the other way is a module this build generates +// for the importer below its consumer's root, filed under the consumer by its +// path (e2e 849): not an upstream, and left out of `importedFrom` rather than +// closing a cycle. +static mcpp::graph::AdjacencyList key_graph(const PrepareState& state, + std::vector>& importedFrom) { + mcpp::graph::AdjacencyList pkgDeps(state.packages.size()); + for (auto& e : state.dependencyEdges) + if (e.consumerPackageIndex < pkgDeps.size() && !serves_root_tests(e)) + pkgDeps[e.consumerPackageIndex].push_back(e.dependencyPackageIndex); + for (std::size_t p = 0; p < importedFrom.size(); ++p) + std::erase_if(importedFrom[p], [&](std::size_t q) { + if (std::ranges::find(pkgDeps[p], q) != pkgDeps[p].end()) return true; + const auto above = mcpp::graph::closure(pkgDeps, std::array{q}, /*include_roots=*/false); + return std::ranges::find(above, p) != above.end(); + }); + for (std::size_t p = 0; p < importedFrom.size(); ++p) + pkgDeps[p].insert(pkgDeps[p].end(), importedFrom[p].begin(), importedFrom[p].end()); + return pkgDeps; +} + +namespace { + +// Each package's key and its inputs, and whether its upstream holds a package +// whose sources are not named. +struct PackageKeys { + std::vector keys; + std::vector inputs; + std::vector localTaint; +}; + +} // namespace + +static std::expected package_keys(PrepareState& state, BuildContext& ctx) { + // ── Per-package keys, bottom-up ────────────────────────────────── + // Axes A/B/C are whole-graph, so they are computed once. Axes D/E are + // per package. Axis F is each direct dependency's key, which forces a + // bottom-up order: `dependencyEdges` is a DAG (the modgraph validator + // rejects cycles), so a simple memoized recursion suffices — with an + // explicit in-progress guard so a cycle that slipped past validation + // fails loudly instead of recursing until the stack dies. + auto axes = ck::build_axes( + *state.tc, *state.m, state.stdFlagAndDialect, + mcpp::toolchain::cppfly::effective_dialect_flags( + *state.tc, state.m->cppStandard.experimental, + mcpp::manifest::dialect_flags(state.m->buildConfig)), + // ONE SLOT, BOTH PLATFORMS. See `min_platform_version`: a target + // is either Apple or Android, and the level selects which bionic + // symbols are visible, so two levels must be two build + // directories. + [&] { + auto tt = mcpp::toolchain::triple::parse(state.tc->targetTriple); + return tt ? min_platform_version(*state.m, *tt, state.tc->binaryPath) + : std::string{}; + }(), + // The GLOBAL registry root — the same one `fill_package_config` + // relativizes against below, so both halves of the key describe + // payload paths the same way. + state.storeRoots.empty() ? std::filesystem::path{} : state.storeRoots.front(), + // The bit `make_plan` decided and `compute_flags` emits. Reading + // it here rather than re-deriving is what keeps the objects a + // cache entry HOLDS and the objects a build ASKS FOR describable + // by one sentence. + ctx.plan.needsPic); + + // Sources belonging to each package, package-root-relative and sorted. + std::vector> pkgSources(state.packages.size()); + std::vector pkgProvidesModule(state.packages.size(), 0); + // The package each unit is filed under, and the package that provides + // each module: the import edges below read both. + std::vector unitPackage; + std::map moduleProvider; + for (auto& cu : ctx.plan.compileUnits) { + // Longest matching root wins. Package roots can nest — a workspace + // member lives under the workspace root — and taking the first match + // would file the member's sources under the outer package, putting + // them in the wrong key. (Index payloads live in the xpkgs store and + // cannot be shadowed this way, so no cached entry is affected today; + // resolving it by specificity rather than by iteration order is what + // keeps that true if roots ever move.) + std::size_t best = state.packages.size(); + std::size_t bestLen = 0; + std::string bestRel; + for (std::size_t p = 0; p < state.packages.size(); ++p) { + std::error_code ec; + auto rel = std::filesystem::relative(cu.source, state.packages[p].root, ec); + if (ec || rel.empty()) continue; + auto rels = rel.generic_string(); + if (rels.starts_with("..")) continue; + auto len = state.packages[p].root.generic_string().size(); + if (best == state.packages.size() || len > bestLen) { + best = p; bestLen = len; bestRel = std::move(rels); + } + } + unitPackage.push_back(best); + if (best != state.packages.size()) { + if (!cu.providesModule.empty()) { + pkgProvidesModule[best] = 1; + moduleProvider.emplace(cu.providesModule, best); + } + pkgSources[best].push_back(std::move(bestRel)); + } + } + for (auto& v : pkgSources) std::ranges::sort(v); + + // A package whose unit imports another package's module reads that + // package's BMI, whether a manifest states the edge or not: the engine + // interface (`mcpp`) the libraries of a build program import is named + // by none of theirs. Its key is an upstream as a declared + // dependency's is; without it an entry compiled against another mcpp + // version's interface was served and failed with "import 'mcpp' has + // CRC mismatch" (2026.10.27.1 -> 2026.10.28.1, mcpp.plugins). + std::vector> importedFrom(state.packages.size()); + for (std::size_t u = 0; u < unitPackage.size(); ++u) { + const auto p = unitPackage[u]; + if (p == state.packages.size()) continue; + for (auto const& m : ctx.plan.compileUnits[u].imports) + if (auto it = moduleProvider.find(m); it != moduleProvider.end() && it->second != p) + importedFrom[p].insert(it->second); + } + + PackageKeys out{std::vector(state.packages.size()), + std::vector(state.packages.size(), nlohmann::json::object()), + std::vector(state.packages.size(), 0)}; + auto& pkgKeys = out.keys; + auto& pkgInputs = out.inputs; + // Does this package's own transitive upstream contain anything that is + // not an immutable index payload? If so it cannot be cached either, even + // when the package itself is an index package. + // + // A key covers an upstream package by folding in that package's KEY, and + // a local package's key covers its file list but not its file CONTENTS — + // nothing could, without hashing a tree that may change between the hash + // and the compile. So editing a local upstream's source would leave a + // downstream entry looking valid. No index descriptor can declare a path + // dependency today, which makes this shape unreachable in practice; it is + // enforced structurally anyway, because "unreachable today" is how the + // transitive path-dep leak got in. + auto& localTaint = out.localTaint; + + // Axis F is each direct dependency's OWN key, which forces a + // bottom-up order: `dependencyEdges` is a DAG (the modgraph + // validator rejects cycles among module imports, but this is the + // package graph, which has no such upstream guard), so the fold + // below walks `mcpp::graph::topological_order` — dependencies before + // dependents — instead of recursing, with an explicit cycle check up + // front standing in for the old in-progress guard. + auto keyOrder = mcpp::graph::topological_order(key_graph(state, importedFrom)); + if (!keyOrder) + return std::unexpected(std::format( + "dependency cycle through package '{}' while computing " + "its build-cache key", + state.packages[keyOrder.error().cycle.front()].manifest.package.name)); + + for (auto idx : *keyOrder) { + ck::PackageAxes pa; + if (idx > 0 && idx - 1 < state.dep_cache_identities.size()) { + pa.indexName = state.dep_cache_identities[idx - 1].indexName; + pa.packageName = state.dep_cache_identities[idx - 1].packageName; + pa.version = state.dep_cache_identities[idx - 1].version; + // A git or path dependency's sources by their own name (mcpp#974), + // so its key, and every key that folds it in, moves with them. + if (sources_named(state, idx)) pa.version += source_identity_suffix(state, idx); + } + if (pa.packageName.empty()) { + // The root package, or a package with no resolution identity. + // It is never cached, but its key still has to exist because + // downstream packages fold it in via axis F. + pa.packageName = state.packages[idx].manifest.package.namespace_.empty() + ? state.packages[idx].manifest.package.name + : std::format("{}.{}", state.packages[idx].manifest.package.namespace_, + state.packages[idx].manifest.package.name); + } + if (pa.version.empty()) pa.version = state.packages[idx].manifest.package.version; + // The GLOBAL registry root — index 0 by construction above. Include + // dirs are relativized against it so a key survives a different + // MCPP_HOME; a project-local payload falls back to the `` + // prefix inside fill_package_config and is equally stable. + ck::fill_package_config(pa, state.packages[idx], + state.storeRoots.empty() ? std::filesystem::path{} + : state.storeRoots.front()); + pa.sources = pkgSources[idx]; + const auto qname = mcpp::build::qualified_package_name(state.packages[idx].manifest); + // A GCC BMI records its imports' paths as the module map spells + // them: the BMI directory and the package's instance, whose digest + // folds in its imports' (`plan.cppm`). See `bmiPathConvention`. + if (pkgProvidesModule[idx] && state.tc->compiler == mcpp::toolchain::CompilerId::GCC) { + auto inst = ctx.plan.instanceDirs.find(qname); + pa.bmiPathConvention = "$root=bmi-dir"; + if (inst != ctx.plan.instanceDirs.end()) + pa.bmiPathConvention += ";instance=" + inst->second; + } + // What `scope_lto` appended to this package's units is part of + // their command, so it is part of the entry's identity. + if (state.ltoWithheld.contains(qname)) { + pa.cflags.push_back(state.ltoOffWord); + pa.cxxflags.push_back(state.ltoOffWord); + } + if (!sources_named(state, idx)) localTaint[idx] = 1; + for (auto& e : state.dependencyEdges) { + if (e.consumerPackageIndex != idx || serves_root_tests(e)) continue; + // `idx`'s dependencies precede it in `keyOrder`, so their + // keys and taint are already folded in below. + auto const& up = pkgKeys[e.dependencyPackageIndex]; + if (!up.empty()) pa.upstreamKeys.push_back(up); + if (localTaint[e.dependencyPackageIndex]) localTaint[idx] = 1; + for (auto& f : e.requestedFeatures) pa.features.push_back(f); + } + for (auto q : importedFrom[idx]) { + if (!pkgKeys[q].empty()) pa.upstreamKeys.push_back(pkgKeys[q]); + if (localTaint[q]) localTaint[idx] = 1; + } + std::ranges::sort(pa.upstreamKeys); + pa.upstreamKeys.erase(std::unique(pa.upstreamKeys.begin(), + pa.upstreamKeys.end()), + pa.upstreamKeys.end()); + std::ranges::sort(pa.features); + pa.features.erase(std::unique(pa.features.begin(), pa.features.end()), + pa.features.end()); + + pkgKeys[idx] = ck::key_hex(axes, pa); + pkgInputs[idx] = ck::to_json(axes, pa); + } + return out; +} + +// One package's entry: its units are served from the cache when the entry holds +// what this build asks for; otherwise a task fills it after the build. +static void serve_or_fill(PrepareState& state, BuildContext& ctx, std::size_t i, const PackageKeys& keys) { + const auto& pkgRoot = state.packages[i]; + const auto* depIdent = i - 1 < state.dep_cache_identities.size() + ? &state.dep_cache_identities[i - 1] + : nullptr; + // Only index ("version") packages are cacheable, and the identity + // recorded at resolution time is the ONLY admissible evidence. + // + // The predicate this replaces looked the package up in the ROOT + // manifest's dependencies/dev-dependencies and skipped it when the + // spec was path/git. A transitively-reached package is in neither + // map, so `specIt == end()` left skipCache false and local sources + // were cached — with `indexName` falling back to defaultIndex, so a + // workspace member `B` landed on disk as `mcpplibs/B@0.1.0`. Its + // sources can then change without changing name@version, i.e. the + // cache key cannot see the change. + // + // Note the direction of the judgment: `mcpp add`'s existence gate + // admits anything it cannot disprove. A build cache must do the + // opposite — anything it cannot prove came from the immutable + // xpkgs store stays out, because the failure mode here is a + // silently wrong object rather than a rejected command. + // + // A GIT OR PATH DEPENDENCY NAMES ITS SOURCES TOO (mcpp#974): its + // resolved commit, or the digest of its tree's content, is in its + // key (`sources_named`), so a change to the sources is a change of + // key. The project's own trees are not named and stay out. + if (!depIdent || !sources_named(state, i)) return; + // ...and neither may anything it was built against be unnamed. + if (keys.localTaint[i]) return; + // ...and the package's sources must ACTUALLY be in the immutable + // store, not merely labelled as coming from it. + // + // The rule stated three paragraphs up is about provenance on disk; + // `sourceKind` is a label recorded at resolution time, which is a + // weaker proxy — and there is already a case where the two + // disagree. Multi-version mangling re-anchors a consumer package's + // root at `/target/.mangled//__self__` and REWRITES + // its sources (module/import declarations renamed) while leaving + // `sourceKind == "version"` and `localTaint` clear. Nothing about + // that copy is immutable or shareable. It stays out of the cache + // today only because axis F happens to fold in the mangled + // secondary's differing key — one axis away from serving objects + // compiled against renamed modules, which is the silent-wrong-`.o` + // failure this gate exists to prevent. + // + // Judge the location, not the label. + // + // LEXICALLY, not via std::filesystem::relative. `relative()` runs + // weakly_canonical on both sides, which RESOLVES SYMLINKS — and a + // store whose entries are symlinks into another store is ordinary + // (tests/e2e/_inherit_toolchain.sh builds exactly that, and so do + // CI caches that link a warm payload tree into a fresh + // MCPP_HOME). Canonicalizing turns + // `/registry/data/xpkgs/` into wherever the link points + // and the package stops looking like a store package at all. The + // question here is where the payload was INSTALLED, which is a + // statement about the path, not about the inode. + const bool fromStore = depIdent->sourceKind == "version"; + if (fromStore && !mcpp::build::path_is_under_any(pkgRoot.root, state.storeRoots)) + return; + + const auto& depName = depIdent->packageName; + const auto depVer = (depIdent->version.empty() + ? pkgRoot.manifest.package.version + : depIdent->version) + (fromStore ? std::string{} : source_identity_suffix(state, i)); + + auto bmiT = mcpp::toolchain::bmi_traits(*state.tc); + mcpp::bmi_cache::CacheKey key { + .cacheRoot = mcpp::home::cache_root(), + .indexName = depIdent->indexName, + .packageName = depName, + .version = depVer, + .keyHex = keys.keys[i], + .inputs = keys.inputs[i], + .bmiDirName = std::string(bmiT.bmiDir), + .manifestTag = std::string(bmiT.manifestPrefix), + }; + + // The artifacts this package contributes, and the compile units + // that produce them. Collected together so a hit can mark exactly + // those units — the artifact list alone would not say which edges + // must stop being compile edges. + mcpp::bmi_cache::DepArtifacts arts; + std::vector unitIdx; + bool addressable = true; + for (std::size_t u = 0; u < ctx.plan.compileUnits.size(); ++u) { + auto& cu = ctx.plan.compileUnits[u]; + std::error_code ec; + auto rel = std::filesystem::relative(cu.source, pkgRoot.root, ec); + if (ec || rel.empty()) continue; + auto rels = rel.string(); + if (rels.starts_with("..")) continue; // not under depRoot + + // ALL OR NOTHING. A unit plan.cppm could not give a + // machine-independent entry address to takes its whole package + // out of the cache, rather than leaving the package half + // staged. Mixing cached and freshly built artifacts within one + // package is the case GCC reports as a BMI CRC mismatch in a + // consumer three edges away, which is far harder to read than + // one extra compile. + if (cu.packageObjectRel.empty()) { addressable = false; break; } + + // The entry names a BMI by its module; the build places it + // below its package's directory (B1), and the entry is filled + // from there. + if (!cu.providesModule.empty()) { + std::string bmi; + for (char c : cu.providesModule) + bmi.push_back(c == ':' ? '-' : c); + bmi += std::string(bmiT.bmiExt); + if (!cu.bmiFile.empty() && cu.bmiFile != bmi) + arts.bmiPlacements.push_back({bmi, cu.bmiFile}); + arts.bmiFiles.push_back(std::move(bmi)); + } + arts.objFiles.push_back({cu.packageObjectRel.generic_string(), + cu.object}); + unitIdx.push_back(u); + } + if (!addressable) return; + + // Validate the entry against THIS build's artifact list, not + // against the entry's own (mcpp#344). Anything short of a full + // match is a miss — never a failure: the stage edges below are + // simply not emitted and the units compile normally. + auto probe = mcpp::bmi_cache::probe_cached(key, arts); + if (probe.ok) { + // Mark the units. The backend turns each into a stage_file + // edge; nothing is copied here. Copying behind ninja's back is + // exactly what made the old cache a no-op: the staged file was + // still declared as a compile edge's output, and an output with + // no .ninja_log command-line record is dirty, so every unit was + // recompiled while the CLI printed "Cached". + for (auto u : unitIdx) { + auto& cu = ctx.plan.compileUnits[u]; + cu.servedFromCache = true; + cu.cachedObject = mcpp::bmi_cache::cached_obj_path( + key, cu.packageObjectRel.generic_string()); + if (!cu.providesModule.empty()) { + std::string bmi; + for (char c : cu.providesModule) + bmi.push_back(c == ':' ? '-' : c); + bmi += std::string(bmiT.bmiExt); + cu.cachedBmi = mcpp::bmi_cache::cached_bmi_path(key, bmi); + } + } + mcpp::bmi_cache::touch_accessed(key); + state.usedCacheRecords.insert(key.entryFile().string()); + // By the name the package report gives it: a dependency key + // is not always its qualified name (`greet` for the default + // namespace's `mcpplibs.greet`), and the line then read + // `Compiling` for a package the cache served. + ctx.cachedDeps.push_back({mcpp::build::qualified_package_name(pkgRoot.manifest), depVer, + unitIdx.size()}); + return; // no populate task; it is already cached + } + // A valid entry that does not hold what we asked for means the + // entry and this build disagree about the layout under one key. + // After #344 that is unreachable; say so out loud if it ever + // happens again, because the alternative presentation is "the + // cache silently never hits", and a cache that lies about its own + // effectiveness went unnoticed for three months once already. + if (!probe.layoutMismatch.empty()) { + mcpp::ui::warning(std::format( + "build cache entry for {}@{} [{}] does not contain the " + "artifacts this build needs ({} of {} missing, e.g. `{}`); " + "treating it as a miss. Run `mcpp cache verify` for details.", + depName, depVer, key.keyHex, + probe.layoutMismatch.size(), + arts.bmiFiles.size() + arts.objFiles.size(), + probe.layoutMismatch.front())); + } + for (auto u : unitIdx) ctx.plan.compileUnits[u].fillsCache = true; + BuildContext::CacheTask task{ std::move(key), std::move(arts) }; + // The tree is read again before its entry is written: a source + // edited while it compiled is not stored under the digest of the + // sources before the edit (mcpp#974). + if (depIdent->sourceKind == "path") { + task.treeRoot = depIdent->sourceRef.empty() ? pkgRoot.root + : std::filesystem::path(depIdent->sourceRef); + task.treeRows = tree_digest_rows(task.treeRoot); + task.treeDigest = tree_content_digest(state, task.treeRoot); + } + ctx.depsToPopulate.push_back(std::move(task)); +} + +std::expected step13_dependency_cache(PrepareState& state, BuildContext& ctx) { + // ─── Global dependency cache: per-package keys, hit → stage edges ── + // + // Every index package gets a key over the axes that actually reach its + // compiler command lines (mcpp.build.cache_key), computed bottom-up so a + // package's key includes its direct dependencies' keys. A hit marks that + // package's compile units `servedFromCache`, and the ninja backend emits + // `stage_file` edges instead of compile edges for them — which is the only + // way ninja will accept a cached artifact. A miss records a populate task + // for after the build. + // + // `--cache=local|off` skips this block entirely: nothing is read and, in + // run_build_plan, nothing is written. + auto cfg2 = state.get_cfg(true); + if (!cfg2 || ctx.cacheMode != CacheMode::Global) return {}; + std::error_code mkEc; + std::filesystem::create_directories(ctx.outputDir, mkEc); + + // NOTE (mcpp#344): there is deliberately no local "derive the entry + // address from the object path" helper here any more. There used to be + // one, and it was the SECOND derivation of a fact plan.cppm already + // owns — it stripped `obj/` off the consumer's build path, so the entry + // layout followed the consumer's package mix while the key did not. + // `CompileUnit::packageObjectRel` is now the only answer to "where does + // this object live inside a cache entry", and it is computed in exactly + // one place. Do not reintroduce a second one. + + auto keys = package_keys(state, ctx); + if (!keys) return std::unexpected(keys.error()); + for (std::size_t i = 1; i < state.packages.size(); ++i) // skip [0] = main + serve_or_fill(state, ctx, i, *keys); + return {}; +} + +// AN EDITED TREE'S PREVIOUS ENTRY IS REPLACED (mcpp#974), as a `path` host +// tool's is (tool_store::write_entry, mcpp#874). Every edit of a `path` +// dependency is a new key, and none would be collected before `[cache] +// unused_days`: the tree an entry was built from is written beside it, and an +// entry of the same package from the same tree under another digest is removed +// as this one is written. Another checkout of the package is another tree, and +// keeps its entries. +static void replace_tree_siblings(const mcpp::bmi_cache::CacheKey& key, const std::filesystem::path& tree) { + namespace fs = std::filesystem; + const auto plus = key.version.find("+path."); + if (plus == std::string::npos) return; + const auto origin = tree.lexically_normal().generic_u8string(); + const std::string text(origin.begin(), origin.end()); + std::error_code ec; + std::ofstream(key.dir() / "source", std::ios::binary | std::ios::trunc) << text; + const auto versionDir = key.dir().parent_path(); + const auto prefix = std::format("{}@{}", key.packageName, key.version.substr(0, plus + 6)); + auto from_this_tree = [&](const fs::path& entry) { + std::ifstream in(entry / "source", std::ios::binary); + return in && std::string{std::istreambuf_iterator(in), {}} == text; + }; + for (auto const& v : fs::directory_iterator(versionDir.parent_path(), ec)) { + const auto name = v.path().filename().u8string(); + if (v.path() == versionDir || !std::string(name.begin(), name.end()).starts_with(prefix)) continue; + std::error_code kec; + for (auto const& entry : fs::directory_iterator(v.path(), kec)) + if (from_this_tree(entry.path())) fs::remove_all(entry.path(), kec); + fs::remove(v.path(), kec); // only when nothing else is left in it + } +} + +void populate_dependency_cache(BuildContext& ctx) { + if (ctx.cacheMode != CacheMode::Global) ctx.depsToPopulate.clear(); + for (auto& task : ctx.depsToPopulate) { + if (!task.treeRoot.empty()) { + if (mcpp::build::source_tree_digest(task.treeRoot, task.treeRows) != task.treeDigest) { + const auto tree = task.treeRoot.lexically_normal().generic_u8string(); + mcpp::diag::degraded("build/cache", std::format( + "the sources of {} changed while it was built ({}), so its build was not stored in the global cache", + task.key.packageName, std::string(tree.begin(), tree.end())), + "the next build compiles it again, from the sources it then finds"); + continue; + } + } + auto pr = mcpp::bmi_cache::populate_from(task.key, ctx.outputDir, task.artifacts); + if (pr && !task.treeRoot.empty()) replace_tree_siblings(task.key, task.treeRoot); + if (!pr) { + mcpp::ui::warning(std::format( + "bmi cache populate failed for {}@{}: {}", + task.key.packageName, task.key.version, pr.error())); + } + } + // Once: a second drive of the same plan has nothing new to add. + ctx.depsToPopulate.clear(); +} + +} // namespace mcpp::build diff --git a/src/build/prepare/driver.cpp b/src/build/prepare/driver.cpp index f7af3fb2f..da101b008 100644 --- a/src/build/prepare/driver.cpp +++ b/src/build/prepare/driver.cpp @@ -66,6 +66,10 @@ prepare_build_pass(bool print_fingerprint, bool includeDevDeps, std::vector extraTargets, BuildOverrides overrides); + +// A series is a handful of releases; a bound keeps a pass that refuses every +// release it is given from restarting without end. +constexpr std::size_t kMaxSeriesSkips = 32; } // namespace // TWO PASSES WHEN THE BUILD PROGRAM STATES THE TOOLCHAIN (mcpp#755). @@ -94,9 +98,20 @@ prepare_build(bool print_fingerprint, if (first) { (void)take_toolchain_restart(); (void)take_platform_restart(); + (void)take_platform_series_restart(); (void)take_target_restart(); return first; } + // A RELEASE OF A PLATFORM SERIES WHOSE VERSION GROUP THE BUILD DOES NOT + // ADMIT (design 2026-11-19 §21 A5): the series is taken below it. The + // package a pass fetched for it is not the platform the next one reads. + if (auto skip = take_platform_series_restart(); + skip && overrides.platform_series_skipped.size() < kMaxSeriesSkips) { + overrides.platform_series_skipped.push_back(std::move(*skip)); + overrides.platform_root.clear(); + return prepare_build(print_fingerprint, includeDevDeps, std::move(extraTargets), + std::move(overrides)); + } if (auto identity = take_target_restart(); identity && !overrides.target_request) { overrides.target_request = overrides.target_triple; overrides.target_triple = std::move(*identity); @@ -251,7 +266,12 @@ resolve_graph(bool includeDevDeps, BuildOverrides overrides) { // A restart a phase asks of `prepare_build` (a toolchain, platform or // target statement) belongs to a build; the graph is read as it stands. struct NoRestart { - ~NoRestart() { (void)take_toolchain_restart(); (void)take_platform_restart(); (void)take_target_restart(); } + ~NoRestart() { + (void)take_toolchain_restart(); + (void)take_platform_restart(); + (void)take_platform_series_restart(); + (void)take_target_restart(); + } } noRestart; if (auto r = phase0_manifest_and_workspace(state); !r) return std::unexpected(r.error()); if (auto r = check_engine_floors(state, /*rootOnly=*/true); !r) return std::unexpected(r.error()); @@ -259,7 +279,18 @@ resolve_graph(bool includeDevDeps, BuildOverrides overrides) { if (auto r = phase2_define_toolchain_resolver(state); !r) return std::unexpected(r.error()); if (auto r = phase3_xlings_before_graph(state); !r) return std::unexpected(r.error()); if (auto r = phase4a_graph_load(state); !r) return std::unexpected(r.error()); - if (auto r = phase4b_graph_worklist(state); !r) return std::unexpected(r.error()); + if (auto r = phase4b_graph_worklist(state); !r) { + // The graph a build would plan: a platform series below the releases + // the build cannot use (§21 A5), as `prepare_build` takes it. + if (auto skip = take_platform_series_restart(); + skip && state.overrides.platform_series_skipped.size() < kMaxSeriesSkips) { + auto overrides = state.overrides; + overrides.platform_series_skipped.push_back(std::move(*skip)); + overrides.platform_root.clear(); + return resolve_graph(includeDevDeps, std::move(overrides)); + } + return std::unexpected(r.error()); + } if (auto r = check_engine_floors(state, /*rootOnly=*/false); !r) return std::unexpected(r.error()); return std::move(state.packages); } diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index 29dbf9b66..b0835c53a 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -104,6 +104,26 @@ aggregatedRequest(PrepareState& state, std::size_t depPkgIndex, Instance instanc return { std::move(feats), anyEdge ? anyDefault : true }; } +// An entry the reading below turns into a dependency of a library belongs to +// each resolution that holds the library (design 2026-11-19 §21 A1), as an +// entry of `[dependencies]` resolved where its consumer is would: the walk +// resolved it as the library's own build program's, which has no program. +static void mirror_into_library_scopes(PrepareState& state, std::size_t p, std::size_t d) { + std::optional> dependency; + for (auto const& [k, r] : state.resolved) + if (r.depIndex + 1 == d) { dependency.emplace(k, r); break; } + if (!dependency) return; + std::vector scopes; + if (state.packages[p].selectedMember) scopes.push_back({}); + for (auto const& [k, r] : state.resolved) + if (r.depIndex + 1 == p) scopes.push_back(k.scope); + for (auto const& scope : scopes) { + auto key = dependency->first; + key.scope = scope; + state.resolved.try_emplace(std::move(key), dependency->second); + } +} + // COMPAT(plugin-imports-from-build-dependencies). Before mcpp#797 a build // plugin's library was compiled inside its consumer's build program, so a // package it listed in its own `[build-dependencies]` with `host-module = true` @@ -155,6 +175,7 @@ static void step6_compat_library_edges(PrepareState& state) { pm.dependencies.emplace(r.key, asDependency); if (p - 1 < state.dep_manifests.size() && state.dep_manifests[p - 1]) state.dep_manifests[p - 1]->dependencies.emplace(r.key, asDependency); + mirror_into_library_scopes(state, p, d); for (auto const& e : state.dependencyEdges) if (e.consumerPackageIndex == p && e.dependencyPackageIndex == d && e.buildTable) { auto copy = e; @@ -274,6 +295,17 @@ PackageInstances package_instances(const PrepareState& state) { stack.push_back(d); } } + // A slot only build programs' resolutions hold is a module of a build + // program and never the target's (design 2026-11-19 §21 A1), whichever + // edges reach it. + std::vector targetRecord(n, false), programRecord(n, false); + for (auto const& [k, r] : state.resolved) + if (r.depIndex + 1 < n) (k.scope.empty() ? targetRecord : programRecord)[r.depIndex + 1] = true; + for (std::size_t i = 1; i < n; ++i) + if (programRecord[i] && !targetRecord[i] && !state.packages[i].selectedMember) { + target[i] = false; + host[i] = true; + } return out; } @@ -1959,17 +1991,12 @@ static std::expected step6_dependency_build_programs(PrepareS bpEnv.requested = pkg.selectedMember; // A registry or git dependency: its tree is the package store's, // and its program may not watch files outside it (#766). - { - auto rn = mcpp::pm::compat::resolve_package_name( - pkg.manifest.package.name, pkg.manifest.package.namespace_); - for (auto const& ns : {rn.namespace_, std::string(mcpp::pm::kDefaultNamespace), - std::string{}}) - if (auto it = state.resolved.find(ResolvedKey{ns, rn.shortName}); - it != state.resolved.end()) { - bpEnv.sealedPackage = it->second.source != "path"; - break; - } - } + // The record of this slot, whichever resolution holds it (§21 A1). + for (auto const& [k, rec] : state.resolved) + if (rec.depIndex + 1 == i) { + bpEnv.sealedPackage = rec.source != "path"; + break; + } // THE PLATFORM'S PLUGIN (SPEC-013 §4.2): the platform package's // program runs in the `platform` phase, and states only the // platform's build, link and runner fields. diff --git a/src/build/prepare/graph.cpp b/src/build/prepare/graph.cpp index ab79cccee..0d27a9430 100644 --- a/src/build/prepare/graph.cpp +++ b/src/build/prepare/graph.cpp @@ -98,141 +98,6 @@ void withdraw_link_flags(PrepareState& state, std::size_t p) { if (rec.depIndex + 1 == p) removeLinkFlags(state, rec.linkFlagsAdded); } -static std::expected, std::string> package_source_files( - const std::filesystem::path& srcRoot, - const mcpp::manifest::Manifest& depManifest) -{ - // Resolve the source globs against the original root, falling - // back to the convention default if the manifest didn't set any. - std::vector globs = depManifest.modules.sources; - if (globs.empty()) { - // Was a fourth hand-written copy of the convention default, and it - // had already drifted: all three assembly extensions were missing, - // so staging a dependency with .S/.s/.asm silently dropped them. - globs = mcpp::default_source_globs( - mcpp::extension_table_for(depManifest.buildConfig.moduleExtensions, - depManifest.buildConfig.deviceExtensions)); - } - // As the scanner selects (`select_sources`). - auto sourceFiles = mcpp::modgraph::select_sources(srcRoot, globs); - if (sourceFiles.empty()) { - return std::unexpected(std::format( - "stage: no source files found under '{}' (globs={})", - srcRoot.string(), globs.size())); - } - return sourceFiles; -} - - // Stage a dep's source files into a fresh directory, rewriting their - // module / import declarations against `rename`. Used by the multi- - // version mangling fallback (Level 1) so two cross-major copies of - // the same package can coexist with distinct module names. - // - // Headers reached through `[build].include_dirs` are NOT staged — those - // keep pointing at the original install dir via absolutized include paths. - // - // HEADERS BESIDE A SOURCE ARE A DIFFERENT CASE, AND THEY ARE STAGED. - // - // `#include "detail.h"` is resolved relative to the directory of the file - // holding the directive, so moving the source moves the search. No - // `include_dirs` entry is involved and absolutizing one cannot help: the - // package never declared a path because it never needed one. Measured - // before this, on a package whose `src/time.cpp` includes `src/sbi.h`: - // - // target/.mangled/openkal-opensbi/__self__/src/time.cpp:44:10: - // fatal error: 'sbi.h' file not found - // - // THE DIAGNOSIS THIS PRODUCES POINTS AT THE WRONG THING. The path in it - // is a staging directory the author never wrote, for a header sitting - // exactly where the source expects it, and the build that triggered it - // asked for nothing unusual — two majors of one dependency is a supported - // arrangement, and this is its most ordinary consequence. - // - // What is copied is every file in a directory that contains a staged - // source and is not itself staged, verbatim: rewriting applies to module - // declarations, and a header has none. Directories with no staged source - // are not visited, so this stays proportional to what is being staged. -static std::expected stage_with_rewrite( - const std::filesystem::path& srcRoot, - const std::filesystem::path& dstRoot, - const mcpp::manifest::Manifest& depManifest, - const std::map& rename) -{ - std::error_code ec; - std::filesystem::create_directories(dstRoot, ec); - if (ec) return std::unexpected(std::format( - "stage: cannot create '{}': {}", dstRoot.string(), ec.message())); - - auto sources = package_source_files(srcRoot, depManifest); - if (!sources) return std::unexpected(sources.error()); - - for (auto const& f : *sources) { - auto rel = std::filesystem::relative(f, srcRoot, ec); - if (ec) return std::unexpected(std::format( - "stage: cannot relativize '{}': {}", f.string(), ec.message())); - auto dst = dstRoot / rel; - std::filesystem::create_directories(dst.parent_path(), ec); - - std::ifstream is(f); - if (!is) return std::unexpected(std::format( - "stage: cannot read '{}'", f.string())); - std::stringstream buf; buf << is.rdbuf(); - std::string content = buf.str(); - - std::string out = mcpp::pm::rewrite_module_decls(content, rename); - std::ofstream os(dst); - if (!os) return std::unexpected(std::format( - "stage: cannot write '{}'", dst.string())); - os << out; - } - - // The files beside those sources, carried across unchanged so a quoted - // include still finds what it named. - std::set sourceDirs; - for (auto const& f : *sources) sourceDirs.insert(f.parent_path()); - for (auto const& dir : sourceDirs) { - for (auto const& entry : std::filesystem::directory_iterator(dir, ec)) { - if (ec) break; - if (!entry.is_regular_file()) continue; - if (sources->contains(entry.path())) continue; - auto rel = std::filesystem::relative(entry.path(), srcRoot, ec); - if (ec) continue; - auto dst = dstRoot / rel; - std::filesystem::create_directories(dst.parent_path(), ec); - std::filesystem::copy_file( - entry.path(), dst, - std::filesystem::copy_options::overwrite_existing, ec); - if (ec) return std::unexpected(std::format( - "stage: cannot copy '{}': {}", - entry.path().string(), ec.message())); - } - ec.clear(); - } - return {}; -} - -static std::expected, std::string> declared_modules_for( - const std::filesystem::path& srcRoot, - const mcpp::manifest::Manifest& depManifest) -{ - auto sources = package_source_files(srcRoot, depManifest); - if (!sources) return std::unexpected(sources.error()); - std::vector modules; - for (auto const& file : *sources) { - std::ifstream is(file); - if (!is) return std::unexpected(std::format( - "mangle: cannot read '{}'", file.string())); - std::stringstream buf; buf << is.rdbuf(); - for (auto& name : mcpp::pm::declared_module_roots(buf.str())) { - if (std::ranges::find(modules, name) == modules.end()) - modules.push_back(std::move(name)); - } - } - if (modules.empty()) return std::unexpected(std::format( - "mangle: package '{}' declares no named C++ module to rewrite", - depManifest.package.name)); - return modules; -} // Stage 2a — feature-activated optional dependencies. Static file-scope // functions (mcpp#722 / T6 split), not local lambdas as originally @@ -482,6 +347,7 @@ struct WorklistItemCtx { bool rootHeld = false; }; + // The identity a root declaration already holds for a dependency's request // by version, under the request's namespace, or the default namespace and none // for a bare name; nothing when the root declares no such package, or the @@ -600,6 +466,13 @@ step4b_resolve_identity(PrepareState& state, WorklistItemCtx& ctx) { spec.version = "=" + pin->second; } if (auto r = step4b_pin_to_platform_group(state, item, name, spec); !r) return r; + // A PROGRAM TAKES A VERSION THE BUILD ALREADY HOLDS WHERE ITS OWN + // REQUEST ALLOWS (§21 A1): the newest version another program or the + // target resolved that the request admits, so a build without a + // conflict keeps the versions, and the build programs' identities, it + // had when the two shared one resolution. + if (!ctx.rootHeld && !item.scope.empty() && spec.isVersion()) + if (auto held = held_version_for(state, item, spec, name)) spec.version = *held; // Pin SemVer constraint before dedup/fetch. if (!ctx.rootHeld) if (auto r = state.resolveSemver(spec, name); !r) { @@ -627,6 +500,7 @@ step4b_resolve_identity(PrepareState& state, WorklistItemCtx& ctx) { spec.patchesDigest = *digest; key.variant = *digest; } + key.scope = item.scope; auto& sourceKind = ctx.sourceKind; sourceKind = spec.isPath() ? "path" @@ -644,10 +518,12 @@ step4b_resolve_identity(PrepareState& state, WorklistItemCtx& ctx) { if (auto clone = state.gitCloneBySource.find(source); clone != state.gitCloneBySource.end()) namesMember = state.gitMemberDeclaring(clone->second.root, key).has_value(); + auto unscoped = key; + unscoped.scope.clear(); if (auto bySource = state.identityBySource.find(source); !namesMember && bySource != state.identityBySource.end() - && !(bySource->second == key)) { - const auto& existing = state.resolved.at(bySource->second); + && !(bySource->second == unscoped)) { + const auto& existing = *any_record_of(state, bySource->second); const auto& declaring = state.declaringManifest.at(bySource->second); if (!declaring.namespaceDeclared) { return std::unexpected(std::format( @@ -663,6 +539,7 @@ step4b_resolve_identity(PrepareState& state, WorklistItemCtx& ctx) { } state.reportAdoption(item, key, bySource->second); key = bySource->second; + key.scope = item.scope; state.stateAdoptedIdentity(item, key); } } @@ -744,15 +621,20 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, // with a message about the package rather than about the // two requests (a project asking for mcpp:plugins 0.21.0 // over a platform that asks for 0.20.0). - if (item.fromBuildTable) { + // ONE PROGRAM, ONE VERSION (§21 A4): the requests of one + // build program's resolution disagree, and the message names + // the program. Two programs never reach here; each resolves + // its own `[build-dependencies]` closure. + if (item.fromBuildTable || !key.scope.empty()) { return std::unexpected(std::format( - "build dependency '{}{}{}' has irreconcilable versions:\n" + "build dependency '{}{}{}' has irreconcilable versions in {}:\n" " '{}', resolved for these requests:{}\n" " '{}' (constraint '{}') requested by '{}'\n" "SemVer merge: {}\n" - "A build dependency is a module of a build program, and a build " - "loads one version of it. Align the requests on one version{}.", + "A build dependency is a module of a build program, and a program " + "loads one version of it. Align the program's requests on one version{}.", key.ns, key.ns.empty() ? "" : ".", key.shortName, + scope_display(key.scope), it->second.version, requests_of(state, it->second), spec.version, request_constraint(item), item.requestedBy, merged.error(), @@ -930,7 +812,7 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, if (!secPackage) return std::unexpected(secPackage.error()); state.packages.push_back(std::move(*secPackage)); recordDependencyEdge(state, item.consumerDepIndex, depPackageIndex, - spec, item.fromBuildTable, name, item.targetRowOnly); + spec, item.fromBuildTable, name, item.targetRowOnly, item.scope); auto linkFlagsAdded = propagateLinkFlags(state, secStage, *state.dep_manifests.back()); ResolvedKey mangledKey{key.ns, mangledPackage}; @@ -967,10 +849,25 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, // no re-fetch needed; just record this consumer edge. recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name, item.targetRowOnly); + spec, item.fromBuildTable, name, item.targetRowOnly, item.scope); return {}; } + // A PROGRAM'S RECORD THAT MOVES LEAVES A SLOT OTHERS HOLD (§21 + // A1): the target or another program keeps the version it has, + // and this program takes the merged one in a slot of its own; + // the request is resolved again at that version, and the edges + // the program alone made are moved there (finish_scope_move). + if (!key.scope.empty() && slot_held_elsewhere(state, it->second.depIndex, key)) { + PrepareState::ScopeMove move{it->second.depIndex + 1, it->second.constraintsBy}; + state.scopeMoves[key] = std::move(move); + state.resolved.erase(it); + WorkItem again = item; + again.spec.version = *merged; + again.originalConstraint = "=" + *merged; + state.worklist.push_front(std::move(again)); + return {}; + } // Merged version differs from the previously-pinned one. // Re-fetch the dep at the merged version and replace the // earlier slot in dep_manifests / packages so the build plan @@ -1029,7 +926,7 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, state.packages[it->second.depIndex + 1] = std::move(*mergedPackage); recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name, item.targetRowOnly); + spec, item.fromBuildTable, name, item.targetRowOnly, item.scope); it->second.version = *merged; it->second.linkFlagsAdded = std::move(linkFlagsAdded); @@ -1047,6 +944,7 @@ step4b_identity_version_merge(PrepareState& state, WorklistItemCtx& ctx, state.worklist.push_back({child_name, child_spec, newLabel, child_spec.version, it->second.depIndex, {}, item.devOnly}); + state.worklist.back().scope = item.scope; } return {}; return {}; @@ -1066,8 +964,14 @@ step4b_walk_other_side(PrepareState& state, std::size_t depIndex, const WorkItem const unsigned char side = item.buildOnly ? kHostSide : kTargetSide; // A slot no walk has marked is one of the paths that walk no children // (a package of programs, a mangled secondary): it stays unwalked. - if (state.depSidesWalked[depIndex] == 0 || (state.depSidesWalked[depIndex] & side)) return {}; + if (state.depSidesWalked[depIndex] == 0) return {}; + // The target walks a table once; each build program walks it once for its + // own resolution (§21 A1). + if (item.scope.empty() ? (state.depSidesWalked[depIndex] & side) != 0 + : state.scopesWalked.contains({depIndex, item.scope})) + return {}; state.depSidesWalked[depIndex] |= side; + if (!item.scope.empty()) state.scopesWalked.insert({depIndex, item.scope}); auto const& dm = *state.dep_manifests[depIndex]; auto requests = side_requests(state, dm, item.buildOnly); if (!requests) return std::unexpected(requests.error()); @@ -1094,6 +998,7 @@ step4b_walk_other_side(PrepareState& state, std::size_t depIndex, const WorkItem state.worklist.push_back({childName, childReq, label + " (build-dep)", childReq.version, depIndex, root, item.devOnly, /*buildOnly=*/true, /*fromBuildTable=*/true}); + state.worklist.back().scope = program_scope(label); } return {}; } @@ -1149,6 +1054,7 @@ step4b_widen_features(PrepareState& state, std::size_t depIndex, const WorkItem& continue; state.worklist.push_back({childName, now, label, now.version, depIndex, root, item.devOnly, item.buildOnly}); + state.worklist.back().scope = item.scope; } return {}; } @@ -1272,7 +1178,7 @@ step4b_handle_already_resolved(PrepareState& state, WorklistItemCtx& ctx, if (it->second.depIndex + 1 < state.packages.size()) { recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name, item.targetRowOnly); + spec, item.fromBuildTable, name, item.targetRowOnly, item.scope); } return step4b_widen_features(state, it->second.depIndex, item, spec); } @@ -1348,12 +1254,60 @@ step4b_handle_already_resolved(PrepareState& state, WorklistItemCtx& ctx, if (it->second.depIndex + 1 < state.packages.size()) { recordDependencyEdge(state, item.consumerDepIndex, it->second.depIndex + 1, - spec, item.fromBuildTable, name, item.targetRowOnly); + spec, item.fromBuildTable, name, item.targetRowOnly, item.scope); } return step4b_widen_features(state, it->second.depIndex, item, spec); return {}; } +// ONE SLOT PER PACKAGE AND VERSION, ONE RECORD PER RESOLUTION (§21 A1). A +// request of a resolution that holds no record of the package, answered by the +// version or the source another resolution already holds, takes that slot: the +// target's request a program resolved first, or a program's request the target +// or another program answered. Nothing is fetched again, and the slot's table +// is walked once for this resolution. Returns whether it did. +static std::expected +step4b_adopt_slot(PrepareState& state, WorklistItemCtx& ctx) { + auto& item = ctx.item; + const auto& key = ctx.key; + const auto& spec = item.spec; + const std::string source = ctx.sourceKind == "version" ? std::string{} + : sourceRefOf(state, ctx.sourceKind, spec, item.resolveRoot, item.originalConstraint); + const ResolvedRecord* held = nullptr; + for (auto const& [k, r] : state.resolved) { + if (k.scope == key.scope || k.ns != key.ns || k.shortName != key.shortName + || k.variant != key.variant || r.source != ctx.sourceKind) + continue; + if (ctx.sourceKind == "version" ? r.version != spec.version : r.sourceRef != source) continue; + held = &r; + break; + } + if (!held) return false; + const std::size_t depIndex = held->depIndex; + ResolvedRecord rec; + rec.version = held->version; + rec.requestedBy = item.requestedBy; + rec.source = held->source; + rec.sourceRef = held->sourceRef; + rec.fromRoot = state.declaredByRoot(item.consumerDepIndex); + rec.fromSelectedMember = item.consumerDepIndex != kMainConsumer && rec.fromRoot; + rec.overridden = rec.fromRoot && spec.override_; + rec.devOnly = item.devOnly; + rec.depIndex = depIndex; + rec.patches = held->patches; + if (ctx.sourceKind == "version") note_request(state, rec, item); + state.resolved[key] = std::move(rec); + if (depIndex + 1 < state.packages.size()) + recordDependencyEdge(state, item.consumerDepIndex, depIndex + 1, spec, item.fromBuildTable, + item.name, item.targetRowOnly, item.scope); + finish_scope_move(state, key, depIndex + 1); + if (ctx.sourceKind == "version" && depIndex < state.dep_manifests.size() && state.dep_manifests[depIndex]) + step4b_note_exact_pin(state, item, *state.dep_manifests[depIndex]); + if (auto r = step4b_walk_other_side(state, depIndex, item); !r) return std::unexpected(r.error()); + if (auto r = step4b_widen_features(state, depIndex, item, spec); !r) return std::unexpected(r.error()); + return true; +} + static std::expected step4b_acquire_dependency_source(PrepareState& state, WorklistItemCtx& ctx) { auto& item = ctx.item; @@ -1793,7 +1747,7 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { auto declaredName = mcpp::pm::compat::resolve_package_name( ctx.dep_manifest->package.name, ctx.dep_manifest->package.namespace_); ResolvedKey declared{ ctx.dep_manifest->package.namespace_, - declaredName.shortName }; + declaredName.shortName, {}, key.scope }; if (!(declared == key)) { state.reportAdoption(item, key, declared); state.stateAdoptedIdentity(item, declared); @@ -1940,16 +1894,19 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { } state.packages.push_back(std::move(*depPackage)); recordDependencyEdge(state, item.consumerDepIndex, depPackageIndex, spec, - item.fromBuildTable, name, item.targetRowOnly); + item.fromBuildTable, name, item.targetRowOnly, item.scope); // Record this dep as resolved so future encounters of the same // (ns, name) hit the fast path (skip / merge / conflict). if (sourceKind != "version") { + // The identity a source names, whichever program resolved it. + auto identity = key; + identity.scope.clear(); state.identityBySource.emplace( sourceRefOf(state, sourceKind, spec, item.resolveRoot, item.originalConstraint) + (gitMember.empty() ? std::string{} : "#member=" + gitMember), - key); - state.declaringManifest[key] = DeclaringManifest{ manifestPath, namespaceDeclared }; + identity); + state.declaringManifest[identity] = DeclaringManifest{ manifestPath, namespaceDeclared }; } state.resolved[key] = ResolvedRecord{ .version = sourceKind == "version" ? spec.version : "", @@ -1971,6 +1928,7 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { state.resolved[key].constraint.clear(); note_request(state, state.resolved[key], item); } + finish_scope_move(state, key, depPackageIndex); if (sourceKind == "version" && !state.dep_manifests.empty() && state.dep_manifests.back()) step4b_note_exact_pin(state, item, *state.dep_manifests.back()); @@ -2016,6 +1974,7 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { // first asks for the package (`step4b_walk_other_side`). if (state.depSidesWalked.size() <= selfIdx) state.depSidesWalked.resize(selfIdx + 1, 0); state.depSidesWalked[selfIdx] |= item.buildOnly ? kHostSide : kTargetSide; + if (!item.scope.empty()) state.scopesWalked.insert({selfIdx, item.scope}); auto side = side_requests(state, *state.dep_manifests.back(), item.buildOnly); if (!side) return std::unexpected(side.error()); for (auto& [child_name, child_spec] : side->dependencies) { @@ -2035,6 +1994,7 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { state.worklist.push_back({child_name, childReq, thisDepLabel + " (dev-dep)", childReq.version, selfIdx, ctx.dep_root, /*devOnly=*/true, item.buildOnly}); + state.worklist.back().scope = item.scope; } } // A dependency's own `[build-dependencies]` — the only channel through @@ -2056,10 +2016,12 @@ step4b_finalize_dependency(PrepareState& state, WorklistItemCtx& ctx) { childReq.version, selfIdx, ctx.dep_root, item.devOnly, /*buildOnly=*/true, /*fromBuildTable=*/true}); + state.worklist.back().scope = program_scope(thisDepLabel); } return {}; } + static std::expected step4b_cycle_check(PrepareState& state) { // ONE PLACE DETECTS A CYCLE OF PACKAGES, AND IT IS HERE, WHERE THE GRAPH // IS RESOLVED (#649 E6). The build-cache key walk was the only reader that @@ -2097,173 +2059,6 @@ static std::expected step4b_cycle_check(PrepareState& state) return {}; } -static std::expected -step4b_define_provisioning_closures(PrepareState& state) { - // Which dependency supplied the runner, for the exactly-one-provider - // error below. A name rather than a bool: the message has to name both. - // ONE PROVIDER PER RUNNER NAME. `runner` has had this rule since #544; - // a NAMED runner inherits it per name, because a board may legitimately - // supply `flash` while a different package supplies `monitor`. - - // The payload facts a build program receives: where each declared payload - // is, and where it came from (mcpp#755). One definition, in sources.cpp, - // beside the override and on-request rules it reads. - state.fillXpkgDirs = [&](mcpp::build::BuildProgramEnv& e, - const mcpp::manifest::Manifest& owner, - std::size_t consumer) { - fill_xpkg_env(state, e, owner, consumer); - }; - - // `linkForms` (#642 E2): when given, each dependency that has a resolved - // library form is also offered under exactly the names its directory is, - // so `dep_linkage(n)` answers for every `n` that `dep_dir(n)` answers for. - // Only the root's program passes it; see the root call site for why. - state.fillDepDirs = [&](mcpp::build::BuildProgramEnv& e, std::size_t consumer, - const std::map* linkForms = nullptr) { - if (consumer >= state.provisionGraph.visible.size()) return; - auto bind = state.bareBindingsFor(consumer); - for (auto const& [tail, b] : bind) { - if (auto note = prov::contest_note(tail, b); !note.empty()) - mcpp::diag::warning("provisions/ambiguous", note); - } - // Every spelling of `publishedNamesFor`: the manifest's name, the - // qualified name a manifest writing `namespace = "ns"` and - // `name = "fw"` is addressed by (#642: the framework's rule asks - // `dep_linkage("huxerui.huxerui")`), and the bound tail. A spelling two - // providers share (#852: `openmui.rules` and `gongfu.rules` are both - // `rules` by their manifests' names) is no package's: it is published - // as ambiguous, with the qualified names that do work. - std::map> byName; - for (auto const& pr : state.provisionGraph.visible[consumer]) { - if (pr.kind != prov::Kind::DepDir || pr.provider >= state.packages.size()) continue; - for (auto const& n : state.publishedNamesFor(pr.provider, bind)) { - auto& who = byName[n]; - if (std::ranges::find(who, pr.provider) == who.end()) who.push_back(pr.provider); - } - } - for (auto const& [n, who] : byName) { - if (who.size() > 1) { - std::string which; - for (auto p : who) { - auto const& pk = state.packages[p].manifest.package; - const bool qualified = pk.namespace_.empty() || pk.name.starts_with(pk.namespace_ + "."); - which += (which.empty() ? "" : ",") - + (qualified ? pk.name : pk.namespace_ + "." + pk.name); - } - e.depAmbiguous.emplace_back(n, std::move(which)); - continue; - } - const auto provider = who.front(); - e.depDirs.emplace_back(n, state.packages[provider].root); - if (linkForms) - if (auto f = linkForms->find(provider); f != linkForms->end()) - e.depLinkages.emplace_back(n, f->second); - } - }; - - // A declared build-graph node's Source outputs must be visible to the - // scan, so they are materialized as placeholders and joined to the source - // set here — the same two lists `generated=` feeds, for the same reason - // (the scanner walks the legacy modules.sources mirror). ninja overwrites - // the placeholder before the compile edge runs, because that compile - // depends on the action's output. - state.programOutDir = [&state](const std::filesystem::path& base, - const mcpp::manifest::Manifest& pm, - const std::vector& features) { - // What changes the program's environment and is known before it runs. - std::string id = std::format("target={}\nprofile={}\naccel={}\n", - state.resolvedTargetCanonical, state.effectiveProfile, state.resolvedAccel()); - if (state.tc) - id += std::format("toolchain={}@{}\n", state.tc->binaryPath.generic_string(), - state.tc->version); - auto sorted = features; - std::ranges::sort(sorted); - for (auto const& f : sorted) id += "feature=" + f + "\n"; - std::string name = pm.package.namespace_.empty() - ? pm.package.name : pm.package.namespace_ + "." + pm.package.name; - for (auto& c : name) - if (!std::isalnum(static_cast(c)) && c != '.' && c != '-' && c != '_') c = '_'; - return base / "target" / ".build-mcpp" / "out" // named in `uses.list` (mcpp#874) - / *state.usedProgramOut.insert(mcpp::toolchain::hash_string(id)).first / (name.empty() ? std::string("root") : name); - }; - - state.adoptActionOutputs = [](mcpp::manifest::Manifest& mm, - const std::filesystem::path& pkgRoot, - std::size_t firstNewAction) { - if (firstNewAction >= mm.buildConfig.actions.size()) return; - std::vector fresh( - mm.buildConfig.actions.begin() - + static_cast(firstNewAction), - mm.buildConfig.actions.end()); - // The package that DECLARED the outputs classifies them: a dependency - // generating a `.ixx` asks its own manifest, not the root project's. - // Built once per package, not once per output. - const auto pkgExtTable = - mcpp::extension_table_for(mm.buildConfig.moduleExtensions, - mm.buildConfig.deviceExtensions); - mcpp::build::directives::prepare_actions(fresh, pkgRoot); - std::copy(fresh.begin(), fresh.end(), - mm.buildConfig.actions.begin() - + static_cast(firstNewAction)); - for (auto const& a : fresh) { - if (a.role != mcpp::manifest::BuildAction::Role::Source) continue; - for (auto const& o : a.outputs) { - if (o.find("${mcpp.") != std::string::npos) continue; - // Companion outputs (protoc's .pb.h next to its .pb.cc) are - // produced by the edge but are NOT translation units. - if (!mcpp::build::directives::is_compilable_output(o, pkgExtTable)) - continue; - mm.buildConfig.sources.push_back(o); - mm.modules.sources.push_back(o); - // A declared unit: the scan takes what the action says the - // file provides and imports, and reads nothing (D6). A unit - // that declares no `provides` asserts nothing; the build's own - // scan finds its imports, as it did with the empty placeholder. - mm.modules.declaredUnits[o] = mcpp::manifest::ScanOverride{a.provides, a.imports}; - // D26: a generated module interface must say which module it - // is. Without `.provides()` nothing writes its BMI, and the - // first importer fails far from the action. - if (a.provides.empty() - && mcpp::classify(o, pkgExtTable) == mcpp::SourceKind::ModuleInterface) - mcpp::diag::degraded("build-program/provides", std::format( - "action '{}' generates the module-interface file '{}' and declares no " - "`.provides(...)`", a.id, o), - "the module it defines has no BMI in this build, so an import of it fails", - "declare the module the generated file defines: " - "mcpp::action(...).provides(\"\")"); - } - } - }; - - - state.appendUniqueFlags = - [](std::vector& flags, - const std::vector& additions) -> bool - { - bool changed = false; - for (auto const& f : additions) { - if (std::find(flags.begin(), flags.end(), f) != flags.end()) continue; - flags.push_back(f); - changed = true; - } - return changed; - }; - - - - - - { - auto rootPackage = makePackageRoot(state, *state.root, *state.m); - if (!rootPackage) return std::unexpected(rootPackage.error()); - state.packages[0] = std::move(*rootPackage); - } - - - define_usage_requirements(state); - - return {}; -} std::expected phase4b_graph_worklist(PrepareState& state) { @@ -2399,14 +2194,27 @@ std::expected phase4b_graph_worklist(PrepareState& state) { state.worklist.push_back({n, req, mainPkgLabel + " (build-dep)", req.version, kMainConsumer, {}, /*devOnly=*/false, /*buildOnly=*/true, /*fromBuildTable=*/true}); + state.worklist.back().scope = root_program_scope(state); } auto drain = [&]() -> std::expected { - while (!state.worklist.empty()) { + while (!state.worklist.empty() || !state.scopedWorklist.empty()) { + // A PROGRAM'S REQUESTS WAIT FOR THE TARGET'S (§21 A1): the target's + // versions are chosen by the target's requests alone, and a program + // takes them where its own requests allow. + if (state.worklist.empty()) { + for (auto& w : state.scopedWorklist) state.worklist.push_back(std::move(w)); + state.scopedWorklist.clear(); + } WorklistItemCtx ctx; ctx.item = std::move(state.worklist.front()); state.worklist.pop_front(); + if (!ctx.item.scope.empty() + && std::ranges::any_of(state.worklist, [](const WorkItem& w) { return w.scope.empty(); })) { + state.scopedWorklist.push_back(std::move(ctx.item)); + continue; + } if (state.supersededRequesters.contains(ctx.item.requestedBy)) { mcpp::log::verbose("resolve", std::format( "{} is not walked: {}, which asked for it, was replaced by a merge", @@ -2423,6 +2231,21 @@ std::expected phase4b_graph_worklist(PrepareState& state) { return std::unexpected(r.error()); continue; } + // A program's request of a package the root holds by path or git + // takes the root's checkout, as the target's does (T2, §21 A1). + if (ctx.rootHeld && !ctx.key.scope.empty()) { + auto target = ctx.key; + target.scope.clear(); + if (auto it = state.resolved.find(target); it != state.resolved.end()) { + if (auto r = step4b_handle_already_resolved(state, ctx, it); !r) + return std::unexpected(r.error()); + continue; + } + } + if (auto adopted = step4b_adopt_slot(state, ctx); !adopted) + return std::unexpected(adopted.error()); + else if (*adopted) + continue; if (auto r = step4b_acquire_dependency_source(state, ctx); !r) return std::unexpected(r.error()); @@ -2454,6 +2277,7 @@ std::expected phase4b_graph_worklist(PrepareState& state) { } if (auto r = step4b_check_platform_versions(state); !r) return r; step4b_platform_member_edges(state); + step4b_drop_program_edges_from_target(state); if (auto r = step4b_cycle_check(state); !r) return std::unexpected(r.error()); diff --git a/src/build/prepare/graph_edges.cpp b/src/build/prepare/graph_edges.cpp index f8df58c45..9a95d7ecc 100644 --- a/src/build/prepare/graph_edges.cpp +++ b/src/build/prepare/graph_edges.cpp @@ -34,7 +34,8 @@ void recordDependencyEdge( const mcpp::manifest::DependencySpec& spec, bool buildTable, const std::string& writtenKey, - bool targetRowOnly) + bool targetRowOnly, + const std::string& scope) { const auto consumerPackageIndex = packageIndexForConsumer(consumerDepIndex); if (consumerPackageIndex >= state.packages.size() @@ -121,6 +122,9 @@ void recordDependencyEdge( it->embed = it->embed || spec.embed; for (auto const& x : spec.exclude) if (std::ranges::find(it->exclude, x) == it->exclude.end()) it->exclude.push_back(x); + // The target's resolution makes the edge as well: it is no longer + // one program's alone (§21 A1). + if (scope.empty()) it->scope.clear(); return; } state.dependencyEdges.push_back(DependencyEdge{ @@ -139,6 +143,7 @@ void recordDependencyEdge( .exclude = spec.exclude, }); state.dependencyEdges.back().targetRowOnly = targetRowOnly; + state.dependencyEdges.back().scope = scope; } // THE PACKAGE BEING BUILT STANDS FOR ITSELF IN ITS TESTS' GRAPH. A dependency @@ -153,7 +158,8 @@ bool names_the_root(PrepareState& state, const WorkItem& item, const ResolvedKey if (!item.devOnly || item.consumerDepIndex == kMainConsumer || root.virtualRoot || !key.variant.empty() || key.shortName != root.name || key.ns != root.namespace_) return false; - recordDependencyEdge(state, item.consumerDepIndex, 0, item.spec, item.fromBuildTable, item.name); + recordDependencyEdge(state, item.consumerDepIndex, 0, item.spec, item.fromBuildTable, item.name, + /*targetRowOnly=*/false, item.scope); if (!item.originalConstraint.empty() && item.originalConstraint != root.version && item.originalConstraint != "*") mcpp::ui::note(std::format( diff --git a/src/build/prepare/graph_provisioning.cpp b/src/build/prepare/graph_provisioning.cpp new file mode 100644 index 000000000..4c15b954d --- /dev/null +++ b/src/build/prepare/graph_provisioning.cpp @@ -0,0 +1,190 @@ +// graph_provisioning.cpp -- P4b: the closures a build program's environment is +// filled by (its payloads, its dependencies' directories, its output +// directory, the outputs its actions declare), and the root package. Split +// from graph.cpp at the boundary of the program's environment, when the +// program scopes of resolution (design 2026-11-19 §21 A1) took the file past +// its limit. + +module mcpp.build.prepare; +import :state; + +import std; +import mcpp.diag; +import mcpp.manifest; +import mcpp.source_kind; +import mcpp.toolchain.fingerprint; +import mcpp.build.build_program; +import mcpp.build.directives; +import mcpp.build.provisions; + +namespace mcpp::build { + +std::expected +step4b_define_provisioning_closures(PrepareState& state) { + // Which dependency supplied the runner, for the exactly-one-provider + // error below. A name rather than a bool: the message has to name both. + // ONE PROVIDER PER RUNNER NAME. `runner` has had this rule since #544; + // a NAMED runner inherits it per name, because a board may legitimately + // supply `flash` while a different package supplies `monitor`. + + // The payload facts a build program receives: where each declared payload + // is, and where it came from (mcpp#755). One definition, in sources.cpp, + // beside the override and on-request rules it reads. + state.fillXpkgDirs = [&](mcpp::build::BuildProgramEnv& e, + const mcpp::manifest::Manifest& owner, + std::size_t consumer) { + fill_xpkg_env(state, e, owner, consumer); + }; + + // `linkForms` (#642 E2): when given, each dependency that has a resolved + // library form is also offered under exactly the names its directory is, + // so `dep_linkage(n)` answers for every `n` that `dep_dir(n)` answers for. + // Only the root's program passes it; see the root call site for why. + state.fillDepDirs = [&](mcpp::build::BuildProgramEnv& e, std::size_t consumer, + const std::map* linkForms = nullptr) { + if (consumer >= state.provisionGraph.visible.size()) return; + auto bind = state.bareBindingsFor(consumer); + for (auto const& [tail, b] : bind) { + if (auto note = prov::contest_note(tail, b); !note.empty()) + mcpp::diag::warning("provisions/ambiguous", note); + } + // Every spelling of `publishedNamesFor`: the manifest's name, the + // qualified name a manifest writing `namespace = "ns"` and + // `name = "fw"` is addressed by (#642: the framework's rule asks + // `dep_linkage("huxerui.huxerui")`), and the bound tail. A spelling two + // providers share (#852: `openmui.rules` and `gongfu.rules` are both + // `rules` by their manifests' names) is no package's: it is published + // as ambiguous, with the qualified names that do work. + std::map> byName; + for (auto const& pr : state.provisionGraph.visible[consumer]) { + if (pr.kind != prov::Kind::DepDir || pr.provider >= state.packages.size()) continue; + for (auto const& n : state.publishedNamesFor(pr.provider, bind)) { + auto& who = byName[n]; + if (std::ranges::find(who, pr.provider) == who.end()) who.push_back(pr.provider); + } + } + for (auto const& [n, who] : byName) { + if (who.size() > 1) { + std::string which; + for (auto p : who) { + auto const& pk = state.packages[p].manifest.package; + const bool qualified = pk.namespace_.empty() || pk.name.starts_with(pk.namespace_ + "."); + which += (which.empty() ? "" : ",") + + (qualified ? pk.name : pk.namespace_ + "." + pk.name); + } + e.depAmbiguous.emplace_back(n, std::move(which)); + continue; + } + const auto provider = who.front(); + e.depDirs.emplace_back(n, state.packages[provider].root); + if (linkForms) + if (auto f = linkForms->find(provider); f != linkForms->end()) + e.depLinkages.emplace_back(n, f->second); + } + }; + + // A declared build-graph node's Source outputs must be visible to the + // scan, so they are materialized as placeholders and joined to the source + // set here — the same two lists `generated=` feeds, for the same reason + // (the scanner walks the legacy modules.sources mirror). ninja overwrites + // the placeholder before the compile edge runs, because that compile + // depends on the action's output. + state.programOutDir = [&state](const std::filesystem::path& base, + const mcpp::manifest::Manifest& pm, + const std::vector& features) { + // What changes the program's environment and is known before it runs. + std::string id = std::format("target={}\nprofile={}\naccel={}\n", + state.resolvedTargetCanonical, state.effectiveProfile, state.resolvedAccel()); + if (state.tc) + id += std::format("toolchain={}@{}\n", state.tc->binaryPath.generic_string(), + state.tc->version); + auto sorted = features; + std::ranges::sort(sorted); + for (auto const& f : sorted) id += "feature=" + f + "\n"; + std::string name = pm.package.namespace_.empty() + ? pm.package.name : pm.package.namespace_ + "." + pm.package.name; + for (auto& c : name) + if (!std::isalnum(static_cast(c)) && c != '.' && c != '-' && c != '_') c = '_'; + return base / "target" / ".build-mcpp" / "out" // named in `uses.list` (mcpp#874) + / *state.usedProgramOut.insert(mcpp::toolchain::hash_string(id)).first / (name.empty() ? std::string("root") : name); + }; + + state.adoptActionOutputs = [](mcpp::manifest::Manifest& mm, + const std::filesystem::path& pkgRoot, + std::size_t firstNewAction) { + if (firstNewAction >= mm.buildConfig.actions.size()) return; + std::vector fresh( + mm.buildConfig.actions.begin() + + static_cast(firstNewAction), + mm.buildConfig.actions.end()); + // The package that DECLARED the outputs classifies them: a dependency + // generating a `.ixx` asks its own manifest, not the root project's. + // Built once per package, not once per output. + const auto pkgExtTable = + mcpp::extension_table_for(mm.buildConfig.moduleExtensions, + mm.buildConfig.deviceExtensions); + mcpp::build::directives::prepare_actions(fresh, pkgRoot); + std::copy(fresh.begin(), fresh.end(), + mm.buildConfig.actions.begin() + + static_cast(firstNewAction)); + for (auto const& a : fresh) { + if (a.role != mcpp::manifest::BuildAction::Role::Source) continue; + for (auto const& o : a.outputs) { + if (o.find("${mcpp.") != std::string::npos) continue; + // Companion outputs (protoc's .pb.h next to its .pb.cc) are + // produced by the edge but are NOT translation units. + if (!mcpp::build::directives::is_compilable_output(o, pkgExtTable)) + continue; + mm.buildConfig.sources.push_back(o); + mm.modules.sources.push_back(o); + // A declared unit: the scan takes what the action says the + // file provides and imports, and reads nothing (D6). A unit + // that declares no `provides` asserts nothing; the build's own + // scan finds its imports, as it did with the empty placeholder. + mm.modules.declaredUnits[o] = mcpp::manifest::ScanOverride{a.provides, a.imports}; + // D26: a generated module interface must say which module it + // is. Without `.provides()` nothing writes its BMI, and the + // first importer fails far from the action. + if (a.provides.empty() + && mcpp::classify(o, pkgExtTable) == mcpp::SourceKind::ModuleInterface) + mcpp::diag::degraded("build-program/provides", std::format( + "action '{}' generates the module-interface file '{}' and declares no " + "`.provides(...)`", a.id, o), + "the module it defines has no BMI in this build, so an import of it fails", + "declare the module the generated file defines: " + "mcpp::action(...).provides(\"\")"); + } + } + }; + + + state.appendUniqueFlags = + [](std::vector& flags, + const std::vector& additions) -> bool + { + bool changed = false; + for (auto const& f : additions) { + if (std::find(flags.begin(), flags.end(), f) != flags.end()) continue; + flags.push_back(f); + changed = true; + } + return changed; + }; + + + + + + { + auto rootPackage = makePackageRoot(state, *state.root, *state.m); + if (!rootPackage) return std::unexpected(rootPackage.error()); + state.packages[0] = std::move(*rootPackage); + } + + + define_usage_requirements(state); + + return {}; +} + +} // namespace mcpp::build diff --git a/src/build/prepare/graph_requests.cpp b/src/build/prepare/graph_requests.cpp index 360c43268..005a016d9 100644 --- a/src/build/prepare/graph_requests.cpp +++ b/src/build/prepare/graph_requests.cpp @@ -1,9 +1,11 @@ // graph_requests.cpp -- P4b: what a request of the dependency worklist is bound -// by, and which side of the build makes it. Split from graph.cpp at the -// boundary of a request's own facts, when a request learned to carry the -// platform's version group into its merges and a package's tables came to be -// read by the row each of its instances is built for (mcpp#956; SPEC-013 §7.5, -// SPEC-004 §4.8). +// by, which side of the build makes it, and which resolution it is of: the +// target's or a build program's (design 2026-11-19 §21 A1). Split from +// graph.cpp at the boundary of a request's own facts, when a request learned to +// carry the platform's version group into its merges and a package's tables +// came to be read by the row each of its instances is built for (mcpp#956; +// SPEC-013 §7.5, SPEC-004 §4.8); the resolutions' helpers followed, when they +// took graph.cpp past its limit. module mcpp.build.prepare; import :state; @@ -12,7 +14,9 @@ import std; import mcpp.build.prepare_inputs; import mcpp.manifest; import mcpp.pm.dep_spec; +import mcpp.pm.resolver; import mcpp.toolchain.triple; +import mcpp.version_req; namespace mcpp::build { @@ -137,7 +141,133 @@ WorkItem side_item(const SideRequests& side, const std::string& childName, WorkItem out{childName, childReq, label, childReq.version, consumer, root, item.devOnly, item.buildOnly || ofThisMachine, /*fromBuildTable=*/ofThisMachine}; out.targetRowOnly = !item.buildOnly && side.ofTheTargetAlone.contains(childName); + // A request of this machine's reading alone serves the consumer's build + // program; every other request is of the program its consumer's was for. + out.scope = item.scope.empty() && ofThisMachine ? program_scope(label) : item.scope; return out; } +std::string program_scope(std::string_view label) { + return std::format("build:{}", label); +} + +std::string root_program_scope(const PrepareState& state) { + auto const& pkg = state.m->package; + const bool qualified = pkg.namespace_.empty() || pkg.name.starts_with(pkg.namespace_ + "."); + return std::format("build:{}{}{}@{}", qualified ? "" : pkg.namespace_, qualified ? "" : ".", + pkg.name, pkg.version); +} + +std::string scope_display(std::string_view scope) { + if (scope.empty()) return "a build program"; + if (scope.starts_with("build:")) scope.remove_prefix(6); + return std::format("the build program of {}", scope); +} + +// A record of `identity` (its scope ignored): the target's first, else the +// first program's; null when no resolution holds the package. +const ResolvedRecord* any_record_of(const PrepareState& state, const ResolvedKey& identity) { + auto target = identity; + target.scope.clear(); + if (auto it = state.resolved.find(target); it != state.resolved.end()) return &it->second; + for (auto const& [k, r] : state.resolved) + if (k.ns == identity.ns && k.shortName == identity.shortName && k.variant == identity.variant) + return &r; + return nullptr; +} + +// Whether `version` answers `constraint` as a request does: a constraint by +// its comparators, a bare version as itself. +static bool request_admits(std::string_view constraint, std::string_view version) { + if (constraint.empty() || constraint == "*") return true; + if (!mcpp::pm::is_version_constraint(std::string(constraint))) return constraint == version; + auto req = mcpp::version_req::parse_req(constraint); + auto ver = mcpp::version_req::parse_version(version); + return req && ver && mcpp::version_req::matches(*req, *ver); +} + +// THE VERSION A PROGRAM'S REQUEST TAKES FROM THE BUILD (§21 A1): the newest +// version of the package that the target or another program resolved and the +// request admits, when the program has no record of the package yet. +std::optional held_version_for(const PrepareState& state, const WorkItem& item, + const mcpp::manifest::DependencySpec& spec, + const std::string& name) { + if (!spec.patches.empty()) return std::nullopt; + const std::string shortName = spec.shortName.empty() ? name : spec.shortName; + if (state.resolved.contains(ResolvedKey{spec.namespace_, shortName, {}, item.scope})) return std::nullopt; + const auto constraint = request_constraint(item); + std::optional best; + std::optional bestVersion; + for (auto const& [k, r] : state.resolved) { + if (k.scope == item.scope || k.ns != spec.namespace_ || k.shortName != shortName + || !k.variant.empty() || r.source != "version" || r.version.empty()) + continue; + if (!request_admits(constraint, r.version)) continue; + auto v = mcpp::version_req::parse_version(r.version); + if (!v) continue; + if (!bestVersion || *bestVersion < *v) { bestVersion = *v; best = r.version; } + } + return best; +} + +// Whether a slot serves a resolution other than `key`'s program: another +// program's record or the target's points at it, or an edge another made. +bool slot_held_elsewhere(const PrepareState& state, std::size_t depIndex, const ResolvedKey& key) { + for (auto const& [k, r] : state.resolved) + if (r.depIndex == depIndex && k.scope != key.scope) return true; + return std::ranges::any_of(state.dependencyEdges, [&](const DependencyEdge& e) { + return e.dependencyPackageIndex == depIndex + 1 && e.scope != key.scope; + }); +} + +// A program's record resolved again after a move (§21 A1): the edges the +// program alone made to the slot it left now reach the new one, and the record +// keeps every request it answered. +void finish_scope_move(PrepareState& state, const ResolvedKey& key, std::size_t newPackageIndex) { + auto mv = state.scopeMoves.find(key); + if (mv == state.scopeMoves.end()) return; + for (auto& e : state.dependencyEdges) + if (e.scope == key.scope && e.dependencyPackageIndex == mv->second.oldPackageIndex) + e.dependencyPackageIndex = newPackageIndex; + // An edge the move made equal to one already there is the same edge once. + std::vector kept; + for (auto& e : state.dependencyEdges) { + const bool seen = std::ranges::any_of(kept, [&](const DependencyEdge& k) { + return k.consumerPackageIndex == e.consumerPackageIndex + && k.dependencyPackageIndex == e.dependencyPackageIndex && k.visibility == e.visibility + && k.buildTable == e.buildTable && k.devTable == e.devTable; + }); + if (!seen) kept.push_back(std::move(e)); + } + state.dependencyEdges = std::move(kept); + if (auto rec = state.resolved.find(key); rec != state.resolved.end()) { + rec->second.constraintsBy = mv->second.constraintsBy; + rec->second.constraint.clear(); + for (auto const& [by, c] : rec->second.constraintsBy) + rec->second.constraint += (rec->second.constraint.empty() ? "" : ",") + c; + } + state.scopeMoves.erase(mv); +} + +// A PROGRAM'S EDGE FROM A PACKAGE OF THE TARGET TO A PACKAGE ONLY PROGRAMS HOLD +// (§21 A1). A program that walked a package the target also holds may resolve +// a dependency of it at another version; the edge is the program's, and the +// target's graph -- its instances, its links, its cache keys -- does not reach +// that version through it. The program's own graph is the build programs' +// sub-build, which resolves its closure from the program's pins. +void step4b_drop_program_edges_from_target(PrepareState& state) { + const auto n = state.packages.size(); + std::vector targetHeld(n, false); + if (n > 0) targetHeld[0] = true; + for (std::size_t i = 1; i < n; ++i) + if (state.packages[i].selectedMember) targetHeld[i] = true; + for (auto const& [k, r] : state.resolved) + if (k.scope.empty() && r.depIndex + 1 < n) targetHeld[r.depIndex + 1] = true; + std::erase_if(state.dependencyEdges, [&](const DependencyEdge& e) { + return !e.scope.empty() && !e.buildTable && e.consumerPackageIndex < n + && e.dependencyPackageIndex < n && targetHeld[e.consumerPackageIndex] + && !targetHeld[e.dependencyPackageIndex]; + }); +} + } // namespace mcpp::build diff --git a/src/build/prepare/graph_stage.cpp b/src/build/prepare/graph_stage.cpp new file mode 100644 index 000000000..3324212a8 --- /dev/null +++ b/src/build/prepare/graph_stage.cpp @@ -0,0 +1,154 @@ +// graph_stage.cpp -- P4b: what the multi-version rename (Level 1) writes: a +// package's sources staged with their module declarations rewritten, the files +// beside them carried across, and the modules a package declares. Split from +// graph.cpp at the boundary of the rename's files, when the program scopes of +// resolution (design 2026-11-19 §21 A1) took the file past its limit. + +module mcpp.build.prepare; +import :state; + +import std; +import mcpp.manifest; +import mcpp.source_kind; +import mcpp.modgraph.scanner; +import mcpp.pm.mangle; + +namespace mcpp::build { + +static std::expected, std::string> package_source_files( + const std::filesystem::path& srcRoot, + const mcpp::manifest::Manifest& depManifest) +{ + // Resolve the source globs against the original root, falling + // back to the convention default if the manifest didn't set any. + std::vector globs = depManifest.modules.sources; + if (globs.empty()) { + // Was a fourth hand-written copy of the convention default, and it + // had already drifted: all three assembly extensions were missing, + // so staging a dependency with .S/.s/.asm silently dropped them. + globs = mcpp::default_source_globs( + mcpp::extension_table_for(depManifest.buildConfig.moduleExtensions, + depManifest.buildConfig.deviceExtensions)); + } + // As the scanner selects (`select_sources`). + auto sourceFiles = mcpp::modgraph::select_sources(srcRoot, globs); + if (sourceFiles.empty()) { + return std::unexpected(std::format( + "stage: no source files found under '{}' (globs={})", + srcRoot.string(), globs.size())); + } + return sourceFiles; +} + + // Stage a dep's source files into a fresh directory, rewriting their + // module / import declarations against `rename`. Used by the multi- + // version mangling fallback (Level 1) so two cross-major copies of + // the same package can coexist with distinct module names. + // + // Headers reached through `[build].include_dirs` are NOT staged — those + // keep pointing at the original install dir via absolutized include paths. + // + // HEADERS BESIDE A SOURCE ARE A DIFFERENT CASE, AND THEY ARE STAGED. + // + // `#include "detail.h"` is resolved relative to the directory of the file + // holding the directive, so moving the source moves the search. No + // `include_dirs` entry is involved and absolutizing one cannot help: the + // package never declared a path because it never needed one. Measured + // before this, on a package whose `src/time.cpp` includes `src/sbi.h`: + // + // target/.mangled/openkal-opensbi/__self__/src/time.cpp:44:10: + // fatal error: 'sbi.h' file not found + // + // THE DIAGNOSIS THIS PRODUCES POINTS AT THE WRONG THING. The path in it + // is a staging directory the author never wrote, for a header sitting + // exactly where the source expects it, and the build that triggered it + // asked for nothing unusual — two majors of one dependency is a supported + // arrangement, and this is its most ordinary consequence. + // + // What is copied is every file in a directory that contains a staged + // source and is not itself staged, verbatim: rewriting applies to module + // declarations, and a header has none. Directories with no staged source + // are not visited, so this stays proportional to what is being staged. +std::expected stage_with_rewrite( + const std::filesystem::path& srcRoot, + const std::filesystem::path& dstRoot, + const mcpp::manifest::Manifest& depManifest, + const std::map& rename) +{ + std::error_code ec; + std::filesystem::create_directories(dstRoot, ec); + if (ec) return std::unexpected(std::format( + "stage: cannot create '{}': {}", dstRoot.string(), ec.message())); + + auto sources = package_source_files(srcRoot, depManifest); + if (!sources) return std::unexpected(sources.error()); + + for (auto const& f : *sources) { + auto rel = std::filesystem::relative(f, srcRoot, ec); + if (ec) return std::unexpected(std::format( + "stage: cannot relativize '{}': {}", f.string(), ec.message())); + auto dst = dstRoot / rel; + std::filesystem::create_directories(dst.parent_path(), ec); + + std::ifstream is(f); + if (!is) return std::unexpected(std::format( + "stage: cannot read '{}'", f.string())); + std::stringstream buf; buf << is.rdbuf(); + std::string content = buf.str(); + + std::string out = mcpp::pm::rewrite_module_decls(content, rename); + std::ofstream os(dst); + if (!os) return std::unexpected(std::format( + "stage: cannot write '{}'", dst.string())); + os << out; + } + + // The files beside those sources, carried across unchanged so a quoted + // include still finds what it named. + std::set sourceDirs; + for (auto const& f : *sources) sourceDirs.insert(f.parent_path()); + for (auto const& dir : sourceDirs) { + for (auto const& entry : std::filesystem::directory_iterator(dir, ec)) { + if (ec) break; + if (!entry.is_regular_file()) continue; + if (sources->contains(entry.path())) continue; + auto rel = std::filesystem::relative(entry.path(), srcRoot, ec); + if (ec) continue; + auto dst = dstRoot / rel; + std::filesystem::create_directories(dst.parent_path(), ec); + std::filesystem::copy_file( + entry.path(), dst, + std::filesystem::copy_options::overwrite_existing, ec); + if (ec) return std::unexpected(std::format( + "stage: cannot copy '{}': {}", + entry.path().string(), ec.message())); + } + ec.clear(); + } + return {}; +} + +std::expected, std::string> declared_modules_for( + const std::filesystem::path& srcRoot, + const mcpp::manifest::Manifest& depManifest) +{ + auto sources = package_source_files(srcRoot, depManifest); + if (!sources) return std::unexpected(sources.error()); + std::vector modules; + for (auto const& file : *sources) { + std::ifstream is(file); + if (!is) return std::unexpected(std::format( + "mangle: cannot read '{}'", file.string())); + std::stringstream buf; buf << is.rdbuf(); + for (auto& name : mcpp::pm::declared_module_roots(buf.str())) { + if (std::ranges::find(modules, name) == modules.end()) + modules.push_back(std::move(name)); + } + } + if (modules.empty()) return std::unexpected(std::format( + "mangle: package '{}' declares no named C++ module to rewrite", + depManifest.package.name)); + return modules; +} + +} // namespace mcpp::build diff --git a/src/build/prepare/host_graph.cpp b/src/build/prepare/host_graph.cpp index 601a88e43..ed7dd18c1 100644 --- a/src/build/prepare/host_graph.cpp +++ b/src/build/prepare/host_graph.cpp @@ -520,6 +520,8 @@ struct Program { std::string member; std::string identity; std::string group; // the group directory's name + // The versions its own resolution holds (design 2026-11-19 §21 A2). + std::map pins; }; // The record of the programs a group directory holds. @@ -544,8 +546,10 @@ void write_if_changed(const fs::path& file, const std::string& text) { fs::rename(tmp, file, ec); } -// The manifests of one group: the workspace and a member per program. -std::map resolution_pins(const PrepareState& state); +// The packages one resolution scope holds by version, at their versions. +std::map scope_pins(const PrepareState& state, std::string_view scope); +// The scope of package `i`'s build program (ResolvedKey::scope). +std::string program_scope_of(const PrepareState& state, std::size_t i); } // namespace @@ -588,7 +592,7 @@ namespace { void write_group(PrepareState& state, const fs::path& dir, const std::vector& programs, const std::string& standard, const fs::path& enginePackage, - const std::string& toolchainSpec) { + const std::string& toolchainSpec, const std::map& pins) { std::string root = std::format( "# Written by mcpp {}: the build programs of {} (mcpp#797).\n" "# `mcpp build` here reproduces their compile; it is rewritten on every build.\n" @@ -665,7 +669,7 @@ void write_group(PrepareState& state, const fs::path& dir, const std::vector resolution_pins(const PrepareState& state) { +// version one resolution scope holds: the target's for the empty scope, a +// build program's own otherwise (design 2026-11-19 §21 A2). +std::map scope_pins(const PrepareState& state, std::string_view scope) { std::map pins; for (auto const& [k, rec] : state.resolved) { - if (rec.depIndex >= state.dep_cache_identities.size()) continue; + if (k.scope != scope || rec.depIndex >= state.dep_cache_identities.size()) continue; auto const& ident = state.dep_cache_identities[rec.depIndex]; if (ident.sourceKind != "version" || rec.version.empty()) continue; pins[k.ns.empty() ? k.shortName : k.ns + "." + k.shortName] = rec.version; @@ -742,10 +747,34 @@ std::map resolution_pins(const PrepareState& state) { return pins; } +// The packages two build programs of this build hold at different versions: +// the programs that disagree on one cannot share a graph, which takes each +// package at one version. +std::set contested_packages(const PrepareState& state) { + std::map> versions; + for (auto const& [k, rec] : state.resolved) + if (!k.scope.empty() && !rec.version.empty() && rec.source == "version") + versions[k.ns.empty() ? k.shortName : k.ns + "." + k.shortName].insert(rec.version); + std::set out; + for (auto const& [name, vs] : versions) + if (vs.size() > 1) out.insert(name); + return out; +} + +std::string program_scope_of(const PrepareState& state, std::size_t i) { + if (i == 0) return root_program_scope(state); + for (auto const& [k, rec] : state.resolved) + if (rec.depIndex + 1 == i) + return program_scope(std::format("{}{}{}@{}", k.ns, k.ns.empty() ? "" : ".", k.shortName, + rec.source == "version" ? rec.version : rec.source)); + return {}; +} + // Build the group's programs `goals`: one sub-build, one ninja. std::expected, std::string> build_group(PrepareState& state, const fs::path& dir, const std::vector& all, - const std::vector& goals, const HostToolchain& tc) { + const std::vector& goals, const HostToolchain& tc, + const std::map& pins) { // The graph `mcpp emit build-database` describes, beside the one it builds. auto describe = [&](const BuildContext& ctx) { if (!state.overrides.describes()) return; @@ -783,7 +812,7 @@ build_group(PrepareState& state, const fs::path& dir, const std::vector std::make_shared(state.runtimeSelection); sub.inherited_runtime_binding = std::make_shared(state.runtimeBindingSnapshot); - sub.resolution_pins = resolution_pins(state); + sub.resolution_pins = pins; sub.build_program_graph = true; sub.host_programs_root = host_root(state); sub.project_index_root = state.projectIndexRoot(); @@ -1032,21 +1061,39 @@ ensure_build_programs(PrepareState& state, const std::vector& packa auto enginePackage = engine::ensure_package(); if (!enginePackage) return std::unexpected(enginePackage.error()); - // One group per standard: a BMI is used only by a compile that agrees on it. - std::map> groups; + // One group per standard: a BMI is used only by a compile that agrees on + // it. Within a standard, one group per set of versions of the packages the + // programs disagree on (§21 A2): a graph takes each package at one + // version. Named by those versions, a program finds its group again + // whichever programs a command builds with it; with no disagreement the + // group is the standard's alone, as before programs resolved apart. + const auto contested = contested_packages(state); + const auto targetPins = scope_pins(state, ""); + std::map, std::vector> groups; for (auto i : wanted) { Program p; p.index = i; p.source = program_source(state, i); p.member = member_name(state, i); p.identity = program_identity(state, i, p.source, *tc); + p.pins = scope_pins(state, program_scope_of(state, i)); const auto standard = standard_of(state, i); + std::string versions; + for (auto const& [name, version] : p.pins) + if (contested.contains(name)) versions += std::format("{}={}\n", name, version); p.group = mcpp::toolchain::hash_string( std::format("{}|{}|{}", tc->identity, standard, mcpp::MCPP_VERSION)).substr(0, 12); - groups[standard].push_back(std::move(p)); + if (!versions.empty()) p.group += "-" + mcpp::toolchain::hash_string(versions).substr(0, 12); + groups[{standard, versions}].push_back(std::move(p)); } - for (auto& [standard, programs] : groups) { + for (auto& [groupKey, programs] : groups) { + auto const& standard = groupKey.first; + // The versions the group's graph takes: its programs' own, which agree + // where they meet, and the target's for a package none of them holds. + auto pins = targetPins; + for (auto const& p : programs) + for (auto const& [name, version] : p.pins) pins[name] = version; // A group the build programs' graph needs is its sibling, named by its // depth: the group that asked holds its own directory's lock while it // builds. @@ -1101,8 +1148,8 @@ ensure_build_programs(PrepareState& state, const std::vector& packa // Described, not built: `mcpp emit build-database` lists the graph // of programs that are current too. if (state.overrides.describes()) { - write_group(state, dir, programs, standard, *enginePackage, tc->spec); - if (auto r = build_group(state, dir, programs, {}, *tc); !r) + write_group(state, dir, programs, standard, *enginePackage, tc->spec, pins); + if (auto r = build_group(state, dir, programs, {}, *tc, pins); !r) mcpp::log::verbose("host-graph", r.error()); } continue; @@ -1115,12 +1162,12 @@ ensure_build_programs(PrepareState& state, const std::vector& packa return std::unexpected(r.error()); } - write_group(state, dir, programs, standard, *enginePackage, tc->spec); + write_group(state, dir, programs, standard, *enginePackage, tc->spec, pins); const auto began = std::chrono::steady_clock::now(); for (auto const* p : stale) mcpp::build::progress::program_compiling(qualified(manifest_of(state, p->index)), p->index == 0 || state.packages[p->index].selectedMember); - auto built = build_group(state, dir, programs, stale, *tc); + auto built = build_group(state, dir, programs, stale, *tc, pins); if (!built) return std::unexpected(built.error()); const auto took = std::chrono::duration_cast( std::chrono::steady_clock::now() - began); diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index 313bd3d96..b60fa7a6b 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -1,6 +1,6 @@ // plan.cpp -- P13: the BuildContext: the plan, prebuilt dependencies, -// assembly units, Windows resources and the global cache. mcpp.lock and -// resolution.json are written by records.cpp. +// assembly units and Windows resources. mcpp.lock and resolution.json are +// written by records.cpp, the global dependency cache by dependency_cache.cpp. module mcpp.build.prepare; import :state; @@ -17,12 +17,10 @@ import mcpp.runtime.elf; // DT_NEEDED of a library the line names import mcpp.build.refusal; import mcpp.build.version_floor; import mcpp.home; -import mcpp.build.source_digest; import mcpplibs.os.platform; import mcpp.invocation; import mcpp.effects; import mcpp.build.capacity; -import nlohmann.json; import mcpp.log; import mcpp.manifest; import mcpp.source_kind; @@ -51,7 +49,6 @@ import mcpp.build.distribution; // dist::Role / dist::Contract to_string import mcpp.build.graph_shape; // #407: the graph says which mode wrote it import mcpp.build.runtime_validation; // declared artifact -> identity verdict import mcpp.build.cache_key; -import mcpp.graph; import mcpp.pack.abi_tag; // the tag a prebuilt dependency is checked against import mcpp.pack.prebuilt; // …and the check itself import mcpp.pack.stage_tree; // where `${mcpp.stage_dir}` points, and its manifest @@ -81,7 +78,6 @@ import mcpp.pm.lock_io; import mcpp.version_req; import mcpp.ui; import mcpp.log; -import mcpp.bmi_cache; namespace mcpp::build { @@ -97,22 +93,6 @@ namespace mcpp::build { static void step13_report_packages(PrepareState& state, BuildContext& ctx); -// WHETHER A PACKAGE'S SOURCES ARE NAMED, so that its build is shared through the -// global cache by that name (mcpp#974): an index package by its version, a git -// dependency by its resolved commit, a path dependency by the digest of its -// tree's content. The project's own trees -- the root, a selected member, any -// package below this build's root or the root of a build above it -- are named -// by nothing and built in place: they are what is being edited. -static bool sources_named(const PrepareState& state, std::size_t i) { - if (i == 0 || i - 1 >= state.dep_cache_identities.size()) return false; - const auto& id = state.dep_cache_identities[i - 1]; - if (id.sourceKind == "version") return true; - if (id.sourceKind == "git") return !id.sourceRef.empty(); - if (id.sourceKind != "path" || state.packages[i].selectedMember) return false; - std::vector own = state.overrides.own_roots; - own.push_back(state.workRoot); - return !mcpp::build::path_is_under_any(state.packages[i].root, own); -} static std::expected step13_source_packages(PrepareState& state, BuildContext& ctx) { { @@ -1591,445 +1571,6 @@ static std::expected step13_assembly_units(PrepareState& stat } -// The root's own `[dev-dependencies]` serve its tests, not its library, which a -// framework that uses the package under test reaches (names_the_root, -// graph_edges.cpp): the root's key does not fold them in. -static bool serves_root_tests(const DependencyEdge& e) { - return e.consumerPackageIndex == 0 && e.devTable; -} - -// The package graph the keys fold along: the declared edges, and the imports no -// declared edge covers. An import a declared edge already covers adds nothing. -// One that a declared path runs the other way is a module this build generates -// for the importer below its consumer's root, filed under the consumer by its -// path (e2e 849): not an upstream, and left out of `importedFrom` rather than -// closing a cycle. -static mcpp::graph::AdjacencyList key_graph(const PrepareState& state, - std::vector>& importedFrom) { - mcpp::graph::AdjacencyList pkgDeps(state.packages.size()); - for (auto& e : state.dependencyEdges) - if (e.consumerPackageIndex < pkgDeps.size() && !serves_root_tests(e)) - pkgDeps[e.consumerPackageIndex].push_back(e.dependencyPackageIndex); - for (std::size_t p = 0; p < importedFrom.size(); ++p) - std::erase_if(importedFrom[p], [&](std::size_t q) { - if (std::ranges::find(pkgDeps[p], q) != pkgDeps[p].end()) return true; - const auto above = mcpp::graph::closure(pkgDeps, std::array{q}, /*include_roots=*/false); - return std::ranges::find(above, p) != above.end(); - }); - for (std::size_t p = 0; p < importedFrom.size(); ++p) - pkgDeps[p].insert(pkgDeps[p].end(), importedFrom[p].begin(), importedFrom[p].end()); - return pkgDeps; -} - -static std::expected step13_dependency_cache(PrepareState& state, BuildContext& ctx) { - // ─── Global dependency cache: per-package keys, hit → stage edges ── - // - // Every index package gets a key over the axes that actually reach its - // compiler command lines (mcpp.build.cache_key), computed bottom-up so a - // package's key includes its direct dependencies' keys. A hit marks that - // package's compile units `servedFromCache`, and the ninja backend emits - // `stage_file` edges instead of compile edges for them — which is the only - // way ninja will accept a cached artifact. A miss records a populate task - // for after the build. - // - // `--cache=local|off` skips this block entirely: nothing is read and, in - // run_build_plan, nothing is written. - auto cfg2 = state.get_cfg(true); - if (cfg2 && ctx.cacheMode == CacheMode::Global) { - std::error_code mkEc; - std::filesystem::create_directories(ctx.outputDir, mkEc); - - // NOTE (mcpp#344): there is deliberately no local "derive the entry - // address from the object path" helper here any more. There used to be - // one, and it was the SECOND derivation of a fact plan.cppm already - // owns — it stripped `obj/` off the consumer's build path, so the entry - // layout followed the consumer's package mix while the key did not. - // `CompileUnit::packageObjectRel` is now the only answer to "where does - // this object live inside a cache entry", and it is computed in exactly - // one place. Do not reintroduce a second one. - - // ── Per-package keys, bottom-up ────────────────────────────────── - // Axes A/B/C are whole-graph, so they are computed once. Axes D/E are - // per package. Axis F is each direct dependency's key, which forces a - // bottom-up order: `dependencyEdges` is a DAG (the modgraph validator - // rejects cycles), so a simple memoized recursion suffices — with an - // explicit in-progress guard so a cycle that slipped past validation - // fails loudly instead of recursing until the stack dies. - namespace ck = mcpp::build::cache_key; - auto axes = ck::build_axes( - *state.tc, *state.m, state.stdFlagAndDialect, - mcpp::toolchain::cppfly::effective_dialect_flags( - *state.tc, state.m->cppStandard.experimental, - mcpp::manifest::dialect_flags(state.m->buildConfig)), - // ONE SLOT, BOTH PLATFORMS. See `min_platform_version`: a target - // is either Apple or Android, and the level selects which bionic - // symbols are visible, so two levels must be two build - // directories. - [&] { - auto tt = mcpp::toolchain::triple::parse(state.tc->targetTriple); - return tt ? min_platform_version(*state.m, *tt, state.tc->binaryPath) - : std::string{}; - }(), - // The GLOBAL registry root — the same one `fill_package_config` - // relativizes against below, so both halves of the key describe - // payload paths the same way. - state.storeRoots.empty() ? std::filesystem::path{} : state.storeRoots.front(), - // The bit `make_plan` decided and `compute_flags` emits. Reading - // it here rather than re-deriving is what keeps the objects a - // cache entry HOLDS and the objects a build ASKS FOR describable - // by one sentence. - ctx.plan.needsPic); - - // Sources belonging to each package, package-root-relative and sorted. - std::vector> pkgSources(state.packages.size()); - std::vector pkgProvidesModule(state.packages.size(), 0); - // The package each unit is filed under, and the package that provides - // each module: the import edges below read both. - std::vector unitPackage; - std::map moduleProvider; - for (auto& cu : ctx.plan.compileUnits) { - // Longest matching root wins. Package roots can nest — a workspace - // member lives under the workspace root — and taking the first match - // would file the member's sources under the outer package, putting - // them in the wrong key. (Index payloads live in the xpkgs store and - // cannot be shadowed this way, so no cached entry is affected today; - // resolving it by specificity rather than by iteration order is what - // keeps that true if roots ever move.) - std::size_t best = state.packages.size(); - std::size_t bestLen = 0; - std::string bestRel; - for (std::size_t p = 0; p < state.packages.size(); ++p) { - std::error_code ec; - auto rel = std::filesystem::relative(cu.source, state.packages[p].root, ec); - if (ec || rel.empty()) continue; - auto rels = rel.generic_string(); - if (rels.starts_with("..")) continue; - auto len = state.packages[p].root.generic_string().size(); - if (best == state.packages.size() || len > bestLen) { - best = p; bestLen = len; bestRel = std::move(rels); - } - } - unitPackage.push_back(best); - if (best != state.packages.size()) { - if (!cu.providesModule.empty()) { - pkgProvidesModule[best] = 1; - moduleProvider.emplace(cu.providesModule, best); - } - pkgSources[best].push_back(std::move(bestRel)); - } - } - for (auto& v : pkgSources) std::ranges::sort(v); - - // A package whose unit imports another package's module reads that - // package's BMI, whether a manifest states the edge or not: the engine - // interface (`mcpp`) the libraries of a build program import is named - // by none of theirs. Its key is an upstream as a declared - // dependency's is; without it an entry compiled against another mcpp - // version's interface was served and failed with "import 'mcpp' has - // CRC mismatch" (2026.10.27.1 -> 2026.10.28.1, mcpp.plugins). - std::vector> importedFrom(state.packages.size()); - for (std::size_t u = 0; u < unitPackage.size(); ++u) { - const auto p = unitPackage[u]; - if (p == state.packages.size()) continue; - for (auto const& m : ctx.plan.compileUnits[u].imports) - if (auto it = moduleProvider.find(m); it != moduleProvider.end() && it->second != p) - importedFrom[p].insert(it->second); - } - - std::vector pkgKeys(state.packages.size()); - std::vector pkgInputs(state.packages.size(), - nlohmann::json::object()); - std::string keyCycleError; - // Does this package's own transitive upstream contain anything that is - // not an immutable index payload? If so it cannot be cached either, even - // when the package itself is an index package. - // - // A key covers an upstream package by folding in that package's KEY, and - // a local package's key covers its file list but not its file CONTENTS — - // nothing could, without hashing a tree that may change between the hash - // and the compile. So editing a local upstream's source would leave a - // downstream entry looking valid. No index descriptor can declare a path - // dependency today, which makes this shape unreachable in practice; it is - // enforced structurally anyway, because "unreachable today" is how the - // transitive path-dep leak got in. - std::vector localTaint(state.packages.size(), 0); - - // Axis F is each direct dependency's OWN key, which forces a - // bottom-up order: `dependencyEdges` is a DAG (the modgraph - // validator rejects cycles among module imports, but this is the - // package graph, which has no such upstream guard), so the fold - // below walks `mcpp::graph::topological_order` — dependencies before - // dependents — instead of recursing, with an explicit cycle check up - // front standing in for the old in-progress guard. - auto keyOrder = mcpp::graph::topological_order(key_graph(state, importedFrom)); - if (!keyOrder) { - keyCycleError = std::format( - "dependency cycle through package '{}' while computing " - "its build-cache key", - state.packages[keyOrder.error().cycle.front()].manifest.package.name); - return std::unexpected(keyCycleError); - } - - for (auto idx : *keyOrder) { - ck::PackageAxes pa; - if (idx > 0 && idx - 1 < state.dep_cache_identities.size()) { - pa.indexName = state.dep_cache_identities[idx - 1].indexName; - pa.packageName = state.dep_cache_identities[idx - 1].packageName; - pa.version = state.dep_cache_identities[idx - 1].version; - // A git or path dependency's sources by their own name (mcpp#974), - // so its key, and every key that folds it in, moves with them. - if (sources_named(state, idx)) pa.version += source_identity_suffix(state, idx); - } - if (pa.packageName.empty()) { - // The root package, or a package with no resolution identity. - // It is never cached, but its key still has to exist because - // downstream packages fold it in via axis F. - pa.packageName = state.packages[idx].manifest.package.namespace_.empty() - ? state.packages[idx].manifest.package.name - : std::format("{}.{}", state.packages[idx].manifest.package.namespace_, - state.packages[idx].manifest.package.name); - } - if (pa.version.empty()) pa.version = state.packages[idx].manifest.package.version; - // The GLOBAL registry root — index 0 by construction above. Include - // dirs are relativized against it so a key survives a different - // MCPP_HOME; a project-local payload falls back to the `` - // prefix inside fill_package_config and is equally stable. - ck::fill_package_config(pa, state.packages[idx], - state.storeRoots.empty() ? std::filesystem::path{} - : state.storeRoots.front()); - pa.sources = pkgSources[idx]; - const auto qname = mcpp::build::qualified_package_name(state.packages[idx].manifest); - // A GCC BMI records its imports' paths as the module map spells - // them: the BMI directory and the package's instance, whose digest - // folds in its imports' (`plan.cppm`). See `bmiPathConvention`. - if (pkgProvidesModule[idx] && state.tc->compiler == mcpp::toolchain::CompilerId::GCC) { - auto inst = ctx.plan.instanceDirs.find(qname); - pa.bmiPathConvention = "$root=bmi-dir"; - if (inst != ctx.plan.instanceDirs.end()) - pa.bmiPathConvention += ";instance=" + inst->second; - } - // What `scope_lto` appended to this package's units is part of - // their command, so it is part of the entry's identity. - if (state.ltoWithheld.contains(qname)) { - pa.cflags.push_back(state.ltoOffWord); - pa.cxxflags.push_back(state.ltoOffWord); - } - if (!sources_named(state, idx)) localTaint[idx] = 1; - for (auto& e : state.dependencyEdges) { - if (e.consumerPackageIndex != idx || serves_root_tests(e)) continue; - // `idx`'s dependencies precede it in `keyOrder`, so their - // keys and taint are already folded in below. - auto const& up = pkgKeys[e.dependencyPackageIndex]; - if (!up.empty()) pa.upstreamKeys.push_back(up); - if (localTaint[e.dependencyPackageIndex]) localTaint[idx] = 1; - for (auto& f : e.requestedFeatures) pa.features.push_back(f); - } - for (auto q : importedFrom[idx]) { - if (!pkgKeys[q].empty()) pa.upstreamKeys.push_back(pkgKeys[q]); - if (localTaint[q]) localTaint[idx] = 1; - } - std::ranges::sort(pa.upstreamKeys); - pa.upstreamKeys.erase(std::unique(pa.upstreamKeys.begin(), - pa.upstreamKeys.end()), - pa.upstreamKeys.end()); - std::ranges::sort(pa.features); - pa.features.erase(std::unique(pa.features.begin(), pa.features.end()), - pa.features.end()); - - pkgKeys[idx] = ck::key_hex(axes, pa); - pkgInputs[idx] = ck::to_json(axes, pa); - } - - for (std::size_t i = 1; i < state.packages.size(); ++i) { // skip [0] = main - const auto& pkgRoot = state.packages[i]; - const auto* depIdent = i - 1 < state.dep_cache_identities.size() - ? &state.dep_cache_identities[i - 1] - : nullptr; - // Only index ("version") packages are cacheable, and the identity - // recorded at resolution time is the ONLY admissible evidence. - // - // The predicate this replaces looked the package up in the ROOT - // manifest's dependencies/dev-dependencies and skipped it when the - // spec was path/git. A transitively-reached package is in neither - // map, so `specIt == end()` left skipCache false and local sources - // were cached — with `indexName` falling back to defaultIndex, so a - // workspace member `B` landed on disk as `mcpplibs/B@0.1.0`. Its - // sources can then change without changing name@version, i.e. the - // cache key cannot see the change. - // - // Note the direction of the judgment: `mcpp add`'s existence gate - // admits anything it cannot disprove. A build cache must do the - // opposite — anything it cannot prove came from the immutable - // xpkgs store stays out, because the failure mode here is a - // silently wrong object rather than a rejected command. - // - // A GIT OR PATH DEPENDENCY NAMES ITS SOURCES TOO (mcpp#974): its - // resolved commit, or the digest of its tree's content, is in its - // key (`sources_named`), so a change to the sources is a change of - // key. The project's own trees are not named and stay out. - if (!depIdent || !sources_named(state, i)) continue; - // ...and neither may anything it was built against be unnamed. - if (localTaint[i]) continue; - // ...and the package's sources must ACTUALLY be in the immutable - // store, not merely labelled as coming from it. - // - // The rule stated three paragraphs up is about provenance on disk; - // `sourceKind` is a label recorded at resolution time, which is a - // weaker proxy — and there is already a case where the two - // disagree. Multi-version mangling re-anchors a consumer package's - // root at `/target/.mangled//__self__` and REWRITES - // its sources (module/import declarations renamed) while leaving - // `sourceKind == "version"` and `localTaint` clear. Nothing about - // that copy is immutable or shareable. It stays out of the cache - // today only because axis F happens to fold in the mangled - // secondary's differing key — one axis away from serving objects - // compiled against renamed modules, which is the silent-wrong-`.o` - // failure this gate exists to prevent. - // - // Judge the location, not the label. - // - // LEXICALLY, not via std::filesystem::relative. `relative()` runs - // weakly_canonical on both sides, which RESOLVES SYMLINKS — and a - // store whose entries are symlinks into another store is ordinary - // (tests/e2e/_inherit_toolchain.sh builds exactly that, and so do - // CI caches that link a warm payload tree into a fresh - // MCPP_HOME). Canonicalizing turns - // `/registry/data/xpkgs/` into wherever the link points - // and the package stops looking like a store package at all. The - // question here is where the payload was INSTALLED, which is a - // statement about the path, not about the inode. - const bool fromStore = depIdent->sourceKind == "version"; - if (fromStore && !mcpp::build::path_is_under_any(pkgRoot.root, state.storeRoots)) - continue; - - const auto& depName = depIdent->packageName; - const auto depVer = (depIdent->version.empty() - ? pkgRoot.manifest.package.version - : depIdent->version) + (fromStore ? std::string{} : source_identity_suffix(state, i)); - - auto bmiT = mcpp::toolchain::bmi_traits(*state.tc); - mcpp::bmi_cache::CacheKey key { - .cacheRoot = mcpp::home::cache_root(), - .indexName = depIdent->indexName, - .packageName = depName, - .version = depVer, - .keyHex = pkgKeys[i], - .inputs = pkgInputs[i], - .bmiDirName = std::string(bmiT.bmiDir), - .manifestTag = std::string(bmiT.manifestPrefix), - }; - - // The artifacts this package contributes, and the compile units - // that produce them. Collected together so a hit can mark exactly - // those units — the artifact list alone would not say which edges - // must stop being compile edges. - mcpp::bmi_cache::DepArtifacts arts; - std::vector unitIdx; - bool addressable = true; - for (std::size_t u = 0; u < ctx.plan.compileUnits.size(); ++u) { - auto& cu = ctx.plan.compileUnits[u]; - std::error_code ec; - auto rel = std::filesystem::relative(cu.source, pkgRoot.root, ec); - if (ec || rel.empty()) continue; - auto rels = rel.string(); - if (rels.starts_with("..")) continue; // not under depRoot - - // ALL OR NOTHING. A unit plan.cppm could not give a - // machine-independent entry address to takes its whole package - // out of the cache, rather than leaving the package half - // staged. Mixing cached and freshly built artifacts within one - // package is the case GCC reports as a BMI CRC mismatch in a - // consumer three edges away, which is far harder to read than - // one extra compile. - if (cu.packageObjectRel.empty()) { addressable = false; break; } - - // The entry names a BMI by its module; the build places it - // below its package's directory (B1), and the entry is filled - // from there. - if (!cu.providesModule.empty()) { - std::string bmi; - for (char c : cu.providesModule) - bmi.push_back(c == ':' ? '-' : c); - bmi += std::string(bmiT.bmiExt); - if (!cu.bmiFile.empty() && cu.bmiFile != bmi) - arts.bmiPlacements.push_back({bmi, cu.bmiFile}); - arts.bmiFiles.push_back(std::move(bmi)); - } - arts.objFiles.push_back({cu.packageObjectRel.generic_string(), - cu.object}); - unitIdx.push_back(u); - } - if (!addressable) continue; - - // Validate the entry against THIS build's artifact list, not - // against the entry's own (mcpp#344). Anything short of a full - // match is a miss — never a failure: the stage edges below are - // simply not emitted and the units compile normally. - auto probe = mcpp::bmi_cache::probe_cached(key, arts); - if (probe.ok) { - // Mark the units. The backend turns each into a stage_file - // edge; nothing is copied here. Copying behind ninja's back is - // exactly what made the old cache a no-op: the staged file was - // still declared as a compile edge's output, and an output with - // no .ninja_log command-line record is dirty, so every unit was - // recompiled while the CLI printed "Cached". - for (auto u : unitIdx) { - auto& cu = ctx.plan.compileUnits[u]; - cu.servedFromCache = true; - cu.cachedObject = mcpp::bmi_cache::cached_obj_path( - key, cu.packageObjectRel.generic_string()); - if (!cu.providesModule.empty()) { - std::string bmi; - for (char c : cu.providesModule) - bmi.push_back(c == ':' ? '-' : c); - bmi += std::string(bmiT.bmiExt); - cu.cachedBmi = mcpp::bmi_cache::cached_bmi_path(key, bmi); - } - } - mcpp::bmi_cache::touch_accessed(key); - state.usedCacheRecords.insert(key.entryFile().string()); - // By the name the package report gives it: a dependency key - // is not always its qualified name (`greet` for the default - // namespace's `mcpplibs.greet`), and the line then read - // `Compiling` for a package the cache served. - ctx.cachedDeps.push_back({mcpp::build::qualified_package_name(pkgRoot.manifest), depVer, - unitIdx.size()}); - continue; // no populate task; it is already cached - } - // A valid entry that does not hold what we asked for means the - // entry and this build disagree about the layout under one key. - // After #344 that is unreachable; say so out loud if it ever - // happens again, because the alternative presentation is "the - // cache silently never hits", and a cache that lies about its own - // effectiveness went unnoticed for three months once already. - if (!probe.layoutMismatch.empty()) { - mcpp::ui::warning(std::format( - "build cache entry for {}@{} [{}] does not contain the " - "artifacts this build needs ({} of {} missing, e.g. `{}`); " - "treating it as a miss. Run `mcpp cache verify` for details.", - depName, depVer, key.keyHex, - probe.layoutMismatch.size(), - arts.bmiFiles.size() + arts.objFiles.size(), - probe.layoutMismatch.front())); - } - for (auto u : unitIdx) ctx.plan.compileUnits[u].fillsCache = true; - BuildContext::CacheTask task{ std::move(key), std::move(arts) }; - // The tree is read again before its entry is written: a source - // edited while it compiled is not stored under the digest of the - // sources before the edit (mcpp#974). - if (depIdent->sourceKind == "path") { - task.treeRoot = depIdent->sourceRef.empty() ? pkgRoot.root - : std::filesystem::path(depIdent->sourceRef); - task.treeRows = tree_digest_rows(task.treeRoot); - task.treeDigest = tree_content_digest(state, task.treeRoot); - } - ctx.depsToPopulate.push_back(std::move(task)); - } - } - // ────────────────────────────────────────────────────────────────── - return {}; -} - // `split_debuginfo = "unpacked"` (design 2026-10-31 §4.1 (b)): a unit // compiled in this build directory keeps its debug information beside its // object (`-gsplit-dwarf`), and the artifacts carry a skeleton that names it, @@ -2399,12 +1940,9 @@ static void step13_report_packages(PrepareState& state, BuildContext& ctx) { // consumer's key: a package reached through `[feature-deps]` or // another table has no entry in the consumer's `[dependencies]`. const ResolvedRecord* rec = nullptr; - { - auto rn = mcpp::pm::compat::resolve_package_name(m.package.name, m.package.namespace_); - for (auto const& ns : {rn.namespace_, std::string(mcpp::pm::kDefaultNamespace), std::string{}}) - if (auto it = state.resolved.find(ResolvedKey{ns, rn.shortName}); - it != state.resolved.end()) { rec = &it->second; break; } - } + // The record of this slot, whichever resolution holds it (§21 A1). + for (auto const& [k, r] : state.resolved) + if (r.depIndex + 1 == i) { rec = &r; break; } const std::string kind = rec ? rec->source : spec && spec->isPath() ? "path" : spec && spec->isGit() ? "git" : "version"; @@ -2468,60 +2006,4 @@ void focus_on_member(BuildContext& ctx) { ctx.projectRoot = m.root; } -// AN EDITED TREE'S PREVIOUS ENTRY IS REPLACED (mcpp#974), as a `path` host -// tool's is (tool_store::write_entry, mcpp#874). Every edit of a `path` -// dependency is a new key, and none would be collected before `[cache] -// unused_days`: the tree an entry was built from is written beside it, and an -// entry of the same package from the same tree under another digest is removed -// as this one is written. Another checkout of the package is another tree, and -// keeps its entries. -static void replace_tree_siblings(const mcpp::bmi_cache::CacheKey& key, const std::filesystem::path& tree) { - namespace fs = std::filesystem; - const auto plus = key.version.find("+path."); - if (plus == std::string::npos) return; - const auto origin = tree.lexically_normal().generic_u8string(); - const std::string text(origin.begin(), origin.end()); - std::error_code ec; - std::ofstream(key.dir() / "source", std::ios::binary | std::ios::trunc) << text; - const auto versionDir = key.dir().parent_path(); - const auto prefix = std::format("{}@{}", key.packageName, key.version.substr(0, plus + 6)); - auto from_this_tree = [&](const fs::path& entry) { - std::ifstream in(entry / "source", std::ios::binary); - return in && std::string{std::istreambuf_iterator(in), {}} == text; - }; - for (auto const& v : fs::directory_iterator(versionDir.parent_path(), ec)) { - const auto name = v.path().filename().u8string(); - if (v.path() == versionDir || !std::string(name.begin(), name.end()).starts_with(prefix)) continue; - std::error_code kec; - for (auto const& entry : fs::directory_iterator(v.path(), kec)) - if (from_this_tree(entry.path())) fs::remove_all(entry.path(), kec); - fs::remove(v.path(), kec); // only when nothing else is left in it - } -} - -void populate_dependency_cache(BuildContext& ctx) { - if (ctx.cacheMode != CacheMode::Global) ctx.depsToPopulate.clear(); - for (auto& task : ctx.depsToPopulate) { - if (!task.treeRoot.empty()) { - if (mcpp::build::source_tree_digest(task.treeRoot, task.treeRows) != task.treeDigest) { - const auto tree = task.treeRoot.lexically_normal().generic_u8string(); - mcpp::diag::degraded("build/cache", std::format( - "the sources of {} changed while it was built ({}), so its build was not stored in the global cache", - task.key.packageName, std::string(tree.begin(), tree.end())), - "the next build compiles it again, from the sources it then finds"); - continue; - } - } - auto pr = mcpp::bmi_cache::populate_from(task.key, ctx.outputDir, task.artifacts); - if (pr && !task.treeRoot.empty()) replace_tree_siblings(task.key, task.treeRoot); - if (!pr) { - mcpp::ui::warning(std::format( - "bmi cache populate failed for {}@{}: {}", - task.key.packageName, task.key.version, pr.error())); - } - } - // Once: a second drive of the same plan has nothing new to add. - ctx.depsToPopulate.clear(); -} - } // namespace mcpp::build diff --git a/src/build/prepare/platform.cpp b/src/build/prepare/platform.cpp index 2482d3224..b7dedf40d 100644 --- a/src/build/prepare/platform.cpp +++ b/src/build/prepare/platform.cpp @@ -808,6 +808,16 @@ void step4b_seed_platform(PrepareState& state) { namespace { thread_local std::optional g_platformRestart; +thread_local std::optional> g_platformSeriesRestart; + +// A REFUSAL BY THE VERSION GROUP OF A RELEASE A SERIES TOOK IS NOT THE BUILD'S +// LAST WORD (design 2026-11-19 §21 A5): the series' newest release is not the +// newest one this build can use, and `prepare_build` takes the series below +// it. `why` names the request that refused it. +void refuse_series_release(const PrepareState& state, std::string why) { + if (!state.platformSeriesTook.empty()) + g_platformSeriesRestart = std::pair{state.platformSeriesTook, std::move(why)}; +} } // namespace std::optional take_platform_restart() { @@ -816,6 +826,12 @@ std::optional take_platform_restart() { return r; } +std::optional> take_platform_series_restart() { + auto r = std::move(g_platformSeriesRestart); + g_platformSeriesRestart.reset(); + return r; +} + std::expected step4b_apply_platform(PrepareState& state) { auto& P = state.platform; if (P.selection.none) return false; @@ -1019,6 +1035,7 @@ std::expected step4b_apply_platform(PrepareState& state) { item.consumerDepIndex = kMainConsumer; item.buildOnly = true; item.fromBuildTable = true; + item.scope = root_program_scope(state); // The root asks for them as it would for a declared build dependency, // so the tool provisioning pass builds them for this machine. state.m->buildDependencies[key] = item.spec; @@ -1127,6 +1144,8 @@ std::expected step4b_pin_to_platform_group(PrepareState& stat if (auto req = mcpp::version_req::parse_req(asked); req && group && !mcpp::version_req::matches(*req, *group)) { refusal::record(refusal::Code::PlatformVersionGroup); + refuse_series_release(state, std::format("its version group names {} {}, and {} requests '{}'", + shortName, g->second, item.requestedBy, asked)); return std::unexpected(std::format( "{} is requested as '{}' by {}, and the platform {} is a version group\n" " that names {} {}.\n" @@ -1180,12 +1199,14 @@ void step4b_read_platform_group(PrepareState& state, const std::string& resolved mcpp::ui::add_failure_notice("platform-newer", *tip); } -// A SERIES TAKES ITS NEWEST RELEASE THIS ENGINE READS (SPEC-013 §14). A -// platform selected as a range (`^0.24`, the target table's default) skips a -// release whose `[package] mcpp` floor is above this mcpp, so an older engine -// keeps building with the newest platform it can read instead of being -// refused by a release made for a newer one. A pin (`=`) and a lock anchor are -// taken as they are. +// A SERIES TAKES ITS NEWEST RELEASE THIS ENGINE READS AND THIS BUILD CAN USE +// (SPEC-013 §14). A platform selected as a range (`^0.24`) skips a release +// whose `[package] mcpp` floor is above this mcpp, so an older engine keeps +// building with the newest platform it can read instead of being refused by a +// release made for a newer one; and a release whose version group a request +// of the build does not admit, which an earlier pass took and was refused for +// (design 2026-11-19 §21 A5). A pin (`=`) and a lock anchor are taken as they +// are. std::expected step4b_platform_within_engine(PrepareState& state) { if (state.platform.selection.none || state.worklist.empty()) return {}; auto& item = state.worklist.front(); @@ -1198,20 +1219,40 @@ std::expected step4b_platform_within_engine(PrepareState& sta auto cfg = state.get_cfg(true); if (!cfg) return std::unexpected(cfg.error()); std::string constraint = spec.version; + auto const& skipped = state.overrides.platform_series_skipped; + for (auto const& [version, why] : skipped) constraint += ", <" + version; + // Said once: each earlier pass said the releases it skipped before. + if (!skipped.empty()) + mcpp::ui::info("Platform", std::format("{} v{} is not usable by this build: {}", + spec.shortName, skipped.back().first, skipped.back().second)); for (int attempt = 0; attempt < 16; ++attempt) { auto found = mcpp::pm::resolve_semver(spec.namespace_, spec.shortName, constraint, state.index_route(*cfg), *state.targetPlatform); - if (!found) return {}; // the walk reports it, with the request that made it + if (!found) { + if (skipped.empty()) return {}; // the walk reports it, with the request that made it + refusal::record(refusal::Code::PlatformVersionGroup); + std::string each; + for (auto const& [version, why] : skipped) each += std::format("\n v{}: {}", version, why); + return std::unexpected(std::format( + "no release of the platform {} {} can be used by this build:{}\n" + " Each release's version group names a package version a request of the\n" + " build does not admit. Admit the group's version where it is requested,\n" + " or select a release outside the series ([target.] platform =\n" + " \"{}@\").", + spec.shortName, spec.version, each, spec.shortName)); + } auto loaded = state.loadVersionDep(item.name, spec.namespace_, spec.shortName, *found); if (!loaded) return {}; auto const& floor = loaded->second.package.mcppFloor; const auto need = mcpp::xpkg_version::parse(floor); if (floor.empty() || !need || mcpp::xpkg_version::compare(*have, *need) >= 0) { - if (attempt > 0) { - mcpp::ui::info("Platform", std::format("{} {} → v{} (the newest this mcpp reads)", - spec.shortName, spec.version, *found)); + if (attempt > 0 || !skipped.empty()) { + mcpp::ui::info("Platform", std::format("{} {} → v{} (the newest {})", + spec.shortName, spec.version, *found, + skipped.empty() ? "this mcpp reads" : "this build can use")); spec.version = "=" + *found; } + state.platformSeriesTook = *found; return {}; } mcpp::ui::info("Platform", std::format("{} v{} needs mcpp >= {}; this is mcpp {}", @@ -1245,6 +1286,9 @@ std::expected step4b_check_platform_versions(PrepareState& st if (auto r = state.resolved.find({n.namespace_, n.shortName}); r != state.resolved.end()) by = r->second.requestedBy; refusal::record(refusal::Code::PlatformVersionGroup); + refuse_series_release(state, std::format("its version group names {} {}, and {} {} is in this build (requested by {})", + n.shortName, it->second, n.shortName, pm.package.version, + by.empty() ? std::string("the graph") : by)); return std::unexpected(std::format( "{} {} is in this build, and the platform {} is a version group that names {} {}.\n" " requested by: {}\n" diff --git a/src/build/prepare/records.cpp b/src/build/prepare/records.cpp index 735d2b234..8278ce77e 100644 --- a/src/build/prepare/records.cpp +++ b/src/build/prepare/records.cpp @@ -129,6 +129,128 @@ static std::optional target_drift(const PrepareState& state, c.why.empty() ? std::string{} : "; " + c.why); } +// The name of a package as the lock and the platform's reach compare it. +static std::string qualified_of(const mcpp::manifest::Manifest& m) { + auto n = mcpp::pm::compat::resolve_package_name(m.package.name, m.package.namespace_); + return mcpp::pm::compat::qualified_name( + n.namespace_.empty() ? std::string(mcpp::pm::kDefaultNamespace) : n.namespace_, n.shortName); +} + +// WHAT THE PROJECT'S OWN GRAPH REACHES (SPEC-013 §15.1): from the root and the +// selected members along the edges, never entering a package the platform +// added. A package the platform added, and one only those reach (its layers' +// implementations, its tools' and its plugin's libraries), is the platform's. +static std::set project_reaches(const PrepareState& state) { + std::set projectReaches; + const auto n = state.packages.size(); + std::vector own(n, false); + std::vector stack; + for (std::size_t i = 0; i < n; ++i) + if (i == 0 || state.packages[i].selectedMember) { own[i] = true; stack.push_back(i); } + while (!stack.empty()) { + const auto p = stack.back(); + stack.pop_back(); + for (auto const& e : state.dependencyEdges) { + const auto d = e.dependencyPackageIndex; + if (e.consumerPackageIndex != p || d >= n || own[d]) continue; + if (state.platformInjected.contains(qualified_of(state.packages[d].manifest))) continue; + own[d] = true; + stack.push_back(d); + } + } + for (std::size_t i = 0; i < n; ++i) + if (own[i]) projectReaches.insert(qualified_of(state.packages[i].manifest)); + return projectReaches; +} + +// The lock's version entries: the whole resolved graph, one entry per package +// and version. +static void lock_version_entries(PrepareState& state, mcpp::lockfile::Lockfile& lock, + const std::set& projectReaches, + const std::function& lock_name_for) { + // Version deps: the whole resolved graph, at the versions actually + // chosen. `resolved` is an ordered map, so the file is deterministic. + // + // ONE ENTRY PER PACKAGE AND VERSION (design 2026-11-19 §21 A3). The + // target's records sort before the build programs' (an empty scope + // first), so a package's first version is the target's, or the first + // program's; a version only other programs hold is an entry of its + // own, `+b`, listing them. + struct Held { std::string first; std::map> programs; }; + std::map, Held> heldBy; + for (auto const& [key, rec] : state.resolved) { + if (rec.source != "version" || rec.version.empty() || rec.devOnly) continue; + auto& h = heldBy[{key.ns, key.shortName, key.variant}]; + if (h.first.empty()) h.first = rec.version; + if (rec.version != h.first) h.programs[rec.version].push_back(key.scope); + } + std::set versionLocked; + for (auto const& [key, rec] : state.resolved) { + if (rec.source != "version") continue; // path / git handled elsewhere + if (rec.version.empty()) continue; + // See ResolvedRecord::devOnly: `mcpp test` resolves dev-deps and + // `mcpp build` does not, so writing them would make the file depend + // on which command ran last. + if (rec.devOnly) continue; + mcpp::lockfile::LockedPackage lp; + lp.name = lock_name_for(key); + lp.namespace_ = key.ns; + lp.version = rec.version; + lp.overridden = rec.overridden; + // A variant is its own entry, and says which patches made it + // (design 2026-11-02 §14.1): each by its path, relative to the + // project where it is inside it, and its SHA-256. + if (!key.variant.empty()) { + lp.name += "+p" + key.variant.substr(0, 12); + lp.variant = key.variant; + for (auto const& p : rec.patches) { + std::filesystem::path path(p); + auto rel = path.lexically_relative(*state.root); + const auto shown = (!rel.empty() && !rel.generic_string().starts_with("..")) + ? rel.generic_string() : path.generic_string(); + lp.patches.push_back(std::format("{}#sha256:{}", shown, + mcpp::pm::patch::file_sha256(path).value_or("unreadable"))); + } + } + // Use the namespace and resolved version as the source identifier. + // For custom indices, include the index name for traceability. + auto sourceIndex = lp.namespace_.empty() + ? std::string(mcpp::pm::kDefaultNamespace) + : lp.namespace_; + lp.source = std::format("index+{}@{}", sourceIndex, lp.version); + // Use a deterministic hash based on namespace + name + version. + // A future PR can replace this with a real content hash from the + // xpkg.lua's declared sha256 or from the install plan. + // + // NOT `std::hash`: its output is implementation-defined + // (MSVC FNV-1a, libstdc++/libc++ MurmurHash), so the same dependency + // used to hash differently on Windows and Linux while the `fnv1a:` + // prefix claimed otherwise. `index_package_digest` is FNV-1a on + // every host. + lp.hash = mcpp::pm::index_package_digest(sourceIndex, lp.name, lp.version); + // After the digest, which names the package's bytes and not the + // programs that hold them. + if (auto const& h = heldBy[{key.ns, key.shortName, key.variant}]; rec.version != h.first) { + auto programs = h.programs.at(rec.version); + std::ranges::sort(programs); + std::string joined; + for (auto const& p : programs) joined += p + "\n"; + lp.name += "+b" + mcpp::toolchain::hash_string(joined).substr(0, 12); + lp.programs = std::move(programs); + } + // The platform's packages say so (SPEC-013 §15.1). + const auto qualified = mcpp::pm::compat::qualified_name( + key.ns.empty() ? std::string(mcpp::pm::kDefaultNamespace) : key.ns, key.shortName); + if (state.platform.active + && (state.platformInjected.contains(qualified) || !projectReaches.contains(qualified))) + lp.origin = "platform"; + // One entry per identity: a dependency declared by the root and by + // a member under different spellings is one package. + if (!versionLocked.insert(lock_identity(lp)).second) continue; + lock.packages.push_back(std::move(lp)); + } +} + std::expected step13_lockfile(PrepareState& state, BuildContext& ctx) { // Write/update mcpp.lock for any version-based deps that succeeded. // Path deps are intentionally NOT locked — their source is local filesystem. @@ -259,95 +381,7 @@ std::expected step13_lockfile(PrepareState& state, BuildConte lock.packages.push_back(std::move(lp)); } - // WHAT THE PROJECT'S OWN GRAPH REACHES (SPEC-013 §15.1): from the - // root and the selected members along the edges, never entering a - // package the platform added. A package the platform added, and one - // only those reach (its layers' implementations, its tools' and its - // plugin's libraries), is the platform's. - auto qualified_of = [](const mcpp::manifest::Manifest& m) { - auto n = mcpp::pm::compat::resolve_package_name(m.package.name, m.package.namespace_); - return mcpp::pm::compat::qualified_name( - n.namespace_.empty() ? std::string(mcpp::pm::kDefaultNamespace) : n.namespace_, n.shortName); - }; - std::set projectReaches; - { - const auto n = state.packages.size(); - std::vector own(n, false); - std::vector stack; - for (std::size_t i = 0; i < n; ++i) - if (i == 0 || state.packages[i].selectedMember) { own[i] = true; stack.push_back(i); } - while (!stack.empty()) { - const auto p = stack.back(); - stack.pop_back(); - for (auto const& e : state.dependencyEdges) { - const auto d = e.dependencyPackageIndex; - if (e.consumerPackageIndex != p || d >= n || own[d]) continue; - if (state.platformInjected.contains(qualified_of(state.packages[d].manifest))) continue; - own[d] = true; - stack.push_back(d); - } - } - for (std::size_t i = 0; i < n; ++i) - if (own[i]) projectReaches.insert(qualified_of(state.packages[i].manifest)); - } - - // Version deps: the whole resolved graph, at the versions actually - // chosen. `resolved` is an ordered map, so the file is deterministic. - std::set versionLocked; - for (auto const& [key, rec] : state.resolved) { - if (rec.source != "version") continue; // path / git handled elsewhere - if (rec.version.empty()) continue; - // See ResolvedRecord::devOnly: `mcpp test` resolves dev-deps and - // `mcpp build` does not, so writing them would make the file depend - // on which command ran last. - if (rec.devOnly) continue; - mcpp::lockfile::LockedPackage lp; - lp.name = lock_name_for(key); - lp.namespace_ = key.ns; - lp.version = rec.version; - lp.overridden = rec.overridden; - // A variant is its own entry, and says which patches made it - // (design 2026-11-02 §14.1): each by its path, relative to the - // project where it is inside it, and its SHA-256. - if (!key.variant.empty()) { - lp.name += "+p" + key.variant.substr(0, 12); - lp.variant = key.variant; - for (auto const& p : rec.patches) { - std::filesystem::path path(p); - auto rel = path.lexically_relative(*state.root); - const auto shown = (!rel.empty() && !rel.generic_string().starts_with("..")) - ? rel.generic_string() : path.generic_string(); - lp.patches.push_back(std::format("{}#sha256:{}", shown, - mcpp::pm::patch::file_sha256(path).value_or("unreadable"))); - } - } - // Use the namespace and resolved version as the source identifier. - // For custom indices, include the index name for traceability. - auto sourceIndex = lp.namespace_.empty() - ? std::string(mcpp::pm::kDefaultNamespace) - : lp.namespace_; - lp.source = std::format("index+{}@{}", sourceIndex, lp.version); - // Use a deterministic hash based on namespace + name + version. - // A future PR can replace this with a real content hash from the - // xpkg.lua's declared sha256 or from the install plan. - // - // NOT `std::hash`: its output is implementation-defined - // (MSVC FNV-1a, libstdc++/libc++ MurmurHash), so the same dependency - // used to hash differently on Windows and Linux while the `fnv1a:` - // prefix claimed otherwise. `index_package_digest` is FNV-1a on - // every host. - lp.hash = mcpp::pm::index_package_digest(sourceIndex, lp.name, lp.version); - // The platform's packages say so (SPEC-013 §15.1). - const auto qualified = mcpp::pm::compat::qualified_name( - key.ns.empty() ? std::string(mcpp::pm::kDefaultNamespace) : key.ns, key.shortName); - if (state.platform.active - && (state.platformInjected.contains(qualified) || !projectReaches.contains(qualified))) - lp.origin = "platform"; - // One entry per identity: a dependency declared by the root and by - // a member under different spellings is one package. - if (!versionLocked.insert(lock_identity(lp)).second) continue; - lock.packages.push_back(std::move(lp)); - } + lock_version_entries(state, lock, project_reaches(state), lock_name_for); // A workspace's lock is at its root and records every member's // resolution (§15). A plan of some of its members updates the entries // it resolved and keeps the others; a plan of all of them writes the @@ -522,9 +556,11 @@ std::expected step13_lockfile(PrepareState& state, BuildConte // Same data, second consumer: the "Compiling v" banner. // It reads this rather than re-deriving from the manifest, so the banner // and the lock cannot disagree about what was built. + // The target's version where a build program holds another (§21 A1): + // the target's records come first, and a later one does not replace it. for (auto const& [key, rec] : state.resolved) { if (rec.source != "version" || rec.version.empty()) continue; - ctx.resolvedVersions[lock_name_for(key)] = rec.version; + ctx.resolvedVersions.try_emplace(lock_name_for(key), rec.version); } } return {}; diff --git a/src/build/prepare/state.cppm b/src/build/prepare/state.cppm index c49b67e68..93cc05ad3 100644 --- a/src/build/prepare/state.cppm +++ b/src/build/prepare/state.cppm @@ -168,6 +168,10 @@ constexpr std::size_t kMainConsumer = static_cast(-1); // dependency's sources are compiled into the consumer. bool embed = false; std::vector exclude; + // The build program whose resolution first made the edge (§21 A1); + // empty once the target's resolution makes it too. A program whose + // record moves to another slot moves the edges it alone made. + std::string scope; }; struct GraphRequest { @@ -192,6 +196,14 @@ constexpr std::size_t kMainConsumer = static_cast(-1); // a patched library is another library, so two patch sets of one // package are two identities, resolved, built and cached apart. std::string variant; + // THE PROGRAM WHOSE RESOLUTION THIS IS (design 2026-11-19 §21 A1, + // mcpp#972): empty for the target's, `build:@` for a + // build program's. A C++ program loads one version of a package; two + // programs do not constrain each other, so each build program's + // `[build-dependencies]` closure is resolved apart from the target's + // and from every other program's. Records of one package in several + // scopes share its slot while they share its version. + std::string scope; auto operator<=>(const ResolvedKey&) const = default; }; @@ -277,6 +289,10 @@ constexpr std::size_t kMainConsumer = static_cast(-1); // machine does not match (SPEC-004 §4.8). Its edge does not make the // dependency a module of a build program. bool targetRowOnly = false; + // The program whose resolution the request belongs to (ResolvedKey:: + // scope): set where a package's `[build-dependencies]` are seeded, and + // inherited by every request below them. + std::string scope; }; struct DeclaringManifest { @@ -610,6 +626,10 @@ struct PrepareState { std::map platformGroup; std::string platformGroupFrom; // `openkal-platform 0.24.1`, for a refusal bool platformGroupRead = false; + // The release a platform selected as a series took in this pass, when + // neither an exact version nor mcpp.lock chose it: a refusal by its version + // group moves the series below it (design 2026-11-19 §21 A5). + std::string platformSeriesTook; std::map packageIdentityLockAnchors; // mcpp.lock as this run read it (design 2026-10-26 §4.1): the write at the // end compares, so a lock another run wrote meanwhile is not overwritten. @@ -860,6 +880,21 @@ struct PrepareState { // built once for each side that asks for it, and each side reads the table // by its own row and makes its own requests. std::vector depSidesWalked; + // The build programs a slot's table has been walked for (§21 A1): each + // program resolves the package's requests in its own scope. + std::set> scopesWalked; + // A program's requests wait until the target's are resolved, so that the + // target's choices are made by the target's requests alone and a program + // takes them where its own requests allow (§21 A1). + std::deque scopedWorklist; + // A program's record that a merge moved off a slot another program or the + // target uses: the slot its edges are moved from, and the requests the + // record held (§21 A1). + struct ScopeMove { + std::size_t oldPackageIndex = 0; + std::vector> constraintsBy; + }; + std::map scopeMoves; // THE ROOT'S OVERRIDES (design 2026-11-10 supplement, T2): each // `override = true` declaration of the root's tables and of // `[workspace.dependencies]`, with the identity it answers for and the @@ -1078,7 +1113,8 @@ std::size_t packageIndexForConsumer(std::size_t consumerDepIndex); void recordDependencyEdge(PrepareState& state, std::size_t consumerDepIndex, std::size_t dependencyPackageIndex, const mcpp::manifest::DependencySpec& spec, bool buildTable, - const std::string& writtenKey, bool targetRowOnly = false); + const std::string& writtenKey, bool targetRowOnly = false, + const std::string& scope = {}); // graph_edges.cpp: the package being built stands for itself in its tests' // graph -- an edge reached through its `[dev-dependencies]` naming it is its // own, recorded here; true when the request is so answered. @@ -1141,6 +1177,11 @@ std::expected phase13_finish(PrepareState& state); // P13's records half (records.cpp), called by phase13_finish. std::expected step13_lockfile(PrepareState& state, BuildContext& ctx); void step13_resolution_json(PrepareState& state, BuildContext& ctx); +// P13's global dependency cache (dependency_cache.cpp), called by phase13_finish: +// whether package `i`'s sources are named (an index version, a git commit, a +// path tree's content digest), and each package's key, hit or fill task. +bool sources_named(const PrepareState& state, std::size_t i); +std::expected step13_dependency_cache(PrepareState& state, BuildContext& ctx); // ── Helpers the phases share, defined in the files named below ───────────── @@ -1421,6 +1462,37 @@ std::expected side_requests(PrepareState& state, WorkItem side_item(const SideRequests& side, const std::string& childName, mcpp::manifest::DependencySpec childReq, const std::string& label, std::size_t consumer, const std::filesystem::path& root, const WorkItem& item); +// The scope of a package's build program (ResolvedKey::scope, design 2026-11-19 +// §21 A1): `build: