From 15d9b7351055dfc7dae1f3cf94f4cf1eb7c411fc Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Thu, 19 Nov 2026 20:06:23 +0900 Subject: [PATCH] 0.4.6: the text another program writes, as UTF-8 mcpplibs.os.text decodes a program's text by the encoding it writes in: UTF-8, or the convention of Windows console programs, which write to a pipe or a file in the output code page of the console they are attached to and in the system's ANSI code page when attached to none, as measured with cl.exe and link.exe (mcpp-community/mcpp#970). The page is read where the program is started; a sequence that is no character of it becomes a replacement character, and valid_utf8 replaces each sequence that is not UTF-8 with U+FFFD, a maximal subpart at a time. windows::console_text_code_page and windows::utf8_from_code_page are the facts beneath it. --- .github/workflows/ci.yml | 8 +-- CHANGELOG.md | 17 ++++++ README.md | 18 ++++++ checks/emscripten/tests/text_test.cpp | 6 ++ checks/hosted/tests/text_test.cpp | 6 ++ mcpp.toml | 2 +- src/os.cppm | 1 + src/text.cpp | 81 +++++++++++++++++++++++++++ src/text.cppm | 55 ++++++++++++++++++ src/windows.cppm | 10 ++++ src/windows/windows.cpp | 30 ++++++++++ tests/text_test.cpp | 81 +++++++++++++++++++++++++++ 12 files changed, 310 insertions(+), 5 deletions(-) create mode 100644 checks/emscripten/tests/text_test.cpp create mode 100644 checks/hosted/tests/text_test.cpp create mode 100644 src/text.cpp create mode 100644 src/text.cppm create mode 100644 tests/text_test.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d7058c..b0b9597 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: run: | profile='${{ matrix.profile }}' mcpp test --profile "$profile" 2>&1 | tee tests.log - for name in platform_test fs_test process_test ipc_test unit_test dirs_terminal_test linux_test watch_test windows_test macos_test; do + for name in platform_test fs_test process_test ipc_test unit_test dirs_terminal_test linux_test watch_test windows_test macos_test text_test; do grep -q "^$name \.\.\. ok" tests.log || { echo "::error::$name did not run or did not pass"; exit 1; } done @@ -219,7 +219,7 @@ jobs: run: | profile='${{ matrix.profile }}' mcpp test --profile "$profile" ${{ matrix.target }} 2>&1 | tee tests.log - for name in platform_test fs_test process_test ipc_test unit_test dirs_terminal_test linux_test library_test watch_test windows_test macos_test; do + for name in platform_test fs_test process_test ipc_test unit_test dirs_terminal_test linux_test library_test watch_test windows_test macos_test text_test; do grep -q "^$name \.\.\. ok" tests.log || { echo "::error::$name did not run or did not pass"; exit 1; } done @@ -271,7 +271,7 @@ jobs: run: | set -euo pipefail mcpp test --target x86_64-openkal 2>&1 | tee tests.log - for name in platform_test fs_test process_test ipc_test unit_test dirs_terminal_test linux_test watch_test windows_test macos_test; do + for name in platform_test fs_test process_test ipc_test unit_test dirs_terminal_test linux_test watch_test windows_test macos_test text_test; do grep -q "^$name \.\.\. ok" tests.log || { echo "::error::$name did not pass as an image"; exit 1; } done mcpp test --target aarch64-openkal --no-run @@ -424,7 +424,7 @@ jobs: run: | set -euo pipefail mcpp test --target wasm32-emscripten --profile '${{ matrix.profile }}' 2>&1 | tee tests.log - for name in platform_test fs_test dirs_terminal_test linux_test library_test watch_test windows_test macos_test; do + for name in platform_test fs_test dirs_terminal_test linux_test library_test watch_test windows_test macos_test text_test; do grep -q "^$name \.\.\. ok" tests.log || { echo "::error::$name did not run or did not pass"; exit 1; } done - name: The compile-time constants of Emscripten (R3) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2576e8..f322b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 0.4.6 + +- **The text another program writes, as UTF-8 (`mcpplibs.os.text`).** A + `decoder` for an `encoding`: `utf8`, or `console`, in which a Windows console + program writes its text to a pipe or a file --- the output code page of the + console it is attached to, else the system's ANSI code page, as measured with + cl.exe and link.exe (mcpp-community/mcpp#970). The page is read when the + decoder is made, in the program that starts the other; a sequence that is no + character of the page becomes a replacement character. Elsewhere `console` + is UTF-8, and `valid_utf8` replaces each sequence that is not UTF-8 with + U+FFFD, a maximal subpart at a time. +- **`windows::console_text_code_page` and `windows::utf8_from_code_page`.** The + page of the console this program is attached to, whatever its own streams + are, or the system's ANSI code page from the registry, which `GetACP` answers + only to a program that declares no code page of its own; and text of a code + page as UTF-8. + ## 0.4.5 - **A start refused for its size says so.** Where the system refuses a start diff --git a/README.md b/README.md index 92d5ca5..a608b69 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,23 @@ states what it spends resources for (balanced, performance, economy). Each is refused with `errc::not_supported` where the system cannot give it, never dropped (0.2.8). +## The text another program writes (0.4.6) + +This package's text is UTF-8, and so is a program's on every system but one +convention: a Windows console program writes its text to a pipe or a file in +the output code page of the console it is attached to, and in the system's ANSI +code page when it is attached to none. Measured with cl.exe and link.exe +(mcpp-community/mcpp#970, windows-2022): under consoles in code pages 437, 1252, +936 and 65001 each wrote in the console's page, and with no console in the +system's. `text::decoder(text::encoding::console)` reads that page when it is +made, in the program that starts the other --- which inherits its console or +the absence of one --- and converts what it reads to UTF-8, a line at a time; +a sequence that is no character of the page becomes a replacement character. +On every other system `console` is UTF-8, and the decoder replaces each +sequence that is not UTF-8 with U+FFFD (`text::valid_utf8`, a maximal subpart +at a time). `windows::console_text_code_page()` and +`windows::utf8_from_code_page()` are the two facts beneath it. + ## Modules | Module | Contents | @@ -119,6 +136,7 @@ dropped (0.2.8). | `mcpplibs.os.process` | `spawn`, `run` (and with bounds: a hard bound, a grace, a drain, output limits), `exec`, a command line joined as it is for the command interpreter (`one_string`), pipes, both outputs on one pipe, output onto this program's error output, units and `end_unit`, declared units, observation and retirement, bound lifetime, bounded waits and reads; `self`, `describe`, `alive`, `threads` | | `mcpplibs.os.interrupt` | `requested`, `wait`, `end_as_requested`, `end_as_output_closed` | | `mcpplibs.os.stdio` | the three streams as bytes; a bounded read of input; the output sent to the error stream for a while (`output_to_error`) | +| `mcpplibs.os.text` | the text another program writes, as UTF-8: `decoder` for an `encoding` (`utf8`, or `console`, the convention of Windows console programs), `valid_utf8` | | `mcpplibs.os.terminal` | `interactive`, `size`, `sequences`, `color`, `raw_mode` (with or without the interrupting keystroke), `foreground` | | `mcpplibs.os.dirs` | `home`, `config`, `data`, `state`, `cache`, `runtime`, `temp` | | `mcpplibs.os.ipc` | a listener and a connection at a name in a directory; `single_instance` | diff --git a/checks/emscripten/tests/text_test.cpp b/checks/emscripten/tests/text_test.cpp new file mode 100644 index 0000000..935be79 --- /dev/null +++ b/checks/emscripten/tests/text_test.cpp @@ -0,0 +1,6 @@ +// text_test.cpp of mcpplibs.os, compiled for Emscripten. The imports are restated +// because the dependency scan reads this file and not the one it includes. +import std; +import mcpplibs.os; +import mcpplibs.mtest; +#include "../../../tests/text_test.cpp" diff --git a/checks/hosted/tests/text_test.cpp b/checks/hosted/tests/text_test.cpp new file mode 100644 index 0000000..f7c4c87 --- /dev/null +++ b/checks/hosted/tests/text_test.cpp @@ -0,0 +1,6 @@ +// text_test.cpp of mcpplibs.os, compiled in this arrangement. The imports are restated +// because the dependency scan reads this file and not the one it includes. +import std; +import mcpplibs.os; +import mcpplibs.mtest; +#include "../../../tests/text_test.cpp" diff --git a/mcpp.toml b/mcpp.toml index a31b563..21004b6 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "os" -version = "0.4.5" +version = "0.4.6" description = "mcpplibs.os: a C++ operating-system library over openkal --- processes, files, environment, terminals, directories and local IPC, with compile-time platform constants and a few platform modules." license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/src/os.cppm b/src/os.cppm index d9f7fcd..8d4829e 100644 --- a/src/os.cppm +++ b/src/os.cppm @@ -13,6 +13,7 @@ export import mcpplibs.os.fs; export import mcpplibs.os.process; export import mcpplibs.os.interrupt; export import mcpplibs.os.stdio; +export import mcpplibs.os.text; export import mcpplibs.os.terminal; export import mcpplibs.os.dirs; export import mcpplibs.os.ipc; diff --git a/src/text.cpp b/src/text.cpp new file mode 100644 index 0000000..ac9b6fe --- /dev/null +++ b/src/text.cpp @@ -0,0 +1,81 @@ +module mcpplibs.os.text; + +import std; +import mcpplibs.os.platform; +import mcpplibs.os.windows; + +namespace mcpplibs::os::text { + +namespace { + +// The first byte's sequence: its length and the range of its second byte, by +// Table 3-7 of The Unicode Standard; a length of 0 for a byte no well-formed +// sequence begins with. +struct lead { + std::size_t length; + unsigned char low, high; +}; + +lead lead_of(unsigned char b) { + if (b >= 0xC2 && b <= 0xDF) return {2, 0x80, 0xBF}; + if (b == 0xE0) return {3, 0xA0, 0xBF}; + if (b == 0xED) return {3, 0x80, 0x9F}; + if (b >= 0xE1 && b <= 0xEF) return {3, 0x80, 0xBF}; + if (b == 0xF0) return {4, 0x90, 0xBF}; + if (b >= 0xF1 && b <= 0xF3) return {4, 0x80, 0xBF}; + if (b == 0xF4) return {4, 0x80, 0x8F}; + return {0, 0, 0}; +} + +constexpr std::string_view replacement = "\xEF\xBF\xBD"; + +} // namespace + +std::string valid_utf8(std::string_view bytes) { + std::string out; + out.reserve(bytes.size()); + std::size_t i = 0; + while (i < bytes.size()) { + const auto b = static_cast(bytes[i]); + if (b < 0x80) { + out.push_back(static_cast(b)); + ++i; + continue; + } + const lead l = lead_of(b); + // How many bytes from `i' belong to the sequence: the lead, a second + // byte in its own range, then continuation bytes. + std::size_t taken = 1; + if (l.length != 0 && i + 1 < bytes.size()) { + const auto second = static_cast(bytes[i + 1]); + if (second >= l.low && second <= l.high) taken = 2; + } + while (taken >= 2 && taken < l.length && i + taken < bytes.size() + && (static_cast(bytes[i + taken]) & 0xC0) == 0x80) + ++taken; + if (l.length != 0 && taken == l.length) out.append(bytes.substr(i, taken)); + else out.append(replacement); + i += taken; + } + return out; +} + +decoder::decoder(encoding from) { + if constexpr (is_windows) { + if (from == encoding::console) page_ = windows::console_text_code_page(); + } +} + +std::string decoder::operator()(std::string_view bytes) const { + if constexpr (is_windows) { + if (page_ != 65001 && !bytes.empty()) { + // A page the system does not know converts nothing: the bytes are + // then read as UTF-8, the encoding of everything else here. + if (std::string converted = windows::utf8_from_code_page(bytes, page_); !converted.empty()) + return converted; + } + } + return valid_utf8(bytes); +} + +} // namespace mcpplibs::os::text diff --git a/src/text.cppm b/src/text.cppm new file mode 100644 index 0000000..10193ad --- /dev/null +++ b/src/text.cppm @@ -0,0 +1,55 @@ +// mcpplibs.os.text --- the text another program writes, as UTF-8 (0.4.6). +// +// THIS PACKAGE'S TEXT IS UTF-8 (R7), AND SO IS A PROGRAM'S ON EVERY SYSTEM BUT +// ONE CONVENTION. A Windows console program writes its text to a pipe or a +// file in the output code page of the console it is attached to, and in the +// system's ANSI code page when it is attached to none. Measured with cl.exe and +// link.exe (mcpp-community/mcpp#970, windows-2022): under consoles in code pages +// 437, 1252, 936 and 65001 each wrote in the console's page, with no console in +// the system's (1252), and a character the page cannot represent arrived as +// `?'. A program that starts such a program and reads what it writes decodes +// the text here. +// +// THE PAGE IS DECIDED WHERE THE PROGRAM IS STARTED. A program inherits the +// console of the program that starts it, or the absence of one, so the page a +// decoder reads when it is made is the page the program it reads writes in. +// +// const text::decoder decode(text::encoding::console); +// ... started with its two streams in one pipe ... +// for each line read: std::string utf8 = decode(line); +// +// A line is a whole unit: no code page of the system, and not UTF-8, puts the +// byte of a line end inside a character, so text decoded a line at a time is +// decoded correctly. +export module mcpplibs.os.text; + +import std; + +export namespace mcpplibs::os::text { + +// How a program writes its text. +enum class encoding : std::uint8_t { + utf8, // UTF-8 + console, // the convention above; UTF-8 on every system but Windows +}; + +// `bytes' with each sequence that is not UTF-8 replaced by U+FFFD, a maximal +// subpart at a time (The Unicode Standard, section 3.9): the result is UTF-8. +std::string valid_utf8(std::string_view bytes); + +// The text of one program, as UTF-8. +class decoder { +public: + explicit decoder(encoding from); + // `bytes' as UTF-8: a sequence that is no character of the encoding + // becomes a replacement character. + [[nodiscard]] std::string operator()(std::string_view bytes) const; + // The Windows code page the text is read in: 65001 for UTF-8, and on every + // other system. + [[nodiscard]] unsigned code_page() const noexcept { return page_; } + +private: + unsigned page_ = 65001; +}; + +} // namespace mcpplibs::os::text diff --git a/src/windows.cppm b/src/windows.cppm index 01793aa..5a86344 100644 --- a/src/windows.cppm +++ b/src/windows.cppm @@ -20,6 +20,16 @@ unsigned code_page(); // The output code page of the console the stream writes to; 0 when the stream // is not a console. unsigned console_output_code_page(bool error_stream = false); +// The code page in which a console program this program starts writes its text +// to a pipe or a file (0.4.6, mcpplibs.os.text): the output code page of the +// console this program is attached to, whatever its own streams are, and the +// system's ANSI code page when it is attached to none --- the page a program +// that declares none of its own uses, whatever this program declares. +unsigned console_text_code_page(); +// `bytes' of code page `page' as UTF-8 (0.4.6): a sequence that is no character +// of the page becomes the page's replacement; empty for a page the system does +// not know. +std::string utf8_from_code_page(std::string_view bytes, unsigned page); // Whether the system lets a program that declares itself long-path aware use // names past MAX_PATH (LongPathsEnabled). bool long_paths_enabled(); diff --git a/src/windows/windows.cpp b/src/windows/windows.cpp index 4d76dca..fc60e3a 100644 --- a/src/windows/windows.cpp +++ b/src/windows/windows.cpp @@ -69,6 +69,36 @@ reg_get_value registry_reader() { void* const kLocalMachine = reinterpret_cast(static_cast(static_cast(0x80000002))); } // namespace +unsigned console_text_code_page() { + // Attached or not, whatever this program's own streams are: a program it + // starts inherits the console, not the streams. + if (const unsigned page = late::GetConsoleOutputCP(); page != 0) return page; + // The system's ANSI code page is the registry's value: `GetACP' answers it + // only to a program that declares no code page of its own, and this one may + // declare UTF-8. + if (const auto get = registry_reader()) { + okos_wchar value[16] = {}; + okos_dword bytes = sizeof value; + if (get(kLocalMachine, u"SYSTEM\\CurrentControlSet\\Control\\Nls\\CodePage", u"ACP", + 0x2 /* RRF_RT_REG_SZ */, nullptr, value, &bytes) == 0) { + unsigned page = 0; + for (std::size_t i = 0; i < 16 && value[i] >= u'0' && value[i] <= u'9'; ++i) + page = page * 10 + static_cast(value[i] - u'0'); + if (page != 0) return page; + } + } + return late::GetACP(); +} + +std::string utf8_from_code_page(std::string_view bytes, unsigned page) { + if (bytes.empty()) return {}; + const int n = MultiByteToWideChar(page, 0, bytes.data(), static_cast(bytes.size()), nullptr, 0); + if (n <= 0) return {}; + std::u16string wide(static_cast(n), u'\0'); + MultiByteToWideChar(page, 0, bytes.data(), static_cast(bytes.size()), wide.data(), n); + return utf8_from_utf16(wide); +} + bool long_paths_enabled() { const auto get = registry_reader(); if (!get) return false; diff --git a/tests/text_test.cpp b/tests/text_test.cpp new file mode 100644 index 0000000..598d9ec --- /dev/null +++ b/tests/text_test.cpp @@ -0,0 +1,81 @@ +// mcpplibs.os.text (0.4.6) --- the text another program writes, as UTF-8. +// Everywhere: UTF-8 passes unchanged and each sequence that is not UTF-8 +// becomes one U+FFFD per maximal subpart. On Windows: the bytes cl.exe and +// link.exe wrote under consoles in code pages 936, 1252 and 437 +// (mcpp-community/mcpp#970, probe run 44508574496) read as the text they are, +// and `console' is the page of the console this program is attached to. +import std; +import mcpplibs.os; +import mcpplibs.mtest; + +namespace os = mcpplibs::os; +namespace text = mcpplibs::os::text; +using namespace mcpplibs::mtest; + +namespace { +constexpr std::string_view fffd = "\xEF\xBF\xBD"; +std::string repeat(std::string_view s, int n) { + std::string out; + for (int i = 0; i < n; ++i) out += s; + return out; +} +} // namespace + +UTest text_tests{"text", [] { + "UTF-8 passes unchanged, in four scripts and beyond the basic plane"_test = [] { + for (const std::string_view s : {"plain", "na\xC3\xAFve", "\xE6\x9C\xAA\xE5\xAE\x9A\xE4\xB9\x89_\xC3\xA9", + "\xCE\x95\xCE\xBB", "\xF0\x9F\x98\x80"}) + expect(text::valid_utf8(s) == s) << s; + expect(text::valid_utf8("").empty()); + }; + + "each sequence that is not UTF-8 becomes U+FFFD, a maximal subpart at a time"_test = [] { + // A lead no sequence begins with, and a lone continuation byte. + expect(text::valid_utf8("a\xFF" "b") == "a" + std::string(fffd) + "b"); + expect(text::valid_utf8("\x80") == fffd); + // A sequence cut short is one subpart, at the end and before ASCII. + expect(text::valid_utf8("\xE6\x9C") == fffd); + expect(text::valid_utf8("\xE6\x9C" "a") == std::string(fffd) + "a"); + // An overlong form and a surrogate: the lead's second byte is out of + // its range, so each byte is its own subpart. + expect(text::valid_utf8("\xC0\xAF") == repeat(fffd, 2)); + expect(text::valid_utf8("\xED\xA0\x80") == repeat(fffd, 3)); + expect(text::valid_utf8("\xF4\x90\x80\x80") == repeat(fffd, 4)) << "beyond U+10FFFF"; + // GBK read as UTF-8: what mcpp#970 handed a JSON writer. + const auto gbk = text::valid_utf8("'\xCE\xB4\xB6\xA8\xD2\xE5_\xA8\xA6'"); + expect(text::valid_utf8(gbk) == gbk) << "the result is UTF-8"; + }; + + "a decoder of UTF-8 is the replacement above"_test = [] { + const text::decoder utf8(text::encoding::utf8); + expect(utf8.code_page() == 65001u); + expect(utf8("x\xFFy") == "x" + std::string(fffd) + "y"); + }; + + if constexpr (os::is_windows) { + "code pages 936, 1252 and 437 read as the text cl.exe wrote in them"_test = [] { + // b.cpp(2): error C2065: '未定义_é' under a console in 936. + expect(os::windows::utf8_from_code_page("'\xCE\xB4\xB6\xA8\xD2\xE5_\xA8\xA6'", 936) + == "'\xE6\x9C\xAA\xE5\xAE\x9A\xE4\xB9\x89_\xC3\xA9'"); + // The Chinese resources' prefix and quotes: 注意: 包含文件: and “缺失.h”. + expect(os::windows::utf8_from_code_page("\xD7\xA2\xD2\xE2: \xB0\xFC\xBA\xAC\xCE\xC4\xBC\xFE:", 936) + == "\xE6\xB3\xA8\xE6\x84\x8F: \xE5\x8C\x85\xE5\x90\xAB\xE6\x96\x87\xE4\xBB\xB6:"); + expect(os::windows::utf8_from_code_page("\xA1\xB0\xC8\xB1\xCA\xA7.h\xA1\xB1", 936) + == "\xE2\x80\x9C\xE7\xBC\xBA\xE5\xA4\xB1.h\xE2\x80\x9D"); + expect(os::windows::utf8_from_code_page("???_\xE9", 1252) == "???_\xC3\xA9"); + expect(os::windows::utf8_from_code_page("???_\x82", 437) == "???_\xC3\xA9"); + expect(os::windows::utf8_from_code_page("", 936).empty()); + expect(os::windows::utf8_from_code_page("x", 12345).empty()) << "a page the system does not know"; + }; + + "console is the page of the console this program is attached to, else the system's"_test = [] { + const unsigned page = os::windows::console_text_code_page(); + expect(page != 0u); + const text::decoder console(text::encoding::console); + expect(console.code_page() == page); + // A line of the probe in that page converts back to the text it was. + const std::string sample = "b.cpp(2): error C2065: 'x': undeclared identifier"; + expect(console(sample) == sample) << "ASCII is ASCII in every page"; + }; + } +}};