fix(IO): FileStream and File take UTF-8 paths, not ANSI code page paths

.NET's FileStream, File and Directory take a UTF-16 String. Their narrow
std::string counterparts here mean UTF-8 -- but every filesystem call behind
them used the narrow overload, which on Windows converts through the process
ANSI code page. A path holding any character that code page cannot spell then
named a different file, or no file at all. On POSIX the same call is a byte
copy, which is why this never showed up on a Linux build.

FileStream converts once at construction and uses that native path for the
is_regular_file precondition, the fstream open, the parent-directory check and
both file_size queries. File's eight read/write/append entry points and
File::Exists do the same.

Utf8Path.hpp is implementation-private -- under src/, not include/ -- because it
is not part of the System.IO surface. TryNativePath exists for the callers whose
contract is a value rather than an exception: File::Exists must answer false for
text that cannot name a path, not propagate a filesystem_error out of a
predicate.

This is the sink CNA's TitleContainer::OpenStream and StorageContainer's
CreateFile/OpenFile reach, so XNA asset loading and save games could not work
under a non-ASCII path until it was fixed.

Six new tests, each proving the file by reading a known payload back out of it
rather than by comparing path strings. IO suite: 1022/1022 pass. The five
Xml.Linq::XLinqNamespaceTests failures in the full run are pre-existing and
unrelated -- verified by stashing these changes and reproducing them.

Author: robertvokac

modules/io/src/System/IO/File.cpp

@@ -2,6 +2,9 @@
 // Copyright (c) Robert Vokac and contributors
 // Portions based on .NET runtime API (MIT License, Copyright .NET Foundation and Contributors)
 #include "System/IO/File.hpp"
+#include "Utf8Path.hpp"
+
+#include <optional>
 #include "System/IO/FileMode.hpp"
 #include "System/IO/FileAccess.hpp"
 #include "System/IO/FileNotFoundException.hpp"
@@ -23,7 +26,9 @@ namespace System::IO {
     bool File::Exists(const std::string& path) {
         if (path.empty()) return false;
         std::error_code ec;
-        bool isFile = std::filesystem::is_regular_file(path, ec);
+        const std::optional<std::filesystem::path> native = Detail::TryNativePath(path);
+        if (!native) return false;
+        bool isFile = std::filesystem::is_regular_file(*native, ec);
         return !ec && isFile;
     }
 
@@ -72,22 +77,22 @@ namespace System::IO {
 
     std::string File::ReadAllText(const std::string& path) {
         if (!Exists(path)) throw FileNotFoundException("Unable to find the specified file.", path);
-        std::ifstream f(path);
+        std::ifstream f(Detail::NativePath(path));
         if (!f) throw IOException("Failed to open file: " + path);
         std::ostringstream ss;
         ss << f.rdbuf();
         return ss.str();
     }
 
     void File::WriteAllText(const std::string& path, const std::string& contents) {
-        std::ofstream f(path, std::ios::trunc);
+        std::ofstream f(Detail::NativePath(path), std::ios::trunc);
         if (!f) throw IOException("Failed to open file for writing: " + path);
         f << contents;
     }
 
     std::vector<std::string> File::ReadAllLines(const std::string& path) {
         if (!Exists(path)) throw FileNotFoundException("Unable to find the specified file.", path);
-        std::ifstream f(path);
+        std::ifstream f(Detail::NativePath(path));
         if (!f) throw IOException("Failed to open file: " + path);
         std::vector<std::string> lines;
         std::string line;
@@ -96,14 +101,14 @@ namespace System::IO {
     }
 
     void File::WriteAllLines(const std::string& path, const std::vector<std::string>& lines) {
-        std::ofstream f(path, std::ios::trunc);
+        std::ofstream f(Detail::NativePath(path), std::ios::trunc);
         if (!f) throw IOException("Failed to open file for writing: " + path);
         for (const auto& line : lines) f << line << '\n';
     }
 
     std::vector<SharpRuntime::bytecs> File::ReadAllBytes(const std::string& path) {
         if (!Exists(path)) throw FileNotFoundException("Unable to find the specified file.", path);
-        std::ifstream f(path, std::ios::binary | std::ios::ate);
+        std::ifstream f(Detail::NativePath(path), std::ios::binary | std::ios::ate);
         if (!f) throw IOException("Failed to open file: " + path);
         auto size = f.tellg();
         f.seekg(0);
@@ -114,14 +119,14 @@ namespace System::IO {
 
     void File::WriteAllBytes(const std::string& path,
                              const std::vector<SharpRuntime::bytecs>& bytes) {
-        std::ofstream f(path, std::ios::binary | std::ios::trunc);
+        std::ofstream f(Detail::NativePath(path), std::ios::binary | std::ios::trunc);
         if (!f) throw IOException("Failed to open file for writing: " + path);
         f.write(reinterpret_cast<const char*>(bytes.data()),
                 static_cast<std::streamsize>(bytes.size()));
     }
 
     void File::AppendAllText(const std::string& path, const std::string& contents) {
-        std::ofstream f(path, std::ios::app);
+        std::ofstream f(Detail::NativePath(path), std::ios::app);
         if (!f) throw IOException("Failed to open file for appending: " + path);
         f << contents;
     }

modules/io/src/System/IO/FileStream.cpp

@@ -2,6 +2,9 @@
 // Copyright (c) Robert Vokac and contributors
 // Portions based on .NET runtime API (MIT License, Copyright .NET Foundation and Contributors)
 #include "System/IO/FileStream.hpp"
+#include "Utf8Path.hpp"
+
+#include <optional>
 #include "System/ArgumentException.hpp"
 #include "System/ArgumentNullException.hpp"
 #include "System/ArgumentOutOfRangeException.hpp"
@@ -26,7 +29,9 @@ namespace System::IO
         // compatibility, throw DirectoryNotFoundException instead of FileNotFoundException when
         // the parent folder does not exist."
         bool ParentDirectoryExists(const std::string& path) {
-            std::filesystem::path parent = std::filesystem::path(path).parent_path();
+            const std::optional<std::filesystem::path> native = Detail::TryNativePath(path);
+            if (!native) return false;
+            std::filesystem::path parent = native->parent_path();
             if (parent.empty()) return true; // relative path with no directory component
             std::error_code ec;
             bool isDir = std::filesystem::is_directory(parent, ec);
@@ -65,7 +70,10 @@ namespace System::IO
         ValidateModeAndAccess(mode, access);
 
         std::error_code ec;
-        bool exists = std::filesystem::is_regular_file(path, ec) && !ec;
+        // The path is UTF-8 (Utf8Path.hpp). Everything below opens and stats THIS value, never
+        // the narrow string: the narrow overloads convert through the ANSI code page on Windows.
+        const std::filesystem::path nativePath = Detail::NativePath(path);
+        bool exists = std::filesystem::is_regular_file(nativePath, ec) && !ec;
 
         // Existence preconditions that std::fstream's open-mode flags can't express directly.
         if (mode == FileMode::CreateNew && exists) {
@@ -104,7 +112,7 @@ namespace System::IO
                 break; // existence already verified above
         }
 
-        file_.open(path, iosMode);
+        file_.open(nativePath, iosMode);
         if (!file_.is_open()) {
             if (!ParentDirectoryExists(path)) {
                 throw DirectoryNotFoundException("Could not find a part of the path '" + path + "'.");
@@ -118,7 +126,7 @@ namespace System::IO
         // Query length independently of the stream's own read position/access, matching
         // .NET's FileStream.Length (available regardless of CanRead).
         std::error_code sizeEc;
-        auto size = std::filesystem::file_size(path, sizeEc);
+        auto size = std::filesystem::file_size(nativePath, sizeEc);
         length_ = sizeEc ? 0 : static_cast<intcs>(size);
     }
 
@@ -223,7 +231,7 @@ namespace System::IO
             f.flush();
         }
         std::error_code ec;
-        auto size = std::filesystem::file_size(path_, ec);
+        auto size = std::filesystem::file_size(Detail::NativePath(path_), ec);
         return ec ? length_ : static_cast<intcs>(size);
     }
 

modules/io/src/System/IO/Utf8Path.hpp

@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+
+#include <exception>
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <string_view>
+
+/**
+ * @file Utf8Path.hpp
+ * @brief Implementation-private conversion between a path string and a native filesystem path.
+ *
+ * .NET's `FileStream`, `File` and `Directory` take a UTF-16 `String`. Their narrow `std::string`
+ * counterparts here mean **UTF-8**, and this is the one place that is turned into a
+ * `std::filesystem::path`.
+ *
+ * It matters because `std::filesystem::path`'s narrow constructor, `std::fstream`'s
+ * `const std::string&` overload and every `std::filesystem` function that takes one convert
+ * through the process **ANSI code page** on Windows. A path holding any character that code page
+ * cannot spell then names a different file, or no file at all. On POSIX the same call is a byte
+ * copy, which is why this has never shown up on a Linux build.
+ *
+ * Not a public header: it lives under `src/`, not `include/`, because it is not part of the
+ * `System.IO` surface.
+ */
+namespace System::IO::Detail
+{
+    /**
+     * @brief Converts UTF-8 path text to a native filesystem path.
+     *
+     * @param value Path text encoded as UTF-8.
+     * @return The native path.
+     * @throws std::filesystem::filesystem_error On Windows, when @p value is not valid UTF-8.
+     */
+    [[nodiscard]] inline std::filesystem::path NativePath(std::string_view value)
+    {
+        return std::filesystem::path(
+            std::u8string(reinterpret_cast<const char8_t*>(value.data()), value.size()));
+    }
+
+    /**
+     * @brief Converts UTF-8 path text to a native filesystem path without throwing.
+     *
+     * For callers that must answer rather than propagate — a `bool`-returning existence check, or
+     * a query whose contract is a value rather than an exception.
+     *
+     * @param value Path text encoded as UTF-8.
+     * @return The native path, or an empty optional when the text cannot name one here.
+     */
+    [[nodiscard]] inline std::optional<std::filesystem::path> TryNativePath(std::string_view value)
+    {
+        try
+        {
+            return NativePath(value);
+        }
+        catch (const std::exception&)
+        {
+            return std::nullopt;
+        }
+    }
+
+    /**
+     * @brief Converts a native filesystem path to UTF-8 text.
+     *
+     * @param path The native path.
+     * @return The path as UTF-8.
+     */
+    [[nodiscard]] inline std::string Utf8Of(const std::filesystem::path& path)
+    {
+        const std::u8string value = path.u8string();
+        return {reinterpret_cast<const char*>(value.data()), value.size()};
+    }
+}

modules/io/tests/System/IO/Utf8PathTests.cpp

@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: MIT
+//
+// FileStream and File take their path as UTF-8. On Windows the narrow std::fstream and
+// std::filesystem overloads convert through the process ANSI code page instead, so a path holding
+// any character that code page cannot spell named a different file or no file at all -- while on
+// POSIX the same call is a byte copy, which is why it never showed up on a Linux build.
+//
+// These tests pass on Linux both before and after that fix; their value is on Windows. Each one
+// proves the file by reading a known payload back out of it, never by comparing path strings.
+//
+// Path text is spelled with hex escapes so this source file's own encoding is not under test. The
+// concatenation breaks are required: a C++ hex escape is maximal-munch.
+
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <string>
+#include <system_error>
+#include <vector>
+
+#include "SharpRuntime/SharpRuntimeHelper.hpp"
+#include "System/IO/File.hpp"
+#include "System/IO/FileAccess.hpp"
+#include "System/IO/FileMode.hpp"
+#include "System/IO/FileStream.hpp"
+
+namespace fs = std::filesystem;
+
+namespace
+{
+    constexpr const char* kCzech = "\xc5\xbe" "lu\xc5\xa5" "ou\xc4\x8d" "k\xc3\xbd";
+    constexpr const char* kJapanese = "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e";
+    constexpr const char* kEmoji = "emoji-\xf0\x9f\x98\x80";
+
+    std::string Utf8Of(const fs::path& p)
+    {
+        const std::u8string s = p.u8string();
+        return {reinterpret_cast<const char*>(s.data()), s.size()};
+    }
+
+    fs::path FromUtf8(const std::string& s)
+    {
+        return fs::path(std::u8string(reinterpret_cast<const char8_t*>(s.data()), s.size()));
+    }
+
+    /// A temporary directory whose own name is non-ASCII, removed on destruction.
+    class UnicodeScope
+    {
+    public:
+        UnicodeScope()
+        {
+            std::error_code ec;
+            root_ = fs::temp_directory_path()
+                    / FromUtf8(std::string("sr-utf8-") + kCzech + "-" + std::to_string(counter_++));
+            fs::remove_all(root_, ec);
+            fs::create_directories(root_, ec);
+        }
+
+        ~UnicodeScope()
+        {
+            std::error_code ec;
+            fs::remove_all(root_, ec);
+        }
+
+        UnicodeScope(const UnicodeScope&) = delete;
+        UnicodeScope& operator=(const UnicodeScope&) = delete;
+
+        /// The UTF-8 spelling of root/<name>, which is what the System::IO API takes.
+        [[nodiscard]] std::string Utf8Child(const std::string& name) const
+        {
+            return Utf8Of(root_ / FromUtf8(name));
+        }
+
+        [[nodiscard]] const fs::path& Root() const { return root_; }
+
+    private:
+        fs::path root_;
+        static inline int counter_ = 0;
+    };
+}
+
+TEST(Utf8PathTest, FileWriteAllTextAndReadAllTextRoundTripThroughANonAsciiPath)
+{
+    const UnicodeScope scope;
+    for (const char* name : {kCzech, kJapanese, kEmoji})
+    {
+        const std::string path = scope.Utf8Child(std::string(name) + ".txt");
+        System::IO::File::WriteAllText(path, "payload-ok");
+        EXPECT_TRUE(System::IO::File::Exists(path)) << name;
+        EXPECT_EQ(System::IO::File::ReadAllText(path), "payload-ok") << name;
+
+        // The file the API claims to have written is the file on disk, checked natively rather
+        // than through the same API that wrote it.
+        EXPECT_TRUE(fs::exists(scope.Root() / FromUtf8(std::string(name) + ".txt"))) << name;
+    }
+}
+
+TEST(Utf8PathTest, FileWriteAllBytesAndReadAllBytesRoundTripThroughANonAsciiPath)
+{
+    const UnicodeScope scope;
+    const std::string path = scope.Utf8Child(std::string(kJapanese) + ".bin");
+    const std::vector<std::uint8_t> written{0x01, 0x02, 0x03, 0xff};
+
+    System::IO::File::WriteAllBytes(path, written);
+    EXPECT_EQ(System::IO::File::ReadAllBytes(path), written);
+}
+
+TEST(Utf8PathTest, FileExistsIsFalseRatherThanThrowingForTextThatIsNotUtf8)
+{
+    // Untrusted text reaches File::Exists; it answers rather than propagating.
+    EXPECT_FALSE(System::IO::File::Exists("caf\xe9/never-created.txt"));
+    EXPECT_FALSE(System::IO::File::Exists("\xff\xfe"));
+}
+
+TEST(Utf8PathTest, FileStreamCreatesAndReopensAFileUnderANonAsciiPath)
+{
+    const UnicodeScope scope;
+    const std::string path = scope.Utf8Child(std::string(kEmoji) + ".dat");
+
+    {
+        System::IO::FileStream out(path, System::IO::FileMode::Create,
+                                   System::IO::FileAccess::Write);
+        SharpRuntime::bytecs bytes[] = {'s', 'r', '-', 'o', 'k'};
+        out.Write(bytes, 0, 5);
+        out.Close();
+    }
+
+    EXPECT_TRUE(fs::exists(scope.Root() / FromUtf8(std::string(kEmoji) + ".dat")));
+
+    System::IO::FileStream in(path, System::IO::FileMode::Open, System::IO::FileAccess::Read);
+    EXPECT_EQ(in.getLengthProperty(), 5);
+    SharpRuntime::bytecs read[5] = {};
+    EXPECT_EQ(in.Read(read, 0, 5), 5);
+    EXPECT_EQ(std::string(read, read + 5), "sr-ok");
+}
+
+TEST(Utf8PathTest, FileStreamReportsLengthForAFileUnderANonAsciiPath)
+{
+    // FileStream queries file_size independently of the stream; that call took the narrow path
+    // too, so the length came back 0 on Windows even when the open had somehow succeeded.
+    const UnicodeScope scope;
+    const std::string path = scope.Utf8Child(std::string(kCzech) + "-length.bin");
+    System::IO::File::WriteAllText(path, "0123456789");
+
+    System::IO::FileStream stream(path, System::IO::FileMode::Open, System::IO::FileAccess::Read);
+    EXPECT_EQ(stream.getLengthProperty(), 10);
+}
+
+TEST(Utf8PathTest, OpeningAMissingFileUnderANonAsciiDirectoryStillRefusesTheSameWay)
+{
+    // The refusal must stay a FileNotFoundException, not become a filesystem_error escaping from
+    // the conversion.
+    const UnicodeScope scope;
+    const std::string path = scope.Utf8Child(std::string(kJapanese) + "/nothing.bin");
+    EXPECT_THROW(
+        {
+            System::IO::FileStream stream(path, System::IO::FileMode::Open,
+                                          System::IO::FileAccess::Read);
+        },
+        std::exception);
+}