Fix some VFS issues

Author: digant73

Utilities/File.h

@@ -198,6 +198,19 @@ namespace fs
 	// Return "path" plus an ending delimiter (if missing) if "path" is an existing directory. Otherwise, an empty string
 	std::string get_path_if_dir(const std::string& path);
 
+	// Check whether the path is the bare name of a drive (e.g. "E:"), that is a root path whose trailing delimiter was
+	// trimmed: such a path does not point to the root of the drive but to the current directory of that drive, so the
+	// delimiter must be restored before using or storing it.
+	// NOTE: always false on the other platforms, where a name ending with ':' is a regular path component
+	inline bool is_drive_name([[maybe_unused]] std::string_view path)
+	{
+#ifdef _WIN32
+		return !path.empty() && path.back() == ':';
+#else
+		return false;
+#endif
+	}
+
 	// Get file information
 	bool get_stat(const std::string& path, stat_t& info);
 

rpcs3/Emu/System.cpp

@@ -49,7 +49,6 @@
 #include "util/sysinfo.hpp"
 
 #include <memory>
-#include <regex>
 #include <shared_mutex>
 
 #include "Utilities/JIT.h"
@@ -4412,16 +4411,30 @@ u32 Emulator::AddGamesFromDir(std::string path)
 
 	fmt::trim_back(path, fs::delim);
 
+	// Don't write "games.yml" on each added game: it is saved once, at the end of the scan.
+	// NOTE: this function is recursive, so the previous value is restored instead of being forced back to enabled,
+	//       otherwise a nested scan would re-enable the write for the remaining part of the outer one
+	const bool save_on_dirty = m_games_config.is_save_on_dirty();
 	m_games_config.set_save_on_dirty(false);
 
+	// A game was found on a path if it has just been added or if it was already registered
+	const auto game_found = [](game_boot_result error)
+	{
+		return error == game_boot_result::no_errors || error == game_boot_result::already_added;
+	};
+
 	// search for a game on the provided path first (game on ISO file or on folder type)
-	if (const game_boot_result error = AddGame(path); error == game_boot_result::no_errors)
+	const game_boot_result path_error = AddGame(path);
+
+	if (path_error == game_boot_result::no_errors)
 	{
 		games_added++;
 	}
 
-	// search for games on subfolders only if not nested inside a discovered game folder
-	if (games_added == 0)
+	// search for games on subfolders only if not nested inside a discovered game folder, otherwise the same title
+	// would be registered again through a different path (e.g. the root of a BD drive "E:/" is registered as a raw
+	// device, its subfolder "E:/PS3_GAME" would register it again as a disc folder)
+	if (!game_found(path_error))
 	{
 		std::vector<fs::dir_entry> entries;
 
@@ -4447,16 +4460,20 @@ u32 Emulator::AddGamesFromDir(std::string path)
 
 				const std::string dir_path = path + "/" + dir_entry.name;
 
-				if (!dir_entry.is_directory && !is_iso_file(dir_path))
+				// The outcome is handed over to "AddGame()" so that the ISO is not recognized twice: each check
+				// reads the volume descriptor, which is a physical read when the path points to an optical drive
+				const bool is_iso = !dir_entry.is_directory && is_iso_file(dir_path);
+
+				if (!dir_entry.is_directory && !is_iso)
 				{
 					continue;
 				}
 
-				if (const game_boot_result error = AddGame(dir_path); error == game_boot_result::no_errors)
+				if (const game_boot_result error = AddGame(dir_path, is_iso); error == game_boot_result::no_errors)
 				{
 					games_added++;
 				}
-				else if (g_cfg.misc.use_recursive_scan)
+				else if (!game_found(error) && g_cfg.misc.use_recursive_scan)
 				{
 					games_added += AddGamesFromDir(dir_path);
 				}
@@ -4471,24 +4488,25 @@ u32 Emulator::AddGamesFromDir(std::string path)
 		});
 	}
 
-	m_games_config.set_save_on_dirty(true);
+	m_games_config.set_save_on_dirty(save_on_dirty);
 
-	if (m_games_config.is_dirty() && !m_games_config.save())
+	// Flush the changes only when the outermost scan is done
+	if (save_on_dirty && m_games_config.is_dirty() && !m_games_config.save())
 	{
 		sys_log.error("Failed to save games.yml after adding games");
 	}
 
 	return games_added;
 }
 
-game_boot_result Emulator::AddGame(std::string path)
+game_boot_result Emulator::AddGame(std::string path, bool is_iso)
 {
 	fmt::trim_back(path, fs::delim);
 
 	// Handle files directly
 	if (!fs::is_dir(path) || fs::get_optical_raw_device(path))
 	{
-		return AddGameToYml(path);
+		return AddGameToYml(path, is_iso);
 	}
 
 	game_boot_result result = game_boot_result::nothing_to_boot;
@@ -4509,7 +4527,7 @@ game_boot_result Emulator::AddGame(std::string path)
 			continue;
 		}
 
-		if (entry.is_directory && std::regex_match(entry.name, std::regex("^PS3_GM[[:digit:]]{2}$")))
+		if (entry.is_directory && rpcs3::utils::is_ps3_gm_dir_name(entry.name))
 		{
 			const std::string elf = path + "/" + entry.name + "/USRDIR/EBOOT.BIN";
 
@@ -4530,7 +4548,7 @@ game_boot_result Emulator::AddGame(std::string path)
 	return result;
 }
 
-game_boot_result Emulator::AddGameToYml(std::string path)
+game_boot_result Emulator::AddGameToYml(std::string path, bool is_iso)
 {
 	fmt::trim_back(path, fs::delim);
 
@@ -4559,7 +4577,9 @@ game_boot_result Emulator::AddGameToYml(std::string path)
 	}
 
 	std::unique_ptr<iso_archive> archive;
-	if (is_iso_file(path))
+
+	// Skip the check if the caller already recognized the path as an ISO: it would read the volume descriptor again
+	if (is_iso || is_iso_file(path))
 	{
 		archive = std::make_unique<iso_archive>(path);
 
@@ -4826,7 +4846,7 @@ void Emulator::GetBdvdDir(std::string& bdvd_dir, std::string& sfb_dir, std::stri
 			continue;
 		}
 
-		if (dir_name == "PS3_GAME"sv || std::regex_match(dir_name.begin(), dir_name.end(), std::regex("^PS3_GM[[:digit:]]{2}$")))
+		if (dir_name == "PS3_GAME"sv || rpcs3::utils::is_ps3_gm_dir_name(dir_name))
 		{
 			if (IsValidSfb(parent_dir + "/PS3_DISC.SFB"))
 			{

rpcs3/Emu/System.h

@@ -496,8 +496,11 @@ class Emulator final
 
 	std::set<std::string> GetGameDirs() const;
 	u32 AddGamesFromDir(std::string path);
-	game_boot_result AddGame(std::string path);
-	game_boot_result AddGameToYml(std::string path);
+
+	// "is_iso" tells the caller has already recognized the path as an ISO file (or as a raw device holding a disc):
+	// checking it again would read its volume descriptor once more, which is a physical read on an optical drive
+	game_boot_result AddGame(std::string path, bool is_iso = false);
+	game_boot_result AddGameToYml(std::string path, bool is_iso = false);
 	u32 RemoveGamesFromDir(const std::string& games_dir, const std::vector<std::string>& serials_to_remove_from_yml = {}, bool save_on_disk = true);
 	u32 RemoveGames(const std::vector<std::string>& title_id_list, bool save_on_disk = true);
 	game_boot_result RemoveGameFromYml(const std::string& title_id);

rpcs3/Emu/games_config.cpp

@@ -55,6 +55,14 @@ games_config::result games_config::add_game(const std::string& key, const std::s
 		return add_game(key, iso_dev->get_loaded_iso());
 	}
 
+	if (fs::is_drive_name(path))
+	{
+		// Restore the trailing delimiter trimmed by the caller if the game is on the root of a raw device (e.g. "E:/"),
+		// which is also the form provided by "rpcs3::utils::get_games_dir()" and the one built by "GetBdvdDir()", so
+		// that the same title is always registered with the very same path
+		return add_game(key, path + '/');
+	}
+
 	std::lock_guard lock(m_mutex);
 
 	// Access or create node if does not exist

rpcs3/Emu/games_config.h

@@ -10,6 +10,7 @@ class games_config
 	virtual ~games_config();
 
 	void set_save_on_dirty(bool enabled) { m_save_on_dirty = enabled; }
+	bool is_save_on_dirty() const { return m_save_on_dirty; }
 
 	const std::map<std::string, std::string> get_games() const;
 	bool is_dirty() const { return m_dirty; }

rpcs3/Emu/system_utils.hpp

@@ -2,6 +2,7 @@
 
 #include "util/types.hpp"
 #include <string>
+#include <string_view>
 #include <set>
 #include <vector>
 
@@ -80,6 +81,13 @@ namespace rpcs3::utils
 	bool verify_c00_unlock_edat(const std::string_view& content_id, bool fast = false);
 	std::string get_sfo_dir_from_game_path(const std::string& game_path, const std::string& title_id = "");
 
+	// Check whether "name" is the name of an additional disc game directory (e.g. "PS3_GM01"), the counterpart of "PS3_GAME".
+	// NOTE: equivalent to the "^PS3_GM[[:digit:]]{2}$" regular expression, without building one for each checked name
+	inline bool is_ps3_gm_dir_name(std::string_view name)
+	{
+		return name.size() == 8 && name.starts_with("PS3_GM") && name[6] >= '0' && name[6] <= '9' && name[7] >= '0' && name[7] <= '9';
+	}
+
 	std::string get_custom_config_dir();
 	std::string get_custom_config_path(const std::string& identifier);
 

rpcs3/Loader/ISO.cpp

@@ -60,14 +60,15 @@ static void* get_aligned_buf()
 
 static bool is_iso_file(iso_file& file, u64* size = nullptr)
 {
-	if (!file || file.size() < 32768ULL + 6)
+	// The standard identifier ("CD001") follows the type of the first volume descriptor
+	if (!file || file.size() < ISO_DESCRIPTORS_OFFSET + 6)
 	{
 		return false;
 	}
 
 	char magic[5];
 
-	if (file.read_at(32768ULL + 1, magic, 5) != 5)
+	if (file.read_at(ISO_DESCRIPTORS_OFFSET + 1, magic, 5) != 5)
 	{
 		return false;
 	}
@@ -1029,18 +1030,30 @@ iso_archive::iso_archive(const std::string& path)
 	// "m_path" is updated with the raw device path in case "path" points to a BD drive
 	fs::get_optical_raw_device(path, &m_path);
 
-	if (!is_iso_file(m_path))
+	// NOTE: the file is opened once here and then handed over to the parsing below. Recognizing the ISO through its
+	//       path (i.e. "is_iso_file(m_path)") would open it and read its volume descriptor a second time, which is a
+	//       physical read when the path points to an optical drive
+	auto file = std::make_unique<iso_file>(m_path);
+
+	if (!is_iso_file(*file))
 	{
 		iso_log.error("iso_archive: Failed to recognize ISO file: '%s'", path);
 		invalidate();
 		return;
 	}
 
-	fs::file iso_file(std::make_unique<iso_file>(m_path));
+	// NOTE: "is_iso_file()" reads through "read_at()", which does not move the position, so the file is still at its
+	//       beginning here
+	fs::file iso_file(std::move(file));
 
 	u8 descriptor_type = -2;
 	bool use_ucs2_decoding = false;
 
+	// Skip the system area: scanning it sector by sector would read 16 sectors (a physical read each, on an optical
+	// drive) only to find boot data, which could even be mistaken for a volume descriptor.
+	// NOTE: "is_iso_file()" above already verified the standard identifier is right here
+	iso_file.seek(ISO_DESCRIPTORS_OFFSET);
+
 	do
 	{
 		const auto descriptor_start = iso_file.pos();

rpcs3/Loader/ISO.h

@@ -15,6 +15,9 @@ void unload_iso();
 
 constexpr u64 ISO_SECTOR_SIZE = 2048;
 
+// The first 16 sectors are the system area (boot data): the volume descriptors always start right after it (ECMA-119)
+constexpr u64 ISO_DESCRIPTORS_OFFSET = ISO_SECTOR_SIZE * 16;
+
 /*
 - Hijacked the "iso_archive::iso_archive" method to test if the ".iso" file is encrypted and sets a flag.
   The flag is set according to the first matching encryption type found following the order below:

rpcs3/rpcs3qt/game_list_frame.cpp

@@ -24,7 +24,6 @@
 
 #include <algorithm>
 #include <memory>
-#include <regex>
 #include <unordered_set>
 
 #include <QtConcurrent>
@@ -596,13 +595,24 @@ void game_list_frame::OnParsingFinished()
 	const auto add_game = [this, localized, localized_title, localized_icon, localized_movie, dev_flash, game_icon_path, _hdd,
 	                       cat_unknown_localized = localized->category.unknown.toStdString(), cat_unknown = cat::cat_unknown.toStdString(),
 	                       play_hover_movies = m_play_hover_movies, play_hover_music = m_play_hover_music, show_custom_icons = m_show_custom_icons]
-	                       (const std::string& dir_or_elf, const std::string& game_dir = "PS3_GAME")
+	                       (const std::string& dir_or_elf, const std::string& game_dir = "PS3_GAME", const std::shared_ptr<iso_archive>& shared_archive = {})
 	{
-		std::unique_ptr<iso_archive> archive;
+		std::shared_ptr<iso_archive> archive;
 		iso_metadata_cache_entry cache_entry{};
 		bool is_raw_device = false;
-		const bool is_archive = is_iso_file(dir_or_elf, nullptr, &is_raw_device);
+		bool is_archive = true;
 		std::string iso_cache_key;
+
+		if (shared_archive)
+		{
+			// The caller provides the archive only for a path it has already recognized: just the raw device flag is
+			// still needed here, and it is answered without reading the volume descriptor of the disc
+			is_raw_device = fs::get_optical_raw_device(dir_or_elf);
+		}
+		else
+		{
+			is_archive = is_iso_file(dir_or_elf, nullptr, &is_raw_device);
+		}
 
 		if (is_archive)
 		{
@@ -611,7 +621,9 @@ void game_list_frame::OnParsingFinished()
 			// when no valid cache entry exists for this ISO path + mtime
 			if (is_raw_device || !iso_cache::load(dir_or_elf, iso_cache_key, cache_entry))
 			{
-				archive = std::make_unique<iso_archive>(dir_or_elf);
+				// Reuse the archive the caller has already built for this path: constructing another one walks the
+				// whole file system of the disc again (and a raw device never uses the cache)
+				archive = shared_archive ? shared_archive : std::make_shared<iso_archive>(dir_or_elf);
 				if (!archive->is_valid()) return;
 			}
 
@@ -648,7 +660,7 @@ void game_list_frame::OnParsingFinished()
 			if (!psf_valid)
 			{
 				game_list_log.warning("Cached psf for iso not valid: '%s'", game.info.path);
-				archive = std::make_unique<iso_archive>(dir_or_elf);
+				archive = shared_archive ? shared_archive : std::make_shared<iso_archive>(dir_or_elf);
 				if (!archive->is_valid()) return;
 
 				cache_entry = {}; // Reset so the cache gets rewritten after scan.
@@ -896,7 +908,7 @@ void game_list_frame::OnParsingFinished()
 				continue;
 			}
 
-			if (entry.name == "PS3_GAME" || std::regex_match(entry.name, std::regex("^PS3_GM[[:digit:]]{2}$")))
+			if (entry.name == "PS3_GAME" || rpcs3::utils::is_ps3_gm_dir_name(entry.name))
 			{
 				push_path(path + "/" + entry.name, legit_paths);
 			}
@@ -924,11 +936,11 @@ void game_list_frame::OnParsingFinished()
 					return;
 				}
 
-				iso_archive archive(entry.path);
-				if (!archive.is_valid()) return;
+				// Shared with "add_game()" below, so that the file system of the disc is walked only once
+				const auto archive = std::make_shared<iso_archive>(entry.path);
+				if (!archive->is_valid()) return;
 
-				const iso_fs_node& root = archive.root();
-				const std::regex ps3_gm_regex("^PS3_GM[[:digit:]]{2}$");
+				const iso_fs_node& root = archive->root();
 
 				for (const auto& child : root.children)
 				{
@@ -942,15 +954,15 @@ void game_list_frame::OnParsingFinished()
 					}
 
 					const std::string& name = child->metadata.name;
-					if (name == "PS3_GAME" || std::regex_match(name, ps3_gm_regex))
+					if (name == "PS3_GAME" || rpcs3::utils::is_ps3_gm_dir_name(name))
 					{
 						subdirs.push_back(name);
-						add_game(entry.path, name);
+						add_game(entry.path, name, archive);
 					}
 				}
 				if (subdirs.empty())
 				{
-					add_game(entry.path);
+					add_game(entry.path, "PS3_GAME", archive);
 					subdirs.push_back("PS3_GAME");
 				}
 				if (!m_refresh_watcher.isCanceled())