Fix some VFS issues - #19378

Utilities/File.h

@@ -198,6 +198,19 @@ namespace fs
 	// Return "path" plus an ending delimiter (if missing) if "path" is an existing directory. Otherwise, an empty string
 	std::string get_path_if_dir(const std::string& path);
 
+	// Check whether the path is the bare name of a drive (e.g. "E:"), that is a root path whose trailing delimiter was
+	// trimmed: such a path does not point to the root of the drive but to the current directory of that drive, so the
+	// delimiter must be restored before using or storing it.
+	// NOTE: always false on the other platforms, where a name ending with ':' is a regular path component
+	inline bool is_drive_name([[maybe_unused]] std::string_view path)
+	{
+#ifdef _WIN32
+		return !path.empty() && path.back() == ':';
+#else
+		return false;
+#endif
+	}
+
 	// Get file information
 	bool get_stat(const std::string& path, stat_t& info);
 

rpcs3/Emu/System.cpp

@@ -49,7 +49,6 @@
 #include "util/sysinfo.hpp"
 
 #include <memory>
-#include <regex>
 #include <shared_mutex>
 
 #include "Utilities/JIT.h"
@@ -4420,16 +4419,30 @@ u32 Emulator::AddGamesFromDir(std::string path)
 
 	fmt::trim_back(path, fs::delim);
 
+	// Don't write "games.yml" on each added game: it is saved once, at the end of the scan.
+	// NOTE: this function is recursive, so the previous value is restored instead of being forced back to enabled,
+	//       otherwise a nested scan would re-enable the write for the remaining part of the outer one
+	const bool save_on_dirty = m_games_config.is_save_on_dirty();
 	m_games_config.set_save_on_dirty(false);
 
+	// A game was found on a path if it has just been added or if it was already registered
+	const auto game_found = [](game_boot_result error)
+	{
+		return error == game_boot_result::no_errors || error == game_boot_result::already_added;
+	};
+
 	// search for a game on the provided path first (game on ISO file or on folder type)
-	if (const game_boot_result error = AddGame(path); error == game_boot_result::no_errors)
+	const game_boot_result path_error = AddGame(path);
+
+	if (path_error == game_boot_result::no_errors)
 	{
 		games_added++;
 	}
 
-	// search for games on subfolders only if not nested inside a discovered game folder
-	if (games_added == 0)
+	// search for games on subfolders only if not nested inside a discovered game folder, otherwise the same title
+	// would be registered again through a different path (e.g. the root of a BD drive "E:/" is registered as a raw
+	// device, its subfolder "E:/PS3_GAME" would register it again as a disc folder)
+	if (!game_found(path_error))
 	{
 		std::vector<fs::dir_entry> entries;
 
@@ -4455,16 +4468,20 @@ u32 Emulator::AddGamesFromDir(std::string path)
 
 				const std::string dir_path = path + "/" + dir_entry.name;
 
-				if (!dir_entry.is_directory && !is_iso_file(dir_path))
+				// The outcome is handed over to "AddGame()" so that the ISO is not recognized twice: each check
+				// reads the volume descriptor, which is a physical read when the path points to an optical drive
+				const bool is_iso = !dir_entry.is_directory && is_iso_file(dir_path);
+
+				if (!dir_entry.is_directory && !is_iso)
 				{
 					continue;
 				}
 
-				if (const game_boot_result error = AddGame(dir_path); error == game_boot_result::no_errors)
+				if (const game_boot_result error = AddGame(dir_path, is_iso); error == game_boot_result::no_errors)
 				{
 					games_added++;
 				}
-				else if (g_cfg.misc.use_recursive_scan)
+				else if (!game_found(error) && g_cfg.misc.use_recursive_scan)
 				{
 					games_added += AddGamesFromDir(dir_path);
 				}
@@ -4479,9 +4496,10 @@ u32 Emulator::AddGamesFromDir(std::string path)
 		});
 	}
 
-	m_games_config.set_save_on_dirty(true);
+	m_games_config.set_save_on_dirty(save_on_dirty);
 
-	if (m_games_config.is_dirty() && !m_games_config.save())
+	// Flush the changes only when the outermost scan is done
+	if (save_on_dirty && m_games_config.is_dirty() && !m_games_config.save())
 	{
 		sys_log.error("Failed to save games.yml after adding games");
 	}
@@ -4489,14 +4507,14 @@ u32 Emulator::AddGamesFromDir(std::string path)
 	return games_added;
 }
 
-game_boot_result Emulator::AddGame(std::string path)
+game_boot_result Emulator::AddGame(std::string path, bool is_iso)
 {
 	fmt::trim_back(path, fs::delim);
 
 	// Handle files directly
 	if (!fs::is_dir(path) || fs::get_optical_raw_device(path))
 	{
-		return AddGameToYml(path);
+		return AddGameToYml(path, is_iso);
 	}
 
 	game_boot_result result = game_boot_result::nothing_to_boot;
@@ -4517,7 +4535,7 @@ game_boot_result Emulator::AddGame(std::string path)
 			continue;
 		}
 
-		if (entry.is_directory && std::regex_match(entry.name, std::regex("^PS3_GM[[:digit:]]{2}$")))
+		if (entry.is_directory && rpcs3::utils::is_ps3_gm_dir_name(entry.name))
 		{
 			const std::string elf = path + "/" + entry.name + "/USRDIR/EBOOT.BIN";
 
@@ -4538,7 +4556,7 @@ game_boot_result Emulator::AddGame(std::string path)
 	return result;
 }
 
-game_boot_result Emulator::AddGameToYml(std::string path)
+game_boot_result Emulator::AddGameToYml(std::string path, bool is_iso)
 {
 	fmt::trim_back(path, fs::delim);
 
@@ -4567,7 +4585,9 @@ game_boot_result Emulator::AddGameToYml(std::string path)
 	}
 
 	std::unique_ptr<iso_archive> archive;
-	if (is_iso_file(path))
+
+	// Skip the check if the caller already recognized the path as an ISO: it would read the volume descriptor again
+	if (is_iso || is_iso_file(path))
 	{
 		archive = std::make_unique<iso_archive>(path);
 
@@ -4834,7 +4854,7 @@ void Emulator::GetBdvdDir(std::string& bdvd_dir, std::string& sfb_dir, std::stri
 			continue;
 		}
 
-		if (dir_name == "PS3_GAME"sv || std::regex_match(dir_name.begin(), dir_name.end(), std::regex("^PS3_GM[[:digit:]]{2}$")))
+		if (dir_name == "PS3_GAME"sv || rpcs3::utils::is_ps3_gm_dir_name(dir_name))
 		{
 			if (IsValidSfb(parent_dir + "/PS3_DISC.SFB"))
 			{

rpcs3/Emu/System.h

@@ -496,8 +496,11 @@ public:
 
 	std::set<std::string> GetGameDirs() const;
 	u32 AddGamesFromDir(std::string path);
-	game_boot_result AddGame(std::string path);
-	game_boot_result AddGameToYml(std::string path);
+
+	// "is_iso" tells the caller has already recognized the path as an ISO file (or as a raw device holding a disc):
+	// checking it again would read its volume descriptor once more, which is a physical read on an optical drive
+	game_boot_result AddGame(std::string path, bool is_iso = false);
+	game_boot_result AddGameToYml(std::string path, bool is_iso = false);
 	u32 RemoveGamesFromDir(const std::string& games_dir, const std::vector<std::string>& serials_to_remove_from_yml = {}, bool save_on_disk = true);
 	u32 RemoveGames(const std::vector<std::string>& title_id_list, bool save_on_disk = true);
 	game_boot_result RemoveGameFromYml(const std::string& title_id);

rpcs3/Emu/game_enumeration.h

@@ -13,7 +13,6 @@
 #include <set>
 #include <unordered_set>
 #include <vector>
-#include <regex>
 
 LOG_CHANNEL(sys_log, "SYS");
 
@@ -50,12 +49,14 @@ public:
 	std::vector<game_info_type> take_games() { return std::move(m_games); }
 
 private:
-	std::optional<game_info_type> get_game_info(const std::string& dir_or_elf, const std::string& game_dir, bool is_iso, bool is_raw);
+	// "shared_archive" is the archive the caller has already built for "dir_or_elf", if any: constructing another one
+	// walks the whole file system of the disc again
+	std::optional<game_info_type> get_game_info(const std::string& dir_or_elf, const std::string& game_dir, bool is_iso, bool is_raw, const std::shared_ptr<iso_archive>& shared_archive = {});
 
 	bool was_canceled() const { return m_canceled_callback && m_canceled_callback(); }
 
 	void push_path(const std::string& path, std::vector<std::string>& legit_paths);
-	void add_game(const std::string& path, const std::string& game_dir = "PS3_GAME", bool is_iso = false, bool is_raw = false);
+	void add_game(const std::string& path, const std::string& game_dir = "PS3_GAME", bool is_iso = false, bool is_raw = false, const std::shared_ptr<iso_archive>& shared_archive = {});
 	virtual void add_game_apply_extras([[maybe_unused]] game_info_type& game) {}
 	void add_disc_dir(const std::string& path, std::vector<std::string>& legit_paths);
 
@@ -106,11 +107,11 @@ void game_enumeration<game_info_type>::set_localization(s32 index, std::string&&
 }
 
 template <typename game_info_type>
-std::optional<game_info_type> game_enumeration<game_info_type>::get_game_info(const std::string& dir_or_elf, const std::string& game_dir, bool is_iso, bool is_raw)
+std::optional<game_info_type> game_enumeration<game_info_type>::get_game_info(const std::string& dir_or_elf, const std::string& game_dir, bool is_iso, bool is_raw, const std::shared_ptr<iso_archive>& shared_archive)
 {
 	game_info_type info{};
 
-	std::unique_ptr<iso_archive> archive;
+	std::shared_ptr<iso_archive> archive;
 	iso_metadata_cache_entry cache_entry{};
 	bool is_raw_device = is_raw;
 	info.is_iso_file = is_iso && is_iso_file(dir_or_elf, nullptr, &is_raw_device);
@@ -123,7 +124,8 @@ std::optional<game_info_type> game_enumeration<game_info_type>::get_game_info(co
 		// when no valid cache entry exists for this ISO path + mtime
 		if (is_raw_device || !iso_cache::load(dir_or_elf, iso_cache_key, cache_entry))
 		{
-			archive = std::make_unique<iso_archive>(dir_or_elf);
+			// Reuse the archive the caller has already built for this path (a raw device never uses the cache)
+			archive = shared_archive ? shared_archive : std::make_shared<iso_archive>(dir_or_elf);
 			if (!archive->is_valid()) return std::nullopt;
 		}
 
@@ -159,7 +161,7 @@ std::optional<game_info_type> game_enumeration<game_info_type>::get_game_info(co
 		if (!psf_valid)
 		{
 			sys_log.warning("Cached psf for iso not valid: '%s'", info.path);
-			archive = std::make_unique<iso_archive>(dir_or_elf);
+			archive = shared_archive ? shared_archive : std::make_shared<iso_archive>(dir_or_elf);
 			if (!archive->is_valid()) return std::nullopt;
 
 			cache_entry = {}; // Reset so the cache gets rewritten after scan.
@@ -347,9 +349,9 @@ void game_enumeration<game_info_type>::push_path(const std::string& path, std::v
 }
 
 template <typename game_info_type>
-void game_enumeration<game_info_type>::add_game(const std::string& path, const std::string& game_dir, bool is_iso, bool is_raw)
+void game_enumeration<game_info_type>::add_game(const std::string& path, const std::string& game_dir, bool is_iso, bool is_raw, const std::shared_ptr<iso_archive>& shared_archive)
 {
-	if (std::optional<game_info_type> game = get_game_info(path, game_dir, is_iso, is_raw))
+	if (std::optional<game_info_type> game = get_game_info(path, game_dir, is_iso, is_raw, shared_archive))
 	{
 		add_game_apply_extras(*game);
 
@@ -373,7 +375,7 @@ void game_enumeration<game_info_type>::add_disc_dir(const std::string& path, std
 			continue;
 		}
 
-		if (entry.name == "PS3_GAME" || std::regex_match(entry.name, std::regex("^PS3_GM[[:digit:]]{2}$")))
+		if (entry.name == "PS3_GAME" || rpcs3::utils::is_ps3_gm_dir_name(entry.name))
 		{
 			push_path(path + "/" + entry.name, legit_paths);
 		}
@@ -450,11 +452,11 @@ void game_enumeration<game_info_type>::parse_entry(const path_entry& entry)
 				return;
 			}
 
-			iso_archive archive(entry.path);
-			if (!archive.is_valid()) return;
+			// Shared with "add_game()" below, so that the file system of the disc is walked only once
+			const auto archive = std::make_shared<iso_archive>(entry.path);
+			if (!archive->is_valid()) return;
 
-			const iso_fs_node& root = archive.root();
-			const std::regex ps3_gm_regex("^PS3_GM[[:digit:]]{2}$");
+			const iso_fs_node& root = archive->root();
 
 			for (const auto& child : root.children)
 			{
@@ -468,16 +470,16 @@ void game_enumeration<game_info_type>::parse_entry(const path_entry& entry)
 				}
 
 				const std::string& name = child->metadata.name;
-				if (name == "PS3_GAME" || std::regex_match(name, ps3_gm_regex))
+				if (name == "PS3_GAME" || rpcs3::utils::is_ps3_gm_dir_name(name))
 				{
 					subdirs.push_back(name);
-					add_game(entry.path, name, true, is_raw_device);
+					add_game(entry.path, name, true, is_raw_device, archive);
 				}
 			}
 			if (subdirs.empty())
 			{
 				subdirs.push_back("PS3_GAME");
-				add_game(entry.path, "PS3_GAME", true, is_raw_device);
+				add_game(entry.path, "PS3_GAME", true, is_raw_device, archive);
 			}
 			if (!was_canceled())
 			{

rpcs3/Emu/games_config.cpp

@@ -55,6 +55,14 @@ games_config::result games_config::add_game(const std::string& key, const std::s
 		return add_game(key, iso_dev->get_loaded_iso());
 	}
 
+	if (fs::is_drive_name(path))
+	{
+		// Restore the trailing delimiter trimmed by the caller if the game is on the root of a raw device (e.g. "E:/"),
+		// which is also the form provided by "rpcs3::utils::get_games_dir()" and the one built by "GetBdvdDir()", so
+		// that the same title is always registered with the very same path
+		return add_game(key, path + '/');
+	}
+
 	std::lock_guard lock(m_mutex);
 
 	// Access or create node if does not exist

rpcs3/Emu/games_config.h

@@ -10,6 +10,7 @@ public:
 	virtual ~games_config();
 
 	void set_save_on_dirty(bool enabled) { m_save_on_dirty = enabled; }
+	bool is_save_on_dirty() const { return m_save_on_dirty; }
 
 	const std::map<std::string, std::string> get_games() const;
 	bool is_dirty() const { return m_dirty; }

rpcs3/Emu/system_utils.hpp

@@ -2,6 +2,7 @@
 
 #include "util/types.hpp"
 #include <string>
+#include <string_view>
 #include <set>
 #include <vector>
 
@@ -80,6 +81,13 @@ namespace rpcs3::utils
 	bool verify_c00_unlock_edat(const std::string_view& content_id, bool fast = false);
 	std::string get_sfo_dir_from_game_path(const std::string& game_path, const std::string& title_id = "");
 
+	// Check whether "name" is the name of an additional disc game directory (e.g. "PS3_GM01"), the counterpart of "PS3_GAME".
+	// NOTE: equivalent to the "^PS3_GM[[:digit:]]{2}$" regular expression, without building one for each checked name
+	inline bool is_ps3_gm_dir_name(std::string_view name)
+	{
+		return name.size() == 8 && name.starts_with("PS3_GM") && name[6] >= '0' && name[6] <= '9' && name[7] >= '0' && name[7] <= '9';
+	}
+
 	std::string get_custom_config_dir();
 	std::string get_custom_config_path(const std::string& identifier);
 

rpcs3/Loader/ISO.cpp

@@ -60,14 +60,15 @@ static void* get_aligned_buf()
 
 static bool is_iso_file(iso_file& file, u64* size = nullptr)
 {
-	if (!file || file.size() < 32768ULL + 6)
+	// The standard identifier ("CD001") follows the type of the first volume descriptor
+	if (!file || file.size() < ISO_DESCRIPTORS_OFFSET + 6)
 	{
 		return false;
 	}
 
 	char magic[5];
 
-	if (file.read_at(32768ULL + 1, magic, 5) != 5)
+	if (file.read_at(ISO_DESCRIPTORS_OFFSET + 1, magic, 5) != 5)
 	{
 		return false;
 	}
@@ -1053,18 +1054,30 @@ iso_archive::iso_archive(const std::string& path)
 	// "m_path" is updated with the raw device path in case "path" points to a BD drive
 	fs::get_optical_raw_device(path, &m_path);
 
-	if (!is_iso_file(m_path))
+	// NOTE: the file is opened once here and then handed over to the parsing below. Recognizing the ISO through its
+	//       path (i.e. "is_iso_file(m_path)") would open it and read its volume descriptor a second time, which is a
+	//       physical read when the path points to an optical drive
+	auto file = std::make_unique<iso_file>(m_path);
+
+	if (!is_iso_file(*file))
 	{
 		iso_log.error("iso_archive: Failed to recognize ISO file: '%s'", path);
 		invalidate();
 		return;
 	}
 
-	fs::file iso_file(std::make_unique<iso_file>(m_path));
+	// NOTE: "is_iso_file()" reads through "read_at()", which does not move the position, so the file is still at its
+	//       beginning here
+	fs::file iso_file(std::move(file));
 
 	u8 descriptor_type = -2;
 	bool use_ucs2_decoding = false;
 
+	// Skip the system area: scanning it sector by sector would read 16 sectors (a physical read each, on an optical
+	// drive) only to find boot data, which could even be mistaken for a volume descriptor.
+	// NOTE: "is_iso_file()" above already verified the standard identifier is right here
+	iso_file.seek(ISO_DESCRIPTORS_OFFSET);
+
 	do
 	{
 		const auto descriptor_start = iso_file.pos();

rpcs3/Loader/ISO.h

@@ -15,6 +15,9 @@ void unload_iso();
 
 constexpr u64 ISO_SECTOR_SIZE = 2048;
 
+// The first 16 sectors are the system area (boot data): the volume descriptors always start right after it (ECMA-119)
+constexpr u64 ISO_DESCRIPTORS_OFFSET = ISO_SECTOR_SIZE * 16;
+
 /*
 - Hijacked the "iso_archive::iso_archive" method to test if the ".iso" file is encrypted and sets a flag.
   The flag is set according to the first matching encryption type found following the order below: