From cd79a1d92f1bea65ce7c66781069a35140f19524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 11 Sep 2021 15:44:40 +0200 Subject: [PATCH 1/7] Fix another GetFileInfo loop (GetFilesInDir now does return sizes) --- Common/File/DirListing.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Common/File/DirListing.cpp b/Common/File/DirListing.cpp index f41009f176..0d1d07ed7d 100644 --- a/Common/File/DirListing.cpp +++ b/Common/File/DirListing.cpp @@ -289,10 +289,7 @@ int64_t GetDirectoryRecursiveSize(const Path &path, const char *filter, int flag std::vector fileInfo; GetFilesInDir(path, &fileInfo, filter, flags); int64_t sizeSum = 0; - // Note: GetFilesInDir does not fill in fileSize. - for (size_t i = 0; i < fileInfo.size(); i++) { - FileInfo finfo; - GetFileInfo(fileInfo[i].fullName, &finfo); + for (const auto &finfo : fileInfo) { if (!finfo.isDirectory) sizeSum += finfo.size; else From e842d395fa156862915dfb47fddf879cb6ea3dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 11 Sep 2021 16:36:35 +0200 Subject: [PATCH 2/7] Add untested function Android_ComputeRecursiveDirectorySize --- Common/File/AndroidStorage.cpp | 13 +++++ Common/File/AndroidStorage.h | 2 + UI/GameInfoCache.cpp | 2 - .../src/org/ppsspp/ppsspp/PpssppActivity.java | 54 +++++++++++++++++++ 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/Common/File/AndroidStorage.cpp b/Common/File/AndroidStorage.cpp index 78d457aabf..c98a649385 100644 --- a/Common/File/AndroidStorage.cpp +++ b/Common/File/AndroidStorage.cpp @@ -20,6 +20,7 @@ static jmethodID contentUriFileExists; static jmethodID contentUriGetFreeStorageSpace; static jmethodID filePathGetFreeStorageSpace; static jmethodID isExternalStoragePreservedLegacy; +static jmethodID computeRecursiveDirectorySize; static jobject g_nativeActivity; @@ -54,6 +55,8 @@ void Android_RegisterStorageCallbacks(JNIEnv * env, jobject obj) { _dbg_assert_(filePathGetFreeStorageSpace); isExternalStoragePreservedLegacy = env->GetMethodID(env->GetObjectClass(obj), "isExternalStoragePreservedLegacy", "()Z"); _dbg_assert_(isExternalStoragePreservedLegacy); + computeRecursiveDirectorySize = env->GetMethodID(env->GetObjectClass(obj), "computeRecursiveDirectorySize", "(Ljava/lang/String;)J"); + _dbg_assert_(computeRecursiveDirectorySize); } bool Android_IsContentUri(const std::string &filename) { @@ -258,6 +261,16 @@ int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath) { return env->CallLongMethod(g_nativeActivity, filePathGetFreeStorageSpace, param); } +int64_t Android_ComputeRecursiveDirectorySize(const std::string &uri) { + if (!g_nativeActivity) { + return false; + } + auto env = getEnv(); + + jstring param = env->NewStringUTF(uri.c_str()); + return env->CallLongMethod(g_nativeActivity, computeRecursiveDirectorySize, param); +} + bool Android_IsExternalStoragePreservedLegacy() { if (!g_nativeActivity) { return false; diff --git a/Common/File/AndroidStorage.h b/Common/File/AndroidStorage.h index 2a890d33b4..40ab19f1b0 100644 --- a/Common/File/AndroidStorage.h +++ b/Common/File/AndroidStorage.h @@ -49,6 +49,7 @@ StorageError Android_RemoveFile(const std::string &fileUri); StorageError Android_RenameFileTo(const std::string &fileUri, const std::string &newName); bool Android_GetFileInfo(const std::string &fileUri, File::FileInfo *info); bool Android_FileExists(const std::string &fileUri); +int64_t Android_ComputeRecursiveDirectorySize(const std::string &fileUri); int64_t Android_GetFreeSpaceByContentUri(const std::string &uri); int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath); bool Android_IsExternalStoragePreservedLegacy(); @@ -72,6 +73,7 @@ inline StorageError Android_RemoveFile(const std::string &fileUri) { return Stor inline StorageError Android_RenameFileTo(const std::string &fileUri, const std::string &newName) { return StorageError::UNKNOWN; } inline bool Android_GetFileInfo(const std::string &fileUri, File::FileInfo *info) { return false; } inline bool Android_FileExists(const std::string &fileUri) { return false; } +inline int64_t Android_GetRecursiveDirectorySize(const std::string &fileUri) { return -1; } inline int64_t Android_GetFreeSpaceByContentUri(const std::string &uri) { return -1; } inline int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath) { return -1; } inline bool Android_IsExternalStoragePreservedLegacy() { return false; } diff --git a/UI/GameInfoCache.cpp b/UI/GameInfoCache.cpp index 803315aa77..25c426f5dd 100644 --- a/UI/GameInfoCache.cpp +++ b/UI/GameInfoCache.cpp @@ -343,13 +343,11 @@ public: void Run() override { // An early-return will result in the destructor running, where we can set // flags like working and pending. - if (!info_->LoadFromPath(gamePath_)) { return; } // In case of a remote file, check if it actually exists before locking. - // This is likely not necessary at a if (!info_->GetFileLoader()->Exists()) { return; } diff --git a/android/src/org/ppsspp/ppsspp/PpssppActivity.java b/android/src/org/ppsspp/ppsspp/PpssppActivity.java index 9f55c799cc..67b74014b0 100644 --- a/android/src/org/ppsspp/ppsspp/PpssppActivity.java +++ b/android/src/org/ppsspp/ppsspp/PpssppActivity.java @@ -178,6 +178,60 @@ public class PpssppActivity extends NativeActivity { return str + size + "|" + documentName + "|" + lastModified; } + private long directorySizeRecursion(Uri uri, String documentId) { + Cursor c = null; + try { + Log.i(TAG, "recursing into " + uri.toString()); + final String[] columns = new String[]{ + DocumentsContract.Document.COLUMN_DOCUMENT_ID, + DocumentsContract.Document.COLUMN_SIZE, + DocumentsContract.Document.COLUMN_MIME_TYPE, // check for MIME_TYPE_DIR + }; + final ContentResolver resolver = getContentResolver(); + final Uri childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(uri, documentId); + c = resolver.query(childrenUri, columns, null, null, null); + long sizeSum = 0; + while (c.moveToNext()) { + final String mimeType = c.getString(2); + final boolean isDirectory = mimeType.equals(DocumentsContract.Document.MIME_TYPE_DIR); + if (isDirectory) { + final String childDocumentId = c.getString(0); + long dirSize = directorySizeRecursion( + DocumentsContract.buildDocumentUriUsingTree(uri, childDocumentId), + documentId + ); + if (dirSize >= 0) { + sizeSum += dirSize; + } + } else { + final long fileSize = c.getLong(1); + sizeSum += fileSize; + } + } + return sizeSum; + } catch (Exception e) { + return -1; + } finally { + if (c != null) { + c.close(); + } + } + } + + public long computeRecursiveDirectorySize(String uriString) { + try { + Uri uri = Uri.parse(uriString); + String documentId = DocumentsContract.getDocumentId(uri); + long totalSize = directorySizeRecursion(uri, documentId); + Log.i(TAG, "directorySizeRecursion(" + uriString + ") returned " + totalSize); + return totalSize; + } + catch (Exception e) { + Log.e(TAG, "computeRecursiveSize exception: " + e.toString()); + return -1; + } + } + // TODO: Maybe add a cheaper version that doesn't extract all the file information? // TODO: Replace with a proper query: // * https://stackoverflow.com/questions/42186820/documentfile-is-very-slow From 48310d15a9ea69d38173f951d5066744a330c118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 11 Sep 2021 17:18:39 +0200 Subject: [PATCH 3/7] Add metafilesystem hook to add optimized implementations of compute recursive directory size --- Common/File/AndroidStorage.h | 2 +- Common/File/FileUtil.cpp | 28 ++++++++++++++++++++++ Common/File/FileUtil.h | 3 +++ Core/FileSystems/BlobFileSystem.h | 2 ++ Core/FileSystems/DirectoryFileSystem.h | 5 ++++ Core/FileSystems/FileSystem.h | 5 ++-- Core/FileSystems/ISOFileSystem.h | 4 ++++ Core/FileSystems/MetaFileSystem.cpp | 30 ++++++++++++++++++------ Core/FileSystems/MetaFileSystem.h | 16 ++++++++++++- Core/FileSystems/VirtualDiscFileSystem.h | 2 ++ Core/HW/MemoryStick.cpp | 4 ++-- 11 files changed, 88 insertions(+), 13 deletions(-) diff --git a/Common/File/AndroidStorage.h b/Common/File/AndroidStorage.h index 40ab19f1b0..a9a88055d1 100644 --- a/Common/File/AndroidStorage.h +++ b/Common/File/AndroidStorage.h @@ -73,7 +73,7 @@ inline StorageError Android_RemoveFile(const std::string &fileUri) { return Stor inline StorageError Android_RenameFileTo(const std::string &fileUri, const std::string &newName) { return StorageError::UNKNOWN; } inline bool Android_GetFileInfo(const std::string &fileUri, File::FileInfo *info) { return false; } inline bool Android_FileExists(const std::string &fileUri) { return false; } -inline int64_t Android_GetRecursiveDirectorySize(const std::string &fileUri) { return -1; } +inline int64_t Android_ComputeRecursiveDirectorySize(const std::string &fileUri) { return -1; } inline int64_t Android_GetFreeSpaceByContentUri(const std::string &uri) { return -1; } inline int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath) { return -1; } inline bool Android_IsExternalStoragePreservedLegacy() { return false; } diff --git a/Common/File/FileUtil.cpp b/Common/File/FileUtil.cpp index 91c0aee6b6..2de9d89598 100644 --- a/Common/File/FileUtil.cpp +++ b/Common/File/FileUtil.cpp @@ -324,6 +324,34 @@ std::string ResolvePath(const std::string &path) { #endif } +static int64_t RecursiveSize(const Path &path) { + // TODO: Some file systems can optimize this. + std::vector fileInfo; + if (!GetFilesInDir(path, &fileInfo)) { + return -1; + } + int64_t result = 0; + for (const auto &file : fileInfo) { + if (file.name == "." || file.name == "..") + continue; + if (file.isDirectory) { + result += RecursiveSize(file.fullName); + } else { + result += file.size; + } + } + return result; +} + +uint64_t ComputeRecursiveDirectorySize(const Path &path) { + if (path.Type() == PathType::CONTENT_URI) { + return Android_ComputeRecursiveDirectorySize(path.ToString()); + } + + // Generic solution. + return RecursiveSize(path); +} + // Returns true if file filename exists. Will return true on directories. bool ExistsInDir(const Path &path, const std::string &filename) { return Exists(path / filename); diff --git a/Common/File/FileUtil.h b/Common/File/FileUtil.h index ed0581b66c..ae77f859a1 100644 --- a/Common/File/FileUtil.h +++ b/Common/File/FileUtil.h @@ -79,6 +79,9 @@ uint64_t GetFileSize(const Path &filename); // Overloaded GetSize, accepts FILE* uint64_t GetFileSize(FILE *f); +// Computes the recursive size of a directory. Warning: Might be slow! +uint64_t ComputeRecursiveDirectorySize(const Path &path); + // Returns true if successful, or path already exists. bool CreateDir(const Path &filename); diff --git a/Core/FileSystems/BlobFileSystem.h b/Core/FileSystems/BlobFileSystem.h index faab90004a..f45576d6a7 100644 --- a/Core/FileSystems/BlobFileSystem.h +++ b/Core/FileSystems/BlobFileSystem.h @@ -54,6 +54,8 @@ public: bool RemoveFile(const std::string &filename) override; u64 FreeSpace(const std::string &path) override; + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: // File positions. std::map entries_; diff --git a/Core/FileSystems/DirectoryFileSystem.h b/Core/FileSystems/DirectoryFileSystem.h index 9d8478d691..daf900a16e 100644 --- a/Core/FileSystems/DirectoryFileSystem.h +++ b/Core/FileSystems/DirectoryFileSystem.h @@ -115,6 +115,9 @@ public: FileSystemFlags Flags() override { return flags; } u64 FreeSpace(const std::string &path) override; + // TODO: Replace with optimized implementation. + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: struct OpenFileEntry { DirectoryFileHandle hFile; @@ -159,6 +162,8 @@ public: FileSystemFlags Flags() override { return FileSystemFlags::FLASH; } u64 FreeSpace(const std::string &path) override { return 0; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: struct OpenFileEntry { u8 *fileData; diff --git a/Core/FileSystems/FileSystem.h b/Core/FileSystems/FileSystem.h index 5df227dd1f..87c87ae569 100644 --- a/Core/FileSystems/FileSystem.h +++ b/Core/FileSystems/FileSystem.h @@ -136,11 +136,11 @@ public: virtual PSPDevType DevType(u32 handle) = 0; virtual FileSystemFlags Flags() = 0; virtual u64 FreeSpace(const std::string &path) = 0; + virtual bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) = 0; }; -class EmptyFileSystem : public IFileSystem -{ +class EmptyFileSystem : public IFileSystem { public: virtual void DoState(PointerWrap &p) override {} std::vector GetDirListing(std::string path) override {std::vector vec; return vec;} @@ -161,6 +161,7 @@ public: virtual PSPDevType DevType(u32 handle) override { return PSPDevType::INVALID; } virtual FileSystemFlags Flags() override { return FileSystemFlags::NONE; } virtual u64 FreeSpace(const std::string &path) override { return 0; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } }; diff --git a/Core/FileSystems/ISOFileSystem.h b/Core/FileSystems/ISOFileSystem.h index 0b8c00512a..608fa444e0 100644 --- a/Core/FileSystems/ISOFileSystem.h +++ b/Core/FileSystems/ISOFileSystem.h @@ -54,6 +54,8 @@ public: int RenameFile(const std::string &from, const std::string &to) override { return -1; } bool RemoveFile(const std::string &filename) override { return false; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: struct TreeEntry { ~TreeEntry(); @@ -150,6 +152,8 @@ public: int RenameFile(const std::string &from, const std::string &to) override { return -1; } bool RemoveFile(const std::string &filename) override { return false; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: std::shared_ptr isoFileSystem_; }; diff --git a/Core/FileSystems/MetaFileSystem.cpp b/Core/FileSystems/MetaFileSystem.cpp index 173cf4a863..c726a87b51 100644 --- a/Core/FileSystems/MetaFileSystem.cpp +++ b/Core/FileSystems/MetaFileSystem.cpp @@ -500,12 +500,9 @@ bool MetaFileSystem::RemoveFile(const std::string &filename) std::string of; IFileSystem *system; int error = MapFilePath(filename, of, &system); - if (error == 0) - { + if (error == 0) { return system->RemoveFile(of); - } - else - { + } else { return false; } } @@ -648,17 +645,36 @@ void MetaFileSystem::DoState(PointerWrap &p) } } -u64 MetaFileSystem::getDirSize(const std::string &dirPath) { +int64_t MetaFileSystem::RecursiveSize(const std::string &dirPath) { u64 result = 0; auto allFiles = GetDirListing(dirPath); for (auto file : allFiles) { if (file.name == "." || file.name == "..") continue; if (file.type == FILETYPE_DIRECTORY) { - result += getDirSize(dirPath + file.name); + result += RecursiveSize(dirPath + file.name); } else { result += file.size; } } return result; } + +int64_t MetaFileSystem::ComputeRecursiveDirectorySize(const std::string &filename) { + std::lock_guard guard(lock); + std::string of; + IFileSystem *system; + int error = MapFilePath(filename, of, &system); + if (error == 0) { + int64_t size; + if (system->ComputeRecursiveDirSizeIfFast(of, &size)) { + // Some file systems can optimize this. + return size; + } else { + // Those that can't, we just run a generic implementation. + return RecursiveSize(filename); + } + } else { + return false; + } +} diff --git a/Core/FileSystems/MetaFileSystem.h b/Core/FileSystems/MetaFileSystem.h index f980fc1900..9a3b041c93 100644 --- a/Core/FileSystems/MetaFileSystem.h +++ b/Core/FileSystems/MetaFileSystem.h @@ -136,5 +136,19 @@ public: startingDirectory = dir; } - u64 getDirSize(const std::string &dirPath); + int64_t ComputeRecursiveDirectorySize(const std::string &dirPath); + + // Shouldn't ever be called, but meh. + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { + int64_t sizeTemp = ComputeRecursiveDirectorySize(path); + if (sizeTemp >= 0) { + *size = sizeTemp; + return true; + } else { + return false; + } + } + +private: + int64_t RecursiveSize(const std::string &dirPath); }; diff --git a/Core/FileSystems/VirtualDiscFileSystem.h b/Core/FileSystems/VirtualDiscFileSystem.h index 6c76d6b6fb..0c1b6db872 100644 --- a/Core/FileSystems/VirtualDiscFileSystem.h +++ b/Core/FileSystems/VirtualDiscFileSystem.h @@ -52,6 +52,8 @@ public: int RenameFile(const std::string &from, const std::string &to) override; bool RemoveFile(const std::string &filename) override; + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: void LoadFileListIndex(); // Warning: modifies input string. diff --git a/Core/HW/MemoryStick.cpp b/Core/HW/MemoryStick.cpp index 2efec86c75..964a002900 100644 --- a/Core/HW/MemoryStick.cpp +++ b/Core/HW/MemoryStick.cpp @@ -96,7 +96,7 @@ static void MemoryStick_CalcInitialFree() { std::unique_lock guard(freeCalcMutex); freeCalcStatus = FreeCalcStatus::RUNNING; freeCalcThread = std::thread([] { - memstickInitialFree = pspFileSystem.FreeSpace("ms0:/") + pspFileSystem.getDirSize("ms0:/PSP/SAVEDATA/"); + memstickInitialFree = pspFileSystem.FreeSpace("ms0:/") + pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/"); std::unique_lock guard(freeCalcMutex); freeCalcStatus = FreeCalcStatus::DONE; @@ -127,7 +127,7 @@ u64 MemoryStick_FreeSpace() { // Assume the memory stick is only used to store savedata. if (!memstickCurrentUseValid) { - memstickCurrentUse = pspFileSystem.getDirSize("ms0:/PSP/SAVEDATA/"); + memstickCurrentUse = pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/"); memstickCurrentUseValid = true; } From fcc0337576e5cf9eadcb15f433d93418d4c72254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 11 Sep 2021 17:50:07 +0200 Subject: [PATCH 4/7] Fix the Android recursive size computation and hook it up. --- Common/File/DirListing.cpp | 13 ------- Common/File/DirListing.h | 1 - Common/File/FileUtil.cpp | 12 +++---- Core/FileSystems/DirectoryFileSystem.cpp | 12 +++++++ Core/FileSystems/DirectoryFileSystem.h | 2 +- UI/GameInfoCache.cpp | 2 +- UI/SavedataScreen.cpp | 3 +- .../src/org/ppsspp/ppsspp/PpssppActivity.java | 35 +++++++++++++------ 8 files changed, 44 insertions(+), 36 deletions(-) diff --git a/Common/File/DirListing.cpp b/Common/File/DirListing.cpp index 0d1d07ed7d..b4b3352669 100644 --- a/Common/File/DirListing.cpp +++ b/Common/File/DirListing.cpp @@ -285,19 +285,6 @@ bool GetFilesInDir(const Path &directory, std::vector *files, const ch return true; } -int64_t GetDirectoryRecursiveSize(const Path &path, const char *filter, int flags) { - std::vector fileInfo; - GetFilesInDir(path, &fileInfo, filter, flags); - int64_t sizeSum = 0; - for (const auto &finfo : fileInfo) { - if (!finfo.isDirectory) - sizeSum += finfo.size; - else - sizeSum += GetDirectoryRecursiveSize(finfo.fullName, filter, flags); - } - return sizeSum; -} - #if PPSSPP_PLATFORM(WINDOWS) // Returns a vector with the device names std::vector GetWindowsDrives() diff --git a/Common/File/DirListing.h b/Common/File/DirListing.h index 99c938630e..c53d420588 100644 --- a/Common/File/DirListing.h +++ b/Common/File/DirListing.h @@ -35,7 +35,6 @@ enum { }; bool GetFilesInDir(const Path &directory, std::vector *files, const char *filter = nullptr, int flags = 0); -int64_t GetDirectoryRecursiveSize(const Path &path, const char *filter = nullptr, int flags = 0); std::vector ApplyFilter(std::vector files, const char *filter); #ifdef _WIN32 diff --git a/Common/File/FileUtil.cpp b/Common/File/FileUtil.cpp index 2de9d89598..a9b3498118 100644 --- a/Common/File/FileUtil.cpp +++ b/Common/File/FileUtil.cpp @@ -327,20 +327,18 @@ std::string ResolvePath(const std::string &path) { static int64_t RecursiveSize(const Path &path) { // TODO: Some file systems can optimize this. std::vector fileInfo; - if (!GetFilesInDir(path, &fileInfo)) { + if (!GetFilesInDir(path, &fileInfo, nullptr, GETFILES_GETHIDDEN)) { return -1; } - int64_t result = 0; + int64_t sizeSum = 0; for (const auto &file : fileInfo) { - if (file.name == "." || file.name == "..") - continue; if (file.isDirectory) { - result += RecursiveSize(file.fullName); + sizeSum += RecursiveSize(file.fullName); } else { - result += file.size; + sizeSum += file.size; } } - return result; + return sizeSum; } uint64_t ComputeRecursiveDirectorySize(const Path &path) { diff --git a/Core/FileSystems/DirectoryFileSystem.cpp b/Core/FileSystems/DirectoryFileSystem.cpp index abad3b9b9e..1da5499310 100644 --- a/Core/FileSystems/DirectoryFileSystem.cpp +++ b/Core/FileSystems/DirectoryFileSystem.cpp @@ -878,6 +878,18 @@ static std::string SimulateVFATBug(std::string filename) { return filename; } +bool DirectoryFileSystem::ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) { + Path localPath = GetLocalPath(path); + + int64_t sizeTemp = File::ComputeRecursiveDirectorySize(localPath); + if (sizeTemp >= 0) { + *size = sizeTemp; + return true; + } else { + return false; + } +} + std::vector DirectoryFileSystem::GetDirListing(std::string path) { std::vector myVector; diff --git a/Core/FileSystems/DirectoryFileSystem.h b/Core/FileSystems/DirectoryFileSystem.h index daf900a16e..fa2b24d013 100644 --- a/Core/FileSystems/DirectoryFileSystem.h +++ b/Core/FileSystems/DirectoryFileSystem.h @@ -116,7 +116,7 @@ public: u64 FreeSpace(const std::string &path) override; // TODO: Replace with optimized implementation. - bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override; private: struct OpenFileEntry { diff --git a/UI/GameInfoCache.cpp b/UI/GameInfoCache.cpp index 25c426f5dd..e3bbfb34d0 100644 --- a/UI/GameInfoCache.cpp +++ b/UI/GameInfoCache.cpp @@ -114,7 +114,7 @@ u64 GameInfo::GetGameSizeInBytes() { switch (fileType) { case IdentifiedFileType::PSP_PBP_DIRECTORY: case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY: - return File::GetDirectoryRecursiveSize(ResolvePBPDirectory(filePath_), nullptr, File::GETFILES_GETHIDDEN); + return File::ComputeRecursiveDirectorySize(ResolvePBPDirectory(filePath_)); default: return GetFileLoader()->FileSize(); diff --git a/UI/SavedataScreen.cpp b/UI/SavedataScreen.cpp index 747ebd5edc..cfd61045a3 100644 --- a/UI/SavedataScreen.cpp +++ b/UI/SavedataScreen.cpp @@ -430,8 +430,7 @@ static time_t GetTotalSize(const SavedataButton *b) { switch (Identify_File(fileLoader.get(), &errorString)) { case IdentifiedFileType::PSP_PBP_DIRECTORY: case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY: - return File::GetDirectoryRecursiveSize(ResolvePBPDirectory(b->GamePath()), nullptr, File::GETFILES_GETHIDDEN); - + return File::ComputeRecursiveDirectorySize(ResolvePBPDirectory(b->GamePath())); default: return fileLoader->FileSize(); } diff --git a/android/src/org/ppsspp/ppsspp/PpssppActivity.java b/android/src/org/ppsspp/ppsspp/PpssppActivity.java index 67b74014b0..4a37e21e4f 100644 --- a/android/src/org/ppsspp/ppsspp/PpssppActivity.java +++ b/android/src/org/ppsspp/ppsspp/PpssppActivity.java @@ -17,6 +17,8 @@ import android.database.Cursor; import android.provider.DocumentsContract; import android.os.Environment; import androidx.documentfile.provider.DocumentFile; + +import java.lang.reflect.Array; import java.util.ArrayList; import java.util.UUID; import java.io.File; @@ -178,36 +180,48 @@ public class PpssppActivity extends NativeActivity { return str + size + "|" + documentName + "|" + lastModified; } - private long directorySizeRecursion(Uri uri, String documentId) { + private long directorySizeRecursion(Uri uri) { Cursor c = null; try { - Log.i(TAG, "recursing into " + uri.toString()); + // Log.i(TAG, "recursing into " + uri.toString()); final String[] columns = new String[]{ DocumentsContract.Document.COLUMN_DOCUMENT_ID, DocumentsContract.Document.COLUMN_SIZE, DocumentsContract.Document.COLUMN_MIME_TYPE, // check for MIME_TYPE_DIR }; final ContentResolver resolver = getContentResolver(); + final String documentId = DocumentsContract.getDocumentId(uri); final Uri childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(uri, documentId); c = resolver.query(childrenUri, columns, null, null, null); long sizeSum = 0; + + // Buffer the URIs so we only have one cursor active at once. I don't trust the storage framework + // to handle more than one... + ArrayList childDirs = new ArrayList(); + while (c.moveToNext()) { final String mimeType = c.getString(2); final boolean isDirectory = mimeType.equals(DocumentsContract.Document.MIME_TYPE_DIR); if (isDirectory) { final String childDocumentId = c.getString(0); - long dirSize = directorySizeRecursion( - DocumentsContract.buildDocumentUriUsingTree(uri, childDocumentId), - documentId - ); - if (dirSize >= 0) { - sizeSum += dirSize; - } + final Uri childUri = DocumentsContract.buildDocumentUriUsingTree(uri, childDocumentId); + childDirs.add(childUri); } else { final long fileSize = c.getLong(1); sizeSum += fileSize; } } + c.close(); + c = null; + + for (Uri childUri : childDirs) { + long dirSize = directorySizeRecursion(childUri); + if (dirSize >= 0) { + sizeSum += dirSize; + } else { + return dirSize; + } + } return sizeSum; } catch (Exception e) { return -1; @@ -221,8 +235,7 @@ public class PpssppActivity extends NativeActivity { public long computeRecursiveDirectorySize(String uriString) { try { Uri uri = Uri.parse(uriString); - String documentId = DocumentsContract.getDocumentId(uri); - long totalSize = directorySizeRecursion(uri, documentId); + long totalSize = directorySizeRecursion(uri); Log.i(TAG, "directorySizeRecursion(" + uriString + ") returned " + totalSize); return totalSize; } From f7895457ee443521d11b5e4440c85cbe999671a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 11 Sep 2021 18:06:26 +0200 Subject: [PATCH 5/7] Add back the dir listing logging. Add hack to avoid confusing Wipeout :( --- Core/HLE/sceIo.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Core/HLE/sceIo.cpp b/Core/HLE/sceIo.cpp index d290d540b0..5ed0ad5c67 100644 --- a/Core/HLE/sceIo.cpp +++ b/Core/HLE/sceIo.cpp @@ -22,6 +22,7 @@ #include "Common/Thread/ThreadUtil.h" #include "Common/Profiler/Profiler.h" +#include "Common/TimeUtil.h" #include "Common/File/FileUtil.h" #include "Common/Serialize/SerializeFuncs.h" @@ -2321,15 +2322,25 @@ static u32 sceIoDopen(const char *path) { DirListing *dir = new DirListing(); SceUID id = kernelObjects.Create(dir); + double startTime = time_now_d(); + dir->listing = pspFileSystem.GetDirListing(path); dir->index = 0; dir->name = std::string(path); + double listTime = time_now_d() - startTime; + + if (listTime > 0.01) { + ERROR_LOG(IO, "Dir listing '%s' took %0.3f", path, listTime); + } + // Blacklist some directories that games should not be able to find out about. // Speeds up directory iteration on slow Android Scoped Storage implementations :( // Might also want to filter out PSP/GAME if not a homebrew, maybe also irrelevant directories // in PSP/SAVEDATA, though iffy to know which ones are irrelevant.. - if (!strcmp(path, "ms0:/PSP")) { + // Also if we're stripping PSP from the path due to setting a directory named PSP as the memstick root, + // these will also show up at ms0: which is not ideal. Should find some other way to deal with that. + if (!strcmp(path, "ms0:/PSP") || !strcmp(path, "ms0:")) { static const char *const pspFolderBlacklist[] = { "CHEATS", "PPSSPP_STATE", @@ -2349,6 +2360,10 @@ static u32 sceIoDopen(const char *path) { break; } } + // Also don't let games see a GAME directory in the root. This confuses Wipeout... + if (!strcasecmp(entry.name.c_str(), "GAME") && !strcmp(path, "ms0:")) { + blacklisted = true; + } if (!blacklisted) { filtered.push_back(entry); From dc520a59d338d9da3df6add2956674214f8698a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 11 Sep 2021 18:17:14 +0200 Subject: [PATCH 6/7] Log the duration of directory size computations too --- Common/File/AndroidStorage.cpp | 8 +++++++- android/src/org/ppsspp/ppsspp/PpssppActivity.java | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Common/File/AndroidStorage.cpp b/Common/File/AndroidStorage.cpp index c98a649385..5e12921904 100644 --- a/Common/File/AndroidStorage.cpp +++ b/Common/File/AndroidStorage.cpp @@ -268,7 +268,13 @@ int64_t Android_ComputeRecursiveDirectorySize(const std::string &uri) { auto env = getEnv(); jstring param = env->NewStringUTF(uri.c_str()); - return env->CallLongMethod(g_nativeActivity, computeRecursiveDirectorySize, param); + + double start = time_now_d(); + int64_t size = env->CallLongMethod(g_nativeActivity, computeRecursiveDirectorySize, param); + double elapsed = time_now_d() - start; + + ERROR_LOG(IO, "ComputeRecursiveDirectorySize(%s) in %0.3f s", uri.c_str(), elapsed); + return size; } bool Android_IsExternalStoragePreservedLegacy() { diff --git a/android/src/org/ppsspp/ppsspp/PpssppActivity.java b/android/src/org/ppsspp/ppsspp/PpssppActivity.java index 4a37e21e4f..95e684659b 100644 --- a/android/src/org/ppsspp/ppsspp/PpssppActivity.java +++ b/android/src/org/ppsspp/ppsspp/PpssppActivity.java @@ -236,7 +236,6 @@ public class PpssppActivity extends NativeActivity { try { Uri uri = Uri.parse(uriString); long totalSize = directorySizeRecursion(uri); - Log.i(TAG, "directorySizeRecursion(" + uriString + ") returned " + totalSize); return totalSize; } catch (Exception e) { From 07413aa4e384ef61ecd8753269f4e39d440dacac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 11 Sep 2021 18:53:46 +0200 Subject: [PATCH 7/7] Feedback --- Common/File/AndroidStorage.cpp | 2 +- Core/FileSystems/DirectoryFileSystem.h | 1 - Core/HLE/sceIo.cpp | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Common/File/AndroidStorage.cpp b/Common/File/AndroidStorage.cpp index 5e12921904..01b5824a8a 100644 --- a/Common/File/AndroidStorage.cpp +++ b/Common/File/AndroidStorage.cpp @@ -273,7 +273,7 @@ int64_t Android_ComputeRecursiveDirectorySize(const std::string &uri) { int64_t size = env->CallLongMethod(g_nativeActivity, computeRecursiveDirectorySize, param); double elapsed = time_now_d() - start; - ERROR_LOG(IO, "ComputeRecursiveDirectorySize(%s) in %0.3f s", uri.c_str(), elapsed); + INFO_LOG(IO, "ComputeRecursiveDirectorySize(%s) in %0.3f s", uri.c_str(), elapsed); return size; } diff --git a/Core/FileSystems/DirectoryFileSystem.h b/Core/FileSystems/DirectoryFileSystem.h index fa2b24d013..9ee4777796 100644 --- a/Core/FileSystems/DirectoryFileSystem.h +++ b/Core/FileSystems/DirectoryFileSystem.h @@ -115,7 +115,6 @@ public: FileSystemFlags Flags() override { return flags; } u64 FreeSpace(const std::string &path) override; - // TODO: Replace with optimized implementation. bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override; private: diff --git a/Core/HLE/sceIo.cpp b/Core/HLE/sceIo.cpp index 5ed0ad5c67..37eda68a42 100644 --- a/Core/HLE/sceIo.cpp +++ b/Core/HLE/sceIo.cpp @@ -2331,7 +2331,7 @@ static u32 sceIoDopen(const char *path) { double listTime = time_now_d() - startTime; if (listTime > 0.01) { - ERROR_LOG(IO, "Dir listing '%s' took %0.3f", path, listTime); + INFO_LOG(IO, "Dir listing '%s' took %0.3f", path, listTime); } // Blacklist some directories that games should not be able to find out about.