diff --git a/Common/File/AndroidStorage.cpp b/Common/File/AndroidStorage.cpp index 78d457aabf..01b5824a8a 100644 --- a/Common/File/AndroidStorage.cpp +++ b/Common/File/AndroidStorage.cpp @@ -20,6 +20,7 @@ static jmethodID contentUriFileExists; static jmethodID contentUriGetFreeStorageSpace; static jmethodID filePathGetFreeStorageSpace; static jmethodID isExternalStoragePreservedLegacy; +static jmethodID computeRecursiveDirectorySize; static jobject g_nativeActivity; @@ -54,6 +55,8 @@ void Android_RegisterStorageCallbacks(JNIEnv * env, jobject obj) { _dbg_assert_(filePathGetFreeStorageSpace); isExternalStoragePreservedLegacy = env->GetMethodID(env->GetObjectClass(obj), "isExternalStoragePreservedLegacy", "()Z"); _dbg_assert_(isExternalStoragePreservedLegacy); + computeRecursiveDirectorySize = env->GetMethodID(env->GetObjectClass(obj), "computeRecursiveDirectorySize", "(Ljava/lang/String;)J"); + _dbg_assert_(computeRecursiveDirectorySize); } bool Android_IsContentUri(const std::string &filename) { @@ -258,6 +261,22 @@ int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath) { return env->CallLongMethod(g_nativeActivity, filePathGetFreeStorageSpace, param); } +int64_t Android_ComputeRecursiveDirectorySize(const std::string &uri) { + if (!g_nativeActivity) { + return false; + } + auto env = getEnv(); + + jstring param = env->NewStringUTF(uri.c_str()); + + double start = time_now_d(); + int64_t size = env->CallLongMethod(g_nativeActivity, computeRecursiveDirectorySize, param); + double elapsed = time_now_d() - start; + + INFO_LOG(IO, "ComputeRecursiveDirectorySize(%s) in %0.3f s", uri.c_str(), elapsed); + return size; +} + bool Android_IsExternalStoragePreservedLegacy() { if (!g_nativeActivity) { return false; diff --git a/Common/File/AndroidStorage.h b/Common/File/AndroidStorage.h index 2a890d33b4..a9a88055d1 100644 --- a/Common/File/AndroidStorage.h +++ b/Common/File/AndroidStorage.h @@ -49,6 +49,7 @@ StorageError Android_RemoveFile(const std::string &fileUri); StorageError Android_RenameFileTo(const std::string &fileUri, const std::string &newName); bool Android_GetFileInfo(const std::string &fileUri, File::FileInfo *info); bool Android_FileExists(const std::string &fileUri); +int64_t Android_ComputeRecursiveDirectorySize(const std::string &fileUri); int64_t Android_GetFreeSpaceByContentUri(const std::string &uri); int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath); bool Android_IsExternalStoragePreservedLegacy(); @@ -72,6 +73,7 @@ inline StorageError Android_RemoveFile(const std::string &fileUri) { return Stor inline StorageError Android_RenameFileTo(const std::string &fileUri, const std::string &newName) { return StorageError::UNKNOWN; } inline bool Android_GetFileInfo(const std::string &fileUri, File::FileInfo *info) { return false; } inline bool Android_FileExists(const std::string &fileUri) { return false; } +inline int64_t Android_ComputeRecursiveDirectorySize(const std::string &fileUri) { return -1; } inline int64_t Android_GetFreeSpaceByContentUri(const std::string &uri) { return -1; } inline int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath) { return -1; } inline bool Android_IsExternalStoragePreservedLegacy() { return false; } diff --git a/Common/File/DirListing.cpp b/Common/File/DirListing.cpp index f41009f176..b4b3352669 100644 --- a/Common/File/DirListing.cpp +++ b/Common/File/DirListing.cpp @@ -285,22 +285,6 @@ bool GetFilesInDir(const Path &directory, std::vector *files, const ch return true; } -int64_t GetDirectoryRecursiveSize(const Path &path, const char *filter, int flags) { - std::vector fileInfo; - GetFilesInDir(path, &fileInfo, filter, flags); - int64_t sizeSum = 0; - // Note: GetFilesInDir does not fill in fileSize. - for (size_t i = 0; i < fileInfo.size(); i++) { - FileInfo finfo; - GetFileInfo(fileInfo[i].fullName, &finfo); - if (!finfo.isDirectory) - sizeSum += finfo.size; - else - sizeSum += GetDirectoryRecursiveSize(finfo.fullName, filter, flags); - } - return sizeSum; -} - #if PPSSPP_PLATFORM(WINDOWS) // Returns a vector with the device names std::vector GetWindowsDrives() diff --git a/Common/File/DirListing.h b/Common/File/DirListing.h index 99c938630e..c53d420588 100644 --- a/Common/File/DirListing.h +++ b/Common/File/DirListing.h @@ -35,7 +35,6 @@ enum { }; bool GetFilesInDir(const Path &directory, std::vector *files, const char *filter = nullptr, int flags = 0); -int64_t GetDirectoryRecursiveSize(const Path &path, const char *filter = nullptr, int flags = 0); std::vector ApplyFilter(std::vector files, const char *filter); #ifdef _WIN32 diff --git a/Common/File/FileUtil.cpp b/Common/File/FileUtil.cpp index 91c0aee6b6..a9b3498118 100644 --- a/Common/File/FileUtil.cpp +++ b/Common/File/FileUtil.cpp @@ -324,6 +324,32 @@ std::string ResolvePath(const std::string &path) { #endif } +static int64_t RecursiveSize(const Path &path) { + // TODO: Some file systems can optimize this. + std::vector fileInfo; + if (!GetFilesInDir(path, &fileInfo, nullptr, GETFILES_GETHIDDEN)) { + return -1; + } + int64_t sizeSum = 0; + for (const auto &file : fileInfo) { + if (file.isDirectory) { + sizeSum += RecursiveSize(file.fullName); + } else { + sizeSum += file.size; + } + } + return sizeSum; +} + +uint64_t ComputeRecursiveDirectorySize(const Path &path) { + if (path.Type() == PathType::CONTENT_URI) { + return Android_ComputeRecursiveDirectorySize(path.ToString()); + } + + // Generic solution. + return RecursiveSize(path); +} + // Returns true if file filename exists. Will return true on directories. bool ExistsInDir(const Path &path, const std::string &filename) { return Exists(path / filename); diff --git a/Common/File/FileUtil.h b/Common/File/FileUtil.h index ed0581b66c..ae77f859a1 100644 --- a/Common/File/FileUtil.h +++ b/Common/File/FileUtil.h @@ -79,6 +79,9 @@ uint64_t GetFileSize(const Path &filename); // Overloaded GetSize, accepts FILE* uint64_t GetFileSize(FILE *f); +// Computes the recursive size of a directory. Warning: Might be slow! +uint64_t ComputeRecursiveDirectorySize(const Path &path); + // Returns true if successful, or path already exists. bool CreateDir(const Path &filename); diff --git a/Core/FileSystems/BlobFileSystem.h b/Core/FileSystems/BlobFileSystem.h index faab90004a..f45576d6a7 100644 --- a/Core/FileSystems/BlobFileSystem.h +++ b/Core/FileSystems/BlobFileSystem.h @@ -54,6 +54,8 @@ public: bool RemoveFile(const std::string &filename) override; u64 FreeSpace(const std::string &path) override; + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: // File positions. std::map entries_; diff --git a/Core/FileSystems/DirectoryFileSystem.cpp b/Core/FileSystems/DirectoryFileSystem.cpp index abad3b9b9e..1da5499310 100644 --- a/Core/FileSystems/DirectoryFileSystem.cpp +++ b/Core/FileSystems/DirectoryFileSystem.cpp @@ -878,6 +878,18 @@ static std::string SimulateVFATBug(std::string filename) { return filename; } +bool DirectoryFileSystem::ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) { + Path localPath = GetLocalPath(path); + + int64_t sizeTemp = File::ComputeRecursiveDirectorySize(localPath); + if (sizeTemp >= 0) { + *size = sizeTemp; + return true; + } else { + return false; + } +} + std::vector DirectoryFileSystem::GetDirListing(std::string path) { std::vector myVector; diff --git a/Core/FileSystems/DirectoryFileSystem.h b/Core/FileSystems/DirectoryFileSystem.h index 9d8478d691..9ee4777796 100644 --- a/Core/FileSystems/DirectoryFileSystem.h +++ b/Core/FileSystems/DirectoryFileSystem.h @@ -115,6 +115,8 @@ public: FileSystemFlags Flags() override { return flags; } u64 FreeSpace(const std::string &path) override; + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override; + private: struct OpenFileEntry { DirectoryFileHandle hFile; @@ -159,6 +161,8 @@ public: FileSystemFlags Flags() override { return FileSystemFlags::FLASH; } u64 FreeSpace(const std::string &path) override { return 0; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: struct OpenFileEntry { u8 *fileData; diff --git a/Core/FileSystems/FileSystem.h b/Core/FileSystems/FileSystem.h index 5df227dd1f..87c87ae569 100644 --- a/Core/FileSystems/FileSystem.h +++ b/Core/FileSystems/FileSystem.h @@ -136,11 +136,11 @@ public: virtual PSPDevType DevType(u32 handle) = 0; virtual FileSystemFlags Flags() = 0; virtual u64 FreeSpace(const std::string &path) = 0; + virtual bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) = 0; }; -class EmptyFileSystem : public IFileSystem -{ +class EmptyFileSystem : public IFileSystem { public: virtual void DoState(PointerWrap &p) override {} std::vector GetDirListing(std::string path) override {std::vector vec; return vec;} @@ -161,6 +161,7 @@ public: virtual PSPDevType DevType(u32 handle) override { return PSPDevType::INVALID; } virtual FileSystemFlags Flags() override { return FileSystemFlags::NONE; } virtual u64 FreeSpace(const std::string &path) override { return 0; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } }; diff --git a/Core/FileSystems/ISOFileSystem.h b/Core/FileSystems/ISOFileSystem.h index 0b8c00512a..608fa444e0 100644 --- a/Core/FileSystems/ISOFileSystem.h +++ b/Core/FileSystems/ISOFileSystem.h @@ -54,6 +54,8 @@ public: int RenameFile(const std::string &from, const std::string &to) override { return -1; } bool RemoveFile(const std::string &filename) override { return false; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: struct TreeEntry { ~TreeEntry(); @@ -150,6 +152,8 @@ public: int RenameFile(const std::string &from, const std::string &to) override { return -1; } bool RemoveFile(const std::string &filename) override { return false; } + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: std::shared_ptr isoFileSystem_; }; diff --git a/Core/FileSystems/MetaFileSystem.cpp b/Core/FileSystems/MetaFileSystem.cpp index 173cf4a863..c726a87b51 100644 --- a/Core/FileSystems/MetaFileSystem.cpp +++ b/Core/FileSystems/MetaFileSystem.cpp @@ -500,12 +500,9 @@ bool MetaFileSystem::RemoveFile(const std::string &filename) std::string of; IFileSystem *system; int error = MapFilePath(filename, of, &system); - if (error == 0) - { + if (error == 0) { return system->RemoveFile(of); - } - else - { + } else { return false; } } @@ -648,17 +645,36 @@ void MetaFileSystem::DoState(PointerWrap &p) } } -u64 MetaFileSystem::getDirSize(const std::string &dirPath) { +int64_t MetaFileSystem::RecursiveSize(const std::string &dirPath) { u64 result = 0; auto allFiles = GetDirListing(dirPath); for (auto file : allFiles) { if (file.name == "." || file.name == "..") continue; if (file.type == FILETYPE_DIRECTORY) { - result += getDirSize(dirPath + file.name); + result += RecursiveSize(dirPath + file.name); } else { result += file.size; } } return result; } + +int64_t MetaFileSystem::ComputeRecursiveDirectorySize(const std::string &filename) { + std::lock_guard guard(lock); + std::string of; + IFileSystem *system; + int error = MapFilePath(filename, of, &system); + if (error == 0) { + int64_t size; + if (system->ComputeRecursiveDirSizeIfFast(of, &size)) { + // Some file systems can optimize this. + return size; + } else { + // Those that can't, we just run a generic implementation. + return RecursiveSize(filename); + } + } else { + return false; + } +} diff --git a/Core/FileSystems/MetaFileSystem.h b/Core/FileSystems/MetaFileSystem.h index f980fc1900..9a3b041c93 100644 --- a/Core/FileSystems/MetaFileSystem.h +++ b/Core/FileSystems/MetaFileSystem.h @@ -136,5 +136,19 @@ public: startingDirectory = dir; } - u64 getDirSize(const std::string &dirPath); + int64_t ComputeRecursiveDirectorySize(const std::string &dirPath); + + // Shouldn't ever be called, but meh. + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { + int64_t sizeTemp = ComputeRecursiveDirectorySize(path); + if (sizeTemp >= 0) { + *size = sizeTemp; + return true; + } else { + return false; + } + } + +private: + int64_t RecursiveSize(const std::string &dirPath); }; diff --git a/Core/FileSystems/VirtualDiscFileSystem.h b/Core/FileSystems/VirtualDiscFileSystem.h index 6c76d6b6fb..0c1b6db872 100644 --- a/Core/FileSystems/VirtualDiscFileSystem.h +++ b/Core/FileSystems/VirtualDiscFileSystem.h @@ -52,6 +52,8 @@ public: int RenameFile(const std::string &from, const std::string &to) override; bool RemoveFile(const std::string &filename) override; + bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; } + private: void LoadFileListIndex(); // Warning: modifies input string. diff --git a/Core/HLE/sceIo.cpp b/Core/HLE/sceIo.cpp index d290d540b0..37eda68a42 100644 --- a/Core/HLE/sceIo.cpp +++ b/Core/HLE/sceIo.cpp @@ -22,6 +22,7 @@ #include "Common/Thread/ThreadUtil.h" #include "Common/Profiler/Profiler.h" +#include "Common/TimeUtil.h" #include "Common/File/FileUtil.h" #include "Common/Serialize/SerializeFuncs.h" @@ -2321,15 +2322,25 @@ static u32 sceIoDopen(const char *path) { DirListing *dir = new DirListing(); SceUID id = kernelObjects.Create(dir); + double startTime = time_now_d(); + dir->listing = pspFileSystem.GetDirListing(path); dir->index = 0; dir->name = std::string(path); + double listTime = time_now_d() - startTime; + + if (listTime > 0.01) { + INFO_LOG(IO, "Dir listing '%s' took %0.3f", path, listTime); + } + // Blacklist some directories that games should not be able to find out about. // Speeds up directory iteration on slow Android Scoped Storage implementations :( // Might also want to filter out PSP/GAME if not a homebrew, maybe also irrelevant directories // in PSP/SAVEDATA, though iffy to know which ones are irrelevant.. - if (!strcmp(path, "ms0:/PSP")) { + // Also if we're stripping PSP from the path due to setting a directory named PSP as the memstick root, + // these will also show up at ms0: which is not ideal. Should find some other way to deal with that. + if (!strcmp(path, "ms0:/PSP") || !strcmp(path, "ms0:")) { static const char *const pspFolderBlacklist[] = { "CHEATS", "PPSSPP_STATE", @@ -2349,6 +2360,10 @@ static u32 sceIoDopen(const char *path) { break; } } + // Also don't let games see a GAME directory in the root. This confuses Wipeout... + if (!strcasecmp(entry.name.c_str(), "GAME") && !strcmp(path, "ms0:")) { + blacklisted = true; + } if (!blacklisted) { filtered.push_back(entry); diff --git a/Core/HW/MemoryStick.cpp b/Core/HW/MemoryStick.cpp index 2efec86c75..964a002900 100644 --- a/Core/HW/MemoryStick.cpp +++ b/Core/HW/MemoryStick.cpp @@ -96,7 +96,7 @@ static void MemoryStick_CalcInitialFree() { std::unique_lock guard(freeCalcMutex); freeCalcStatus = FreeCalcStatus::RUNNING; freeCalcThread = std::thread([] { - memstickInitialFree = pspFileSystem.FreeSpace("ms0:/") + pspFileSystem.getDirSize("ms0:/PSP/SAVEDATA/"); + memstickInitialFree = pspFileSystem.FreeSpace("ms0:/") + pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/"); std::unique_lock guard(freeCalcMutex); freeCalcStatus = FreeCalcStatus::DONE; @@ -127,7 +127,7 @@ u64 MemoryStick_FreeSpace() { // Assume the memory stick is only used to store savedata. if (!memstickCurrentUseValid) { - memstickCurrentUse = pspFileSystem.getDirSize("ms0:/PSP/SAVEDATA/"); + memstickCurrentUse = pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/"); memstickCurrentUseValid = true; } diff --git a/UI/GameInfoCache.cpp b/UI/GameInfoCache.cpp index 803315aa77..e3bbfb34d0 100644 --- a/UI/GameInfoCache.cpp +++ b/UI/GameInfoCache.cpp @@ -114,7 +114,7 @@ u64 GameInfo::GetGameSizeInBytes() { switch (fileType) { case IdentifiedFileType::PSP_PBP_DIRECTORY: case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY: - return File::GetDirectoryRecursiveSize(ResolvePBPDirectory(filePath_), nullptr, File::GETFILES_GETHIDDEN); + return File::ComputeRecursiveDirectorySize(ResolvePBPDirectory(filePath_)); default: return GetFileLoader()->FileSize(); @@ -343,13 +343,11 @@ public: void Run() override { // An early-return will result in the destructor running, where we can set // flags like working and pending. - if (!info_->LoadFromPath(gamePath_)) { return; } // In case of a remote file, check if it actually exists before locking. - // This is likely not necessary at a if (!info_->GetFileLoader()->Exists()) { return; } diff --git a/UI/SavedataScreen.cpp b/UI/SavedataScreen.cpp index 747ebd5edc..cfd61045a3 100644 --- a/UI/SavedataScreen.cpp +++ b/UI/SavedataScreen.cpp @@ -430,8 +430,7 @@ static time_t GetTotalSize(const SavedataButton *b) { switch (Identify_File(fileLoader.get(), &errorString)) { case IdentifiedFileType::PSP_PBP_DIRECTORY: case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY: - return File::GetDirectoryRecursiveSize(ResolvePBPDirectory(b->GamePath()), nullptr, File::GETFILES_GETHIDDEN); - + return File::ComputeRecursiveDirectorySize(ResolvePBPDirectory(b->GamePath())); default: return fileLoader->FileSize(); } diff --git a/android/src/org/ppsspp/ppsspp/PpssppActivity.java b/android/src/org/ppsspp/ppsspp/PpssppActivity.java index 9f55c799cc..95e684659b 100644 --- a/android/src/org/ppsspp/ppsspp/PpssppActivity.java +++ b/android/src/org/ppsspp/ppsspp/PpssppActivity.java @@ -17,6 +17,8 @@ import android.database.Cursor; import android.provider.DocumentsContract; import android.os.Environment; import androidx.documentfile.provider.DocumentFile; + +import java.lang.reflect.Array; import java.util.ArrayList; import java.util.UUID; import java.io.File; @@ -178,6 +180,70 @@ public class PpssppActivity extends NativeActivity { return str + size + "|" + documentName + "|" + lastModified; } + private long directorySizeRecursion(Uri uri) { + Cursor c = null; + try { + // Log.i(TAG, "recursing into " + uri.toString()); + final String[] columns = new String[]{ + DocumentsContract.Document.COLUMN_DOCUMENT_ID, + DocumentsContract.Document.COLUMN_SIZE, + DocumentsContract.Document.COLUMN_MIME_TYPE, // check for MIME_TYPE_DIR + }; + final ContentResolver resolver = getContentResolver(); + final String documentId = DocumentsContract.getDocumentId(uri); + final Uri childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(uri, documentId); + c = resolver.query(childrenUri, columns, null, null, null); + long sizeSum = 0; + + // Buffer the URIs so we only have one cursor active at once. I don't trust the storage framework + // to handle more than one... + ArrayList childDirs = new ArrayList(); + + while (c.moveToNext()) { + final String mimeType = c.getString(2); + final boolean isDirectory = mimeType.equals(DocumentsContract.Document.MIME_TYPE_DIR); + if (isDirectory) { + final String childDocumentId = c.getString(0); + final Uri childUri = DocumentsContract.buildDocumentUriUsingTree(uri, childDocumentId); + childDirs.add(childUri); + } else { + final long fileSize = c.getLong(1); + sizeSum += fileSize; + } + } + c.close(); + c = null; + + for (Uri childUri : childDirs) { + long dirSize = directorySizeRecursion(childUri); + if (dirSize >= 0) { + sizeSum += dirSize; + } else { + return dirSize; + } + } + return sizeSum; + } catch (Exception e) { + return -1; + } finally { + if (c != null) { + c.close(); + } + } + } + + public long computeRecursiveDirectorySize(String uriString) { + try { + Uri uri = Uri.parse(uriString); + long totalSize = directorySizeRecursion(uri); + return totalSize; + } + catch (Exception e) { + Log.e(TAG, "computeRecursiveSize exception: " + e.toString()); + return -1; + } + } + // TODO: Maybe add a cheaper version that doesn't extract all the file information? // TODO: Replace with a proper query: // * https://stackoverflow.com/questions/42186820/documentfile-is-very-slow