Merge pull request #14826 from hrydgard/more-io-optimizations

More I/O optimizations

Common/File/AndroidStorage.cpp

@@ -20,6 +20,7 @@ static jmethodID contentUriFileExists;
 static jmethodID contentUriGetFreeStorageSpace;
 static jmethodID filePathGetFreeStorageSpace;
 static jmethodID isExternalStoragePreservedLegacy;
+static jmethodID computeRecursiveDirectorySize;
 
 static jobject g_nativeActivity;
 
@@ -54,6 +55,8 @@ void Android_RegisterStorageCallbacks(JNIEnv * env, jobject obj) {
 	_dbg_assert_(filePathGetFreeStorageSpace);
 	isExternalStoragePreservedLegacy = env->GetMethodID(env->GetObjectClass(obj), "isExternalStoragePreservedLegacy", "()Z");
 	_dbg_assert_(isExternalStoragePreservedLegacy);
+	computeRecursiveDirectorySize = env->GetMethodID(env->GetObjectClass(obj), "computeRecursiveDirectorySize", "(Ljava/lang/String;)J");
+	_dbg_assert_(computeRecursiveDirectorySize);
 }
 
 bool Android_IsContentUri(const std::string &filename) {
@@ -258,6 +261,22 @@ int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath) {
 	return env->CallLongMethod(g_nativeActivity, filePathGetFreeStorageSpace, param);
 }
 
+int64_t Android_ComputeRecursiveDirectorySize(const std::string &uri) {
+	if (!g_nativeActivity) {
+		return false;
+	}
+	auto env = getEnv();
+
+	jstring param = env->NewStringUTF(uri.c_str());
+
+	double start = time_now_d();
+	int64_t size = env->CallLongMethod(g_nativeActivity, computeRecursiveDirectorySize, param);
+	double elapsed = time_now_d() - start;
+
+	INFO_LOG(IO, "ComputeRecursiveDirectorySize(%s) in %0.3f s", uri.c_str(), elapsed);
+	return size;
+}
+
 bool Android_IsExternalStoragePreservedLegacy() {
 	if (!g_nativeActivity) {
 		return false;

Common/File/AndroidStorage.h

@@ -49,6 +49,7 @@ StorageError Android_RemoveFile(const std::string &fileUri);
 StorageError Android_RenameFileTo(const std::string &fileUri, const std::string &newName);
 bool Android_GetFileInfo(const std::string &fileUri, File::FileInfo *info);
 bool Android_FileExists(const std::string &fileUri);
+int64_t Android_ComputeRecursiveDirectorySize(const std::string &fileUri);
 int64_t Android_GetFreeSpaceByContentUri(const std::string &uri);
 int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath);
 bool Android_IsExternalStoragePreservedLegacy();
@@ -72,6 +73,7 @@ inline StorageError Android_RemoveFile(const std::string &fileUri) { return Stor
 inline StorageError Android_RenameFileTo(const std::string &fileUri, const std::string &newName) { return StorageError::UNKNOWN; }
 inline bool Android_GetFileInfo(const std::string &fileUri, File::FileInfo *info) { return false; }
 inline bool Android_FileExists(const std::string &fileUri) { return false; }
+inline int64_t Android_ComputeRecursiveDirectorySize(const std::string &fileUri) { return -1; }
 inline int64_t Android_GetFreeSpaceByContentUri(const std::string &uri) { return -1; }
 inline int64_t Android_GetFreeSpaceByFilePath(const std::string &filePath) { return -1; }
 inline bool Android_IsExternalStoragePreservedLegacy() { return false; }

Common/File/DirListing.cpp

@@ -285,22 +285,6 @@ bool GetFilesInDir(const Path &directory, std::vector<FileInfo> *files, const ch
 	return true;
 }
 
-int64_t GetDirectoryRecursiveSize(const Path &path, const char *filter, int flags) {
-	std::vector<FileInfo> fileInfo;
-	GetFilesInDir(path, &fileInfo, filter, flags);
-	int64_t sizeSum = 0;
-	// Note: GetFilesInDir does not fill in fileSize.
-	for (size_t i = 0; i < fileInfo.size(); i++) {
-		FileInfo finfo;
-		GetFileInfo(fileInfo[i].fullName, &finfo);
-		if (!finfo.isDirectory)
-			sizeSum += finfo.size;
-		else
-			sizeSum += GetDirectoryRecursiveSize(finfo.fullName, filter, flags);
-	}
-	return sizeSum;
-}
-
 #if PPSSPP_PLATFORM(WINDOWS)
 // Returns a vector with the device names
 std::vector<std::string> GetWindowsDrives()

Common/File/DirListing.h

@@ -35,7 +35,6 @@ enum {
 };
 
 bool GetFilesInDir(const Path &directory, std::vector<FileInfo> *files, const char *filter = nullptr, int flags = 0);
-int64_t GetDirectoryRecursiveSize(const Path &path, const char *filter = nullptr, int flags = 0);
 std::vector<File::FileInfo> ApplyFilter(std::vector<File::FileInfo> files, const char *filter);
 
 #ifdef _WIN32

Common/File/FileUtil.cpp

@@ -324,6 +324,32 @@ std::string ResolvePath(const std::string &path) {
 #endif
 }
 
+static int64_t RecursiveSize(const Path &path) {
+	// TODO: Some file systems can optimize this.
+	std::vector<FileInfo> fileInfo;
+	if (!GetFilesInDir(path, &fileInfo, nullptr, GETFILES_GETHIDDEN)) {
+		return -1;
+	}
+	int64_t sizeSum = 0;
+	for (const auto &file : fileInfo) {
+		if (file.isDirectory) {
+			sizeSum += RecursiveSize(file.fullName);
+		} else {
+			sizeSum += file.size;
+		}
+	}
+	return sizeSum;
+}
+
+uint64_t ComputeRecursiveDirectorySize(const Path &path) {
+	if (path.Type() == PathType::CONTENT_URI) {
+		return Android_ComputeRecursiveDirectorySize(path.ToString());
+	}
+
+	// Generic solution.
+	return RecursiveSize(path);
+}
+
 // Returns true if file filename exists. Will return true on directories.
 bool ExistsInDir(const Path &path, const std::string &filename) {
 	return Exists(path / filename);

Common/File/FileUtil.h

@@ -79,6 +79,9 @@ uint64_t GetFileSize(const Path &filename);
 // Overloaded GetSize, accepts FILE*
 uint64_t GetFileSize(FILE *f);
 
+// Computes the recursive size of a directory. Warning: Might be slow!
+uint64_t ComputeRecursiveDirectorySize(const Path &path);
+
 // Returns true if successful, or path already exists.
 bool CreateDir(const Path &filename);
 

Core/FileSystems/BlobFileSystem.h

@@ -54,6 +54,8 @@ public:
 	bool RemoveFile(const std::string &filename) override;
 	u64 FreeSpace(const std::string &path) override;
 
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; }
+
 private:
 	// File positions.
 	std::map<u32, s64> entries_;

Core/FileSystems/DirectoryFileSystem.cpp

@@ -878,6 +878,18 @@ static std::string SimulateVFATBug(std::string filename) {
 	return filename;
 }
 
+bool DirectoryFileSystem::ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) {
+	Path localPath = GetLocalPath(path);
+
+	int64_t sizeTemp = File::ComputeRecursiveDirectorySize(localPath);
+	if (sizeTemp >= 0) {
+		*size = sizeTemp;
+		return true;
+	} else {
+		return false;
+	}
+}
+
 std::vector<PSPFileInfo> DirectoryFileSystem::GetDirListing(std::string path) {
 	std::vector<PSPFileInfo> myVector;
 

Core/FileSystems/DirectoryFileSystem.h

@@ -115,6 +115,8 @@ public:
 	FileSystemFlags Flags() override { return flags; }
 	u64 FreeSpace(const std::string &path) override;
 
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override;
+
 private:
 	struct OpenFileEntry {
 		DirectoryFileHandle hFile;
@@ -159,6 +161,8 @@ public:
 	FileSystemFlags Flags() override { return FileSystemFlags::FLASH; }
 	u64 FreeSpace(const std::string &path) override { return 0; }
 
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; }
+
 private:
 	struct OpenFileEntry {
 		u8 *fileData;

Core/FileSystems/FileSystem.h

@@ -136,11 +136,11 @@ public:
 	virtual PSPDevType DevType(u32 handle) = 0;
 	virtual FileSystemFlags Flags() = 0;
 	virtual u64      FreeSpace(const std::string &path) = 0;
+	virtual bool     ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) = 0;
 };
 
 
-class EmptyFileSystem : public IFileSystem
-{
+class EmptyFileSystem : public IFileSystem {
 public:
 	virtual void DoState(PointerWrap &p) override {}
 	std::vector<PSPFileInfo> GetDirListing(std::string path) override {std::vector<PSPFileInfo> vec; return vec;}
@@ -161,6 +161,7 @@ public:
 	virtual PSPDevType DevType(u32 handle) override { return PSPDevType::INVALID; }
 	virtual FileSystemFlags Flags() override { return FileSystemFlags::NONE; }
 	virtual u64 FreeSpace(const std::string &path) override { return 0; }
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; }
 };
 
 

Core/FileSystems/ISOFileSystem.h

@@ -54,6 +54,8 @@ public:
 	int  RenameFile(const std::string &from, const std::string &to) override { return -1; }
 	bool RemoveFile(const std::string &filename) override { return false; }
 
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; }
+
 private:
 	struct TreeEntry {
 		~TreeEntry();
@@ -150,6 +152,8 @@ public:
 	int  RenameFile(const std::string &from, const std::string &to) override { return -1; }
 	bool RemoveFile(const std::string &filename) override { return false; }
 
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; }
+
 private:
 	std::shared_ptr<IFileSystem> isoFileSystem_;
 };

Core/FileSystems/MetaFileSystem.cpp

@@ -500,12 +500,9 @@ bool MetaFileSystem::RemoveFile(const std::string &filename)
 	std::string of;
 	IFileSystem *system;
 	int error = MapFilePath(filename, of, &system);
-	if (error == 0)
-	{
+	if (error == 0) {
 		return system->RemoveFile(of);
-	}
-	else
-	{
+	} else {
 		return false;
 	}
 }
@@ -648,17 +645,36 @@ void MetaFileSystem::DoState(PointerWrap &p)
 	}
 }
 
-u64 MetaFileSystem::getDirSize(const std::string &dirPath) {
+int64_t MetaFileSystem::RecursiveSize(const std::string &dirPath) {
 	u64 result = 0;
 	auto allFiles = GetDirListing(dirPath);
 	for (auto file : allFiles) {
 		if (file.name == "." || file.name == "..")
 			continue;
 		if (file.type == FILETYPE_DIRECTORY) {
-			result += getDirSize(dirPath + file.name);
+			result += RecursiveSize(dirPath + file.name);
 		} else {
 			result += file.size;
 		}
 	}
 	return result;
 }
+
+int64_t MetaFileSystem::ComputeRecursiveDirectorySize(const std::string &filename) {
+	std::lock_guard<std::recursive_mutex> guard(lock);
+	std::string of;
+	IFileSystem *system;
+	int error = MapFilePath(filename, of, &system);
+	if (error == 0) {
+		int64_t size;
+		if (system->ComputeRecursiveDirSizeIfFast(of, &size)) {
+			// Some file systems can optimize this.
+			return size;
+		} else {
+			// Those that can't, we just run a generic implementation.
+			return RecursiveSize(filename);
+		}
+	} else {
+		return false;
+	}
+}

Core/FileSystems/MetaFileSystem.h

@@ -136,5 +136,19 @@ public:
 		startingDirectory = dir;
 	}
 
-	u64 getDirSize(const std::string &dirPath);
+	int64_t ComputeRecursiveDirectorySize(const std::string &dirPath);
+
+	// Shouldn't ever be called, but meh.
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override {
+		int64_t sizeTemp = ComputeRecursiveDirectorySize(path);
+		if (sizeTemp >= 0) {
+			*size = sizeTemp;
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+private:
+	int64_t RecursiveSize(const std::string &dirPath);
 };

Core/FileSystems/VirtualDiscFileSystem.h

@@ -52,6 +52,8 @@ public:
 	int  RenameFile(const std::string &from, const std::string &to) override;
 	bool RemoveFile(const std::string &filename) override;
 
+	bool ComputeRecursiveDirSizeIfFast(const std::string &path, int64_t *size) override { return false; }
+
 private:
 	void LoadFileListIndex();
 	// Warning: modifies input string.

Core/HLE/sceIo.cpp

@@ -22,6 +22,7 @@
 
 #include "Common/Thread/ThreadUtil.h"
 #include "Common/Profiler/Profiler.h"
+#include "Common/TimeUtil.h"
 
 #include "Common/File/FileUtil.h"
 #include "Common/Serialize/SerializeFuncs.h"
@@ -2321,15 +2322,25 @@ static u32 sceIoDopen(const char *path) {
 	DirListing *dir = new DirListing();
 	SceUID id = kernelObjects.Create(dir);
 
+	double startTime = time_now_d();
+
 	dir->listing = pspFileSystem.GetDirListing(path);
 	dir->index = 0;
 	dir->name = std::string(path);
 
+	double listTime = time_now_d() - startTime;
+
+	if (listTime > 0.01) {
+		INFO_LOG(IO, "Dir listing '%s' took %0.3f", path, listTime);
+	}
+
 	// Blacklist some directories that games should not be able to find out about.
 	// Speeds up directory iteration on slow Android Scoped Storage implementations :(
 	// Might also want to filter out PSP/GAME if not a homebrew, maybe also irrelevant directories
 	// in PSP/SAVEDATA, though iffy to know which ones are irrelevant..
-	if (!strcmp(path, "ms0:/PSP")) {
+	// Also if we're stripping PSP from the path due to setting a directory named PSP as the memstick root,
+	// these will also show up at ms0: which is not ideal. Should find some other way to deal with that.
+	if (!strcmp(path, "ms0:/PSP") || !strcmp(path, "ms0:")) {
 		static const char *const pspFolderBlacklist[] = {
 			"CHEATS",
 			"PPSSPP_STATE",
@@ -2349,6 +2360,10 @@ static u32 sceIoDopen(const char *path) {
 					break;
 				}
 			}
+			// Also don't let games see a GAME directory in the root. This confuses Wipeout...
+			if (!strcasecmp(entry.name.c_str(), "GAME") && !strcmp(path, "ms0:")) {
+				blacklisted = true;
+			}
 
 			if (!blacklisted) {
 				filtered.push_back(entry);

Core/HW/MemoryStick.cpp

@@ -96,7 +96,7 @@ static void MemoryStick_CalcInitialFree() {
 	std::unique_lock<std::mutex> guard(freeCalcMutex);
 	freeCalcStatus = FreeCalcStatus::RUNNING;
 	freeCalcThread = std::thread([] {
-		memstickInitialFree = pspFileSystem.FreeSpace("ms0:/") + pspFileSystem.getDirSize("ms0:/PSP/SAVEDATA/");
+		memstickInitialFree = pspFileSystem.FreeSpace("ms0:/") + pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/");
 
 		std::unique_lock<std::mutex> guard(freeCalcMutex);
 		freeCalcStatus = FreeCalcStatus::DONE;
@@ -127,7 +127,7 @@ u64 MemoryStick_FreeSpace() {
 
 	// Assume the memory stick is only used to store savedata.
 	if (!memstickCurrentUseValid) {
-		memstickCurrentUse = pspFileSystem.getDirSize("ms0:/PSP/SAVEDATA/");
+		memstickCurrentUse = pspFileSystem.ComputeRecursiveDirectorySize("ms0:/PSP/SAVEDATA/");
 		memstickCurrentUseValid = true;
 	}
 

UI/GameInfoCache.cpp

@@ -114,7 +114,7 @@ u64 GameInfo::GetGameSizeInBytes() {
 	switch (fileType) {
 	case IdentifiedFileType::PSP_PBP_DIRECTORY:
 	case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY:
-		return File::GetDirectoryRecursiveSize(ResolvePBPDirectory(filePath_), nullptr, File::GETFILES_GETHIDDEN);
+		return File::ComputeRecursiveDirectorySize(ResolvePBPDirectory(filePath_));
 
 	default:
 		return GetFileLoader()->FileSize();
@@ -343,13 +343,11 @@ public:
 	void Run() override {
 		// An early-return will result in the destructor running, where we can set
 		// flags like working and pending.
-
 		if (!info_->LoadFromPath(gamePath_)) {
 			return;
 		}
 
 		// In case of a remote file, check if it actually exists before locking.
-		// This is likely not necessary at a
 		if (!info_->GetFileLoader()->Exists()) {
 			return;
 		}

UI/SavedataScreen.cpp

@@ -430,8 +430,7 @@ static time_t GetTotalSize(const SavedataButton *b) {
 	switch (Identify_File(fileLoader.get(), &errorString)) {
 	case IdentifiedFileType::PSP_PBP_DIRECTORY:
 	case IdentifiedFileType::PSP_SAVEDATA_DIRECTORY:
-		return File::GetDirectoryRecursiveSize(ResolvePBPDirectory(b->GamePath()), nullptr, File::GETFILES_GETHIDDEN);
-
+		return File::ComputeRecursiveDirectorySize(ResolvePBPDirectory(b->GamePath()));
 	default:
 		return fileLoader->FileSize();
 	}

android/src/org/ppsspp/ppsspp/PpssppActivity.java

@@ -17,6 +17,8 @@ import android.database.Cursor;
 import android.provider.DocumentsContract;
 import android.os.Environment;
 import androidx.documentfile.provider.DocumentFile;
+
+import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.UUID;
 import java.io.File;
@@ -178,6 +180,70 @@ public class PpssppActivity extends NativeActivity {
 		return str + size + "|" + documentName + "|" + lastModified;
 	}
 
+	private long directorySizeRecursion(Uri uri) {
+		Cursor c = null;
+		try {
+			// Log.i(TAG, "recursing into " + uri.toString());
+			final String[] columns = new String[]{
+					DocumentsContract.Document.COLUMN_DOCUMENT_ID,
+					DocumentsContract.Document.COLUMN_SIZE,
+					DocumentsContract.Document.COLUMN_MIME_TYPE,  // check for MIME_TYPE_DIR
+			};
+			final ContentResolver resolver = getContentResolver();
+			final String documentId = DocumentsContract.getDocumentId(uri);
+			final Uri childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(uri, documentId);
+			c = resolver.query(childrenUri, columns, null, null, null);
+			long sizeSum = 0;
+
+			// Buffer the URIs so we only have one cursor active at once. I don't trust the storage framework
+			// to handle more than one...
+			ArrayList<Uri> childDirs = new ArrayList<Uri>();
+
+			while (c.moveToNext()) {
+				final String mimeType = c.getString(2);
+				final boolean isDirectory = mimeType.equals(DocumentsContract.Document.MIME_TYPE_DIR);
+				if (isDirectory) {
+					final String childDocumentId = c.getString(0);
+					final Uri childUri = DocumentsContract.buildDocumentUriUsingTree(uri, childDocumentId);
+					childDirs.add(childUri);
+				} else {
+					final long fileSize = c.getLong(1);
+					sizeSum += fileSize;
+				}
+			}
+			c.close();
+			c = null;
+
+			for (Uri childUri : childDirs) {
+				long dirSize = directorySizeRecursion(childUri);
+				if (dirSize >= 0) {
+					sizeSum += dirSize;
+				} else {
+					return dirSize;
+				}
+			}
+			return sizeSum;
+		} catch (Exception e) {
+			return -1;
+		} finally {
+			if (c != null) {
+				c.close();
+			}
+		}
+	}
+
+	public long computeRecursiveDirectorySize(String uriString) {
+		try {
+			Uri uri = Uri.parse(uriString);
+			long totalSize = directorySizeRecursion(uri);
+			return totalSize;
+		}
+		catch (Exception e) {
+			Log.e(TAG, "computeRecursiveSize exception: " + e.toString());
+			return -1;
+		}
+	}
+
 	// TODO: Maybe add a cheaper version that doesn't extract all the file information?
 	// TODO: Replace with a proper query:
 	// * https://stackoverflow.com/questions/42186820/documentfile-is-very-slow