From 102b7b03fa7d91cb1a9469124609074a560903ba Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Nov 2014 13:25:32 -0800 Subject: [PATCH 01/16] Add an interface to load files with. This allows loading from other than raw files. --- Core/Loaders.cpp | 153 +++++++++++++++++++++++++++++++------------ Core/Loaders.h | 60 ++++++++++++++++- Core/PSPLoaders.cpp | 15 +++-- Core/PSPLoaders.h | 4 +- Core/System.cpp | 11 +++- UI/GameInfoCache.cpp | 4 +- 6 files changed, 190 insertions(+), 57 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index afd7418160..c6c4341c9c 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -16,6 +16,7 @@ // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. #include "file/file_util.h" +#include #include "Common/FileUtil.h" #include "Core/MIPS/MIPS.h" @@ -30,34 +31,106 @@ #include "Core/ELF/PBPReader.h" #include "Core/ELF/ParamSFO.h" -// TODO : improve, look in the file more -IdentifiedFileType Identify_File(std::string &filename) -{ - if (filename.size() == 0) { - ERROR_LOG(LOADER, "invalid filename %s", filename.c_str()); - return FILETYPE_ERROR; +LocalFileLoader::LocalFileLoader(const std::string &filename) { + Reopen(filename); +} + +bool LocalFileLoader::Reopen(const std::string &filename) { + fd_ = 0; + f_ = nullptr; + filesize_ = 0; + filename_ = filename; + + f_ = File::OpenCFile(filename, "rb"); + if (!f_) { + return false; } +#ifdef ANDROID + fd_ = fileno(f_); + + off64_t off = lseek64(fd_, 0, SEEK_END); + filesize_ = off; + lseek64(fd_, 0, SEEK_SET); +#else + fseek(f_, 0, SEEK_END); + filesize_ = ftello(f_); + fseek(f_, 0, SEEK_SET); +#endif + + return true; +} + +LocalFileLoader::~LocalFileLoader() { + fclose(f_); +} + +bool LocalFileLoader::Exists() { FileInfo info; - if (!getFileInfo(filename.c_str(), &info)) { + return getFileInfo(filename_.c_str(), &info); +} + +bool LocalFileLoader::IsDirectory() { + FileInfo info; + if (getFileInfo(filename_.c_str(), &info)) { + return info.isDirectory; + } + return false; +} + +s64 LocalFileLoader::FileSize() { + return filesize_; +} + +std::string LocalFileLoader::Path() const { + return filename_; +} + +void LocalFileLoader::Seek(s64 absolutePos) { +#ifdef ANDROID + lseek64(fd_, absolutePos, SEEK_SET); +#else + fseeko(f_, absolutePos, SEEK_SET); +#endif +} + +size_t LocalFileLoader::Read(size_t bytes, size_t count, void *data) { +#ifdef ANDROID + return read(fd_, data, bytes * count) / bytes; +#else + return fread(data, bytes, count, f_); +#endif +} + +size_t LocalFileLoader::ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) { + Seek(absolutePos); + return Read(bytes, count, data); +} + +// TODO : improve, look in the file more +IdentifiedFileType Identify_File(FileLoader *fileLoader) +{ + if (fileLoader == nullptr) { + ERROR_LOG(LOADER, "Invalid fileLoader"); + return FILETYPE_ERROR; + } + if (fileLoader->Path().size() == 0) { + ERROR_LOG(LOADER, "Invalid filename %s", fileLoader->Path().c_str()); return FILETYPE_ERROR; } - std::string extension = filename.size() >= 5 ? filename.substr(filename.size() - 4) : ""; + if (!fileLoader->Exists()) { + return FILETYPE_ERROR; + } + + std::string extension = fileLoader->Extension(); if (!strcasecmp(extension.c_str(),".iso")) { // may be a psx iso, they have 2352 byte sectors. You never know what some people try to open - if ((info.size % 2352) == 0) + if ((fileLoader->FileSize() % 2352) == 0) { - FILE *f = File::OpenCFile(filename.c_str(), "rb"); - if (!f) { - // File does not exists - return FILETYPE_ERROR; - } - unsigned char sync[12]; - fread(sync,1,12,f); - fclose(f); + fileLoader->ReadAt(0, 12, sync); // each sector in a mode2 image starts with these 12 bytes if (memcmp(sync,"\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00",12) == 0) @@ -76,7 +149,8 @@ IdentifiedFileType Identify_File(std::string &filename) // First, check if it's a directory with an EBOOT.PBP in it. - if (info.isDirectory) { + if (fileLoader->IsDirectory()) { + std::string filename = fileLoader->Path(); if (filename.size() > 4) { FileInfo ebootInfo; // Check for existence of EBOOT.PBP, as required for "Directory games". @@ -97,17 +171,10 @@ IdentifiedFileType Identify_File(std::string &filename) return FILETYPE_NORMAL_DIRECTORY; } - FILE *f = File::OpenCFile(filename.c_str(), "rb"); - if (!f) { - // File does not exists - return FILETYPE_ERROR; - } - u32_le id; - size_t readSize = fread(&id, 4, 1, f); + size_t readSize = fileLoader->ReadAt(0, 4, 1, &id); if (readSize != 1) { - fclose(f); return FILETYPE_ERROR; } @@ -115,10 +182,8 @@ IdentifiedFileType Identify_File(std::string &filename) u32 _id = id; switch (_id) { case 'PBP\x00': - fseek(f, 0x24, SEEK_SET); - fread(&psar_offset, 4, 1, f); - fseek(f, psar_offset, SEEK_SET); - fread(&psar_id, 4, 1, f); + fileLoader->ReadAt(0x24, 4, 1, &psar_offset); + fileLoader->ReadAt(psar_offset, 4, 1, &psar_id); break; case '!raR': return FILETYPE_ARCHIVE_RAR; @@ -128,9 +193,8 @@ IdentifiedFileType Identify_File(std::string &filename) return FILETYPE_ARCHIVE_ZIP; } - fclose(f); - if (id == 'FLE\x7F') { + std::string filename = fileLoader->Path(); // There are a few elfs misnamed as pbp (like Trig Wars), accept that. if (!strcasecmp(extension.c_str(), ".plf") || strstr(filename.c_str(),"BOOT.BIN") || !strcasecmp(extension.c_str(), ".elf") || !strcasecmp(extension.c_str(), ".prx") || @@ -142,6 +206,8 @@ IdentifiedFileType Identify_File(std::string &filename) else if (id == 'PBP\x00') { // Do this PS1 eboot check FIRST before checking other eboot types. // It seems like some are malformed and slip through the PSAR check below. + // TODO: Change PBPReader to read FileLoader objects? + std::string filename = fileLoader->Path(); PBPReader pbp(filename.c_str()); if (pbp.IsValid()) { if (!pbp.IsELF()) { @@ -199,21 +265,22 @@ IdentifiedFileType Identify_File(std::string &filename) return FILETYPE_UNKNOWN; } -bool LoadFile(std::string &filename, std::string *error_string) { +bool LoadFile(FileLoader *fileLoader, std::string *error_string) { // Note that this can modify filename! - switch (Identify_File(filename)) { + switch (Identify_File(fileLoader)) { case FILETYPE_PSP_PBP_DIRECTORY: { + std::string filename = fileLoader->Path(); std::string ebootFilename = filename + "/EBOOT.PBP"; - FileInfo fileInfo; - getFileInfo((filename + "/EBOOT.PBP").c_str(), &fileInfo); - if (fileInfo.exists) { + // Switch fileLoader to the EBOOT. + fileLoader->Reopen(ebootFilename); + + if (fileLoader->Exists()) { INFO_LOG(LOADER, "File is a PBP in a directory!"); - std::string ebootPath = filename + "/EBOOT.PBP"; - IdentifiedFileType ebootType = Identify_File(ebootPath); + IdentifiedFileType ebootType = Identify_File(fileLoader); if (ebootType == FILETYPE_PSP_ISO_NP) { - InitMemoryForGameISO(ebootPath); + InitMemoryForGameISO(fileLoader); pspFileSystem.SetStartingDirectory("disc0:/PSP_GAME/USRDIR"); return Load_PSP_ISO(filename.c_str(), error_string); } @@ -225,7 +292,7 @@ bool LoadFile(std::string &filename, std::string *error_string) { size_t pos = path.find("/PSP/GAME/"); if (pos != std::string::npos) pspFileSystem.SetStartingDirectory("ms0:" + path.substr(pos)); - return Load_PSP_ELF_PBP(ebootFilename.c_str(), error_string); + return Load_PSP_ELF_PBP(fileLoader->Path().c_str(), error_string); } else { *error_string = "No EBOOT.PBP, misidentified game"; return false; @@ -236,14 +303,14 @@ bool LoadFile(std::string &filename, std::string *error_string) { case FILETYPE_PSP_ELF: { INFO_LOG(LOADER,"File is an ELF or loose PBP!"); - return Load_PSP_ELF_PBP(filename.c_str(), error_string); + return Load_PSP_ELF_PBP(fileLoader->Path().c_str(), error_string); } case FILETYPE_PSP_ISO: case FILETYPE_PSP_ISO_NP: case FILETYPE_PSP_DISC_DIRECTORY: // behaves the same as the mounting is already done by now pspFileSystem.SetStartingDirectory("disc0:/PSP_GAME/USRDIR"); - return Load_PSP_ISO(filename.c_str(), error_string); + return Load_PSP_ISO(fileLoader->Path().c_str(), error_string); case FILETYPE_PSP_PS1_PBP: *error_string = "PS1 EBOOTs are not supported by PPSSPP."; diff --git a/Core/Loaders.h b/Core/Loaders.h index ddf1ee5b43..c39fc471d6 100644 --- a/Core/Loaders.h +++ b/Core/Loaders.h @@ -46,9 +46,65 @@ enum IdentifiedFileType { FILETYPE_UNKNOWN }; +class FileLoader { +public: + virtual ~FileLoader() {} + + // Needed when we switch from a directory to a PBP, etc. + virtual bool Reopen(const std::string &filename) = 0; + + virtual bool Exists() = 0; + virtual bool IsDirectory() = 0; + virtual s64 FileSize() = 0; + virtual std::string Path() const = 0; + virtual std::string Extension() { + const std::string filename = Path(); + size_t pos = filename.find_last_of('.'); + if (pos == filename.npos) { + return ""; + } else { + return filename.substr(pos); + } + } + + virtual void Seek(s64 absolutePos) = 0; + virtual size_t Read(size_t bytes, size_t count, void *data) = 0; + virtual size_t Read(size_t bytes, void *data) { + return Read(1, bytes, data); + } + virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) = 0; + virtual size_t ReadAt(s64 absolutePos, size_t bytes, void *data) { + return ReadAt(absolutePos, 1, bytes, data); + } +}; + +class LocalFileLoader : public FileLoader { +public: + LocalFileLoader(const std::string &filename); + virtual ~LocalFileLoader(); + + virtual bool Reopen(const std::string &filename); + + virtual bool Exists() override; + virtual bool IsDirectory() override; + virtual s64 FileSize() override; + virtual std::string Path() const override; + + virtual void Seek(s64 absolutePos) override; + virtual size_t Read(size_t bytes, size_t count, void *data) override; + virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) override; + +private: + // First only used by Android, but we can keep it here for everyone. + int fd_; + FILE *f_; + u64 filesize_; + std::string filename_; +}; + // This can modify the string, for example for stripping off the "/EBOOT.PBP" // for a FILETYPE_PSP_PBP_DIRECTORY. -IdentifiedFileType Identify_File(std::string &str); +IdentifiedFileType Identify_File(FileLoader *fileLoader); // Can modify the string filename, as it calls IdentifyFile above. -bool LoadFile(std::string &filename, std::string *error_string); +bool LoadFile(FileLoader *fileLoader, std::string *error_string); diff --git a/Core/PSPLoaders.cpp b/Core/PSPLoaders.cpp index 5a14cd7173..e020d4865d 100644 --- a/Core/PSPLoaders.cpp +++ b/Core/PSPLoaders.cpp @@ -32,6 +32,7 @@ #include "FileSystems/MetaFileSystem.h" #include "FileSystems/VirtualDiscFileSystem.h" +#include "Core/Loaders.h" #include "Core/MemMap.h" #include "Core/HDRemaster.h" @@ -53,21 +54,21 @@ // We gather the game info before actually loading/booting the ISO // to determine if the emulator should enable extra memory and // double-sized texture coordinates. -void InitMemoryForGameISO(std::string fileToStart) { +void InitMemoryForGameISO(FileLoader *fileLoader) { IFileSystem* umd2; - // check if it's a disc directory - FileInfo info; - if (!getFileInfo(fileToStart.c_str(), &info)) return; + if (!fileLoader->Exists()) { + return; + } bool actualIso = false; - if (info.isDirectory) + if (fileLoader->IsDirectory()) { - umd2 = new VirtualDiscFileSystem(&pspFileSystem, fileToStart); + umd2 = new VirtualDiscFileSystem(&pspFileSystem, fileLoader->Path()); } else { - auto bd = constructBlockDevice(fileToStart.c_str()); + auto bd = constructBlockDevice(fileLoader->Path().c_str()); // Can't init anything without a block device... if (!bd) return; diff --git a/Core/PSPLoaders.h b/Core/PSPLoaders.h index 4faa7287d9..dc788772e4 100644 --- a/Core/PSPLoaders.h +++ b/Core/PSPLoaders.h @@ -19,6 +19,8 @@ #include +class FileLoader; + bool Load_PSP_ISO(const char *filename, std::string *error_string); bool Load_PSP_ELF_PBP(const char *filename, std::string *error_string); -void InitMemoryForGameISO(std::string fileToStart); +void InitMemoryForGameISO(FileLoader *fileLoader); diff --git a/Core/System.cpp b/Core/System.cpp index 7a87f7b61d..924463c9e6 100644 --- a/Core/System.cpp +++ b/Core/System.cpp @@ -72,6 +72,7 @@ MetaFileSystem pspFileSystem; ParamSFOData g_paramSFO; static GlobalUIState globalUIState; static CoreParameter coreParameter; +static FileLoader *loadedFile; static PSPMixer *mixer; static std::thread *cpuThread = NULL; static std::thread::id cpuThreadID; @@ -181,7 +182,8 @@ void CPU_Init() { Memory::g_PSPModel = g_Config.iPSPModel; std::string filename = coreParameter.fileToStart; - IdentifiedFileType type = Identify_File(filename); + loadedFile = new LocalFileLoader(filename); + IdentifiedFileType type = Identify_File(loadedFile); MIPSAnalyst::Reset(); Replacement_Init(); @@ -190,7 +192,7 @@ void CPU_Init() { case FILETYPE_PSP_ISO: case FILETYPE_PSP_ISO_NP: case FILETYPE_PSP_DISC_DIRECTORY: - InitMemoryForGameISO(filename); + InitMemoryForGameISO(loadedFile); break; default: break; @@ -213,7 +215,7 @@ void CPU_Init() { // TODO: Check Game INI here for settings, patches and cheats, and modify coreParameter accordingly // Why did we check for CORE_POWERDOWN here? - if (!LoadFile(filename, &coreParameter.errorString)) { + if (!LoadFile(loadedFile, &coreParameter.errorString)) { CPU_Shutdown(); coreParameter.fileToStart = ""; CPU_SetState(CPU_THREAD_NOT_RUNNING); @@ -245,6 +247,9 @@ void CPU_Shutdown() { pspFileSystem.Shutdown(); mipsr4k.Shutdown(); Memory::Shutdown(); + + delete loadedFile; + loadedFile = 0; } void CPU_RunLoop() { diff --git a/UI/GameInfoCache.cpp b/UI/GameInfoCache.cpp index 985950b5ff..872eeeebc6 100644 --- a/UI/GameInfoCache.cpp +++ b/UI/GameInfoCache.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include "base/logging.h" @@ -246,8 +247,9 @@ public: return; std::string filename = gamePath_; + std::unique_ptr fileLoader(new LocalFileLoader(filename)); info_->path = gamePath_; - info_->fileType = Identify_File(filename); + info_->fileType = Identify_File(fileLoader.get()); // Fallback title info_->title = getFilename(info_->path); From 967b58977859b9712603361f2e636cf281c2ebe4 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Nov 2014 13:59:56 -0800 Subject: [PATCH 02/16] Consistently use FileLoader for standard ISOs. --- Core/CoreParameter.h | 6 +- Core/FileSystems/BlockDevices.cpp | 128 ++++++++---------------------- Core/FileSystems/BlockDevices.h | 23 +++--- Core/HLE/sceUmd.cpp | 13 ++- Core/Loaders.cpp | 10 ++- Core/PSPLoaders.cpp | 12 +-- Core/PSPLoaders.h | 4 +- Core/System.cpp | 16 +++- Core/System.h | 2 + UI/GameInfoCache.cpp | 2 +- 10 files changed, 90 insertions(+), 126 deletions(-) diff --git a/Core/CoreParameter.h b/Core/CoreParameter.h index 3cb404f46e..b4c3a78991 100644 --- a/Core/CoreParameter.h +++ b/Core/CoreParameter.h @@ -31,9 +31,11 @@ enum GPUCore { GPU_DIRECTX9, }; +class FileLoader; + // PSP_CoreParameter() struct CoreParameter { - CoreParameter() : collectEmuLog(0), unthrottle(false), fpsLimit(0), updateRecent(true), freezeNext(false), frozen(false) {} + CoreParameter() : collectEmuLog(0), unthrottle(false), fpsLimit(0), updateRecent(true), freezeNext(false), frozen(false), mountIsoLoader(nullptr) {} CPUCore cpuCore; GPUCore gpuCore; bool enableSound; // there aren't multiple sound cores. @@ -65,4 +67,6 @@ struct CoreParameter { // Freeze-frame. For nvidia perfhud profiling. Developers only. bool freezeNext; bool frozen; + + FileLoader *mountIsoLoader; }; diff --git a/Core/FileSystems/BlockDevices.cpp b/Core/FileSystems/BlockDevices.cpp index 1ddf3fad38..41863baa06 100644 --- a/Core/FileSystems/BlockDevices.cpp +++ b/Core/FileSystems/BlockDevices.cpp @@ -17,6 +17,7 @@ #include "Common/FileUtil.h" +#include "Core/Loaders.h" #include "Core/FileSystems/BlockDevices.h" #include #include @@ -29,20 +30,19 @@ extern "C" #include "ext/libkirk/kirk_engine.h" }; -BlockDevice *constructBlockDevice(const char *filename) { +BlockDevice *constructBlockDevice(FileLoader *fileLoader) { // Check for CISO - FILE *f = File::OpenCFile(filename, "rb"); - if (!f) - return 0; + if (!fileLoader->Exists()) + return nullptr; char buffer[4]; - auto size = fread(buffer, 1, 4, f); //size_t - fseek(f, 0, SEEK_SET); + size_t size = fileLoader->ReadAt(0, 1, 4, buffer); + fileLoader->Seek(0); if (!memcmp(buffer, "CISO", 4) && size == 4) - return new CISOFileBlockDevice(f); + return new CISOFileBlockDevice(fileLoader); else if (!memcmp(buffer, "\x00PBP", 4) && size == 4) - return new NPDRMDemoBlockDevice(f); + return new NPDRMDemoBlockDevice(fileLoader); else - return new FileBlockDevice(f); + return new FileBlockDevice(fileLoader); } RAMBlockDevice::RAMBlockDevice(BlockDevice *device) { @@ -74,67 +74,16 @@ u32 RAMBlockDevice::GetNumBlocks() { - -// Android NDK does not support 64-bit file I/O using C streams -// so we fall back onto syscalls - -#ifdef ANDROID - -FileBlockDevice::FileBlockDevice(FILE *file) -: f(file) -{ - fd = fileno(file); - - off64_t off = lseek64(fd, 0, SEEK_END); - filesize = off; - lseek64(fd, 0, SEEK_SET); +FileBlockDevice::FileBlockDevice(FileLoader *fileLoader) + : fileLoader_(fileLoader) { + filesize_ = fileLoader->FileSize(); } -FileBlockDevice::~FileBlockDevice() -{ - fclose(f); +FileBlockDevice::~FileBlockDevice() { } -bool FileBlockDevice::ReadBlock(int blockNumber, u8 *outPtr) -{ - lseek64(fd, (u64)blockNumber * (u64)GetBlockSize(), SEEK_SET); - if (read(fd, outPtr, 2048) != 2048) { - ERROR_LOG(FILESYS, "Could not read() 2048 bytes from block"); - return false; - } - return true; -} - -bool FileBlockDevice::ReadBlocks(u32 minBlock, int count, u8 *outPtr) -{ - lseek64(fd, (u64)minBlock * (u64)GetBlockSize(), SEEK_SET); - const s32 bytes = GetBlockSize() * count; - if (read(fd, outPtr, bytes) != bytes) { - ERROR_LOG(FILESYS, "Could not read() %d bytes from block", bytes); - return false; - } - return true; -} - -#else - -FileBlockDevice::FileBlockDevice(FILE *file) - : f(file) -{ - fseek(f, 0, SEEK_END); - filesize = ftello(f); - fseek(f, 0, SEEK_SET); -} - -FileBlockDevice::~FileBlockDevice() -{ - fclose(f); -} - -bool FileBlockDevice::ReadBlock(int blockNumber, u8 *outPtr) -{ - fseeko(f, (u64)blockNumber * (u64)GetBlockSize(), SEEK_SET); - if (fread(outPtr, 1, 2048, f) != 2048) { +bool FileBlockDevice::ReadBlock(int blockNumber, u8 *outPtr) { + if (fileLoader_->ReadAt((u64)blockNumber * (u64)GetBlockSize(), 1, 2048, outPtr) != 2048) { DEBUG_LOG(FILESYS, "Could not read 2048 bytes from block"); return false; } @@ -142,18 +91,14 @@ bool FileBlockDevice::ReadBlock(int blockNumber, u8 *outPtr) return true; } -bool FileBlockDevice::ReadBlocks(u32 minBlock, int count, u8 *outPtr) -{ - fseeko(f, (u64)minBlock * (u64)GetBlockSize(), SEEK_SET); - if (fread(outPtr, 2048, count, f) != count) { +bool FileBlockDevice::ReadBlocks(u32 minBlock, int count, u8 *outPtr) { + if (fileLoader_->ReadAt((u64)minBlock * (u64)GetBlockSize(), 2048, count, outPtr) != count) { ERROR_LOG(FILESYS, "Could not read %d bytes from block", 2048 * count); return false; } return true; } -#endif - // .CSO format // compressed ISO(9660) header format @@ -184,14 +129,13 @@ typedef struct ciso_header static const u32 CSO_READ_BUFFER_SIZE = 256 * 1024; -CISOFileBlockDevice::CISOFileBlockDevice(FILE *file) - : f(file) +CISOFileBlockDevice::CISOFileBlockDevice(FileLoader *fileLoader) + : fileLoader_(fileLoader) { // CISO format is fairly simple, but most tools do not write the header_size. - f = file; CISO_H hdr; - size_t readSize = fread(&hdr, sizeof(CISO_H), 1, f); + size_t readSize = fileLoader->ReadAt(0, sizeof(CISO_H), 1, &hdr); if (readSize != 1 || memcmp(hdr.magic, "CISO", 4) != 0) { WARN_LOG(LOADER, "Invalid CSO!"); @@ -235,13 +179,13 @@ CISOFileBlockDevice::CISOFileBlockDevice(FILE *file) #if COMMON_LITTLE_ENDIAN index = new u32[indexSize]; - if (fread(index, sizeof(u32), indexSize, f) != indexSize) + if (fileLoader->ReadAt(sizeof(hdr), sizeof(u32), indexSize, index) != indexSize) memset(index, 0, indexSize * sizeof(u32)); #else index = new u32[indexSize]; u32_le *indexTemp = new u32_le[indexSize]; - if (fread(indexTemp, sizeof(u32), indexSize, f) != indexSize) + if (fileLoader->ReadAt(sizeof(hdr), sizeof(u32), indexSize, indexTemp) != indexSize) { memset(indexTemp, 0, indexSize * sizeof(u32_le)); } @@ -257,7 +201,6 @@ CISOFileBlockDevice::CISOFileBlockDevice(FILE *file) CISOFileBlockDevice::~CISOFileBlockDevice() { - fclose(f); delete [] index; delete [] readBuffer; delete [] zlibBuffer; @@ -285,8 +228,7 @@ bool CISOFileBlockDevice::ReadBlock(int blockNumber, u8 *outPtr) const int plain = idx & 0x80000000; if (plain) { - fseeko(f, compressedReadPos + compressedOffset, SEEK_SET); - int readSize = (u32)fread(outPtr, 1, GetBlockSize(), f); + int readSize = (u32)fileLoader_->ReadAt(compressedReadPos + compressedOffset, 1, GetBlockSize(), outPtr); if (readSize < GetBlockSize()) memset(outPtr + readSize, 0, GetBlockSize() - readSize); } @@ -297,8 +239,7 @@ bool CISOFileBlockDevice::ReadBlock(int blockNumber, u8 *outPtr) } else { - fseeko(f, compressedReadPos, SEEK_SET); - const u32 readSize = (u32)fread(readBuffer, 1, compressedReadSize, f); + const u32 readSize = (u32)fileLoader_->ReadAt(compressedReadPos, 1, compressedReadSize, readBuffer); z.zalloc = Z_NULL; z.zfree = Z_NULL; @@ -387,8 +328,7 @@ bool CISOFileBlockDevice::ReadBlocks(u32 minBlock, int count, u8 *outPtr) { const s64 maxNeeded = totalReadEnd - frameReadPos; const size_t chunkSize = (size_t)std::min(maxNeeded, (s64)std::max(frameReadSize, CSO_READ_BUFFER_SIZE)); - fseeko(f, frameReadPos, SEEK_SET); - const u32 readSize = (u32)fread(readBuffer, 1, chunkSize, f); + const u32 readSize = (u32)fileLoader_->ReadAt(frameReadPos, 1, chunkSize, readBuffer); if (readSize < chunkSize) { memset(readBuffer + readSize, 0, chunkSize - readSize); } @@ -432,8 +372,8 @@ bool CISOFileBlockDevice::ReadBlocks(u32 minBlock, int count, u8 *outPtr) { } -NPDRMDemoBlockDevice::NPDRMDemoBlockDevice(FILE *file) - : f(file) +NPDRMDemoBlockDevice::NPDRMDemoBlockDevice(FileLoader *fileLoader) + : fileLoader_(fileLoader) { MAC_KEY mkey; CIPHER_KEY ckey; @@ -441,10 +381,8 @@ NPDRMDemoBlockDevice::NPDRMDemoBlockDevice(FILE *file) u32 tableOffset, tableSize; u32 lbaStart, lbaEnd; - fseek(f, 0x24, SEEK_SET); - fread(&psarOffset, 1, 4, f); - fseek(f, psarOffset, SEEK_SET); - size_t readSize = fread(&np_header, 1, 256, f); + fileLoader_->ReadAt(0x24, 1, 4, &psarOffset); + size_t readSize = fileLoader_->ReadAt(psarOffset, 1, 256, &np_header); if(readSize!=256){ ERROR_LOG(LOADER, "Invalid NPUMDIMG header!"); } @@ -473,12 +411,11 @@ NPDRMDemoBlockDevice::NPDRMDemoBlockDevice(FILE *file) tempBuf = new u8[blockSize]; tableOffset = *(u32*)(np_header+0x6c); // table offset - fseek(f, psarOffset+tableOffset, SEEK_SET); tableSize = numBlocks*32; table = new table_info[numBlocks]; - readSize = fread(table, 1, tableSize, f); + readSize = fileLoader_->ReadAt(psarOffset + tableOffset, 1, tableSize, table); if(readSize!=tableSize){ ERROR_LOG(LOADER, "Invalid NPUMDIMG table!"); } @@ -503,7 +440,6 @@ NPDRMDemoBlockDevice::NPDRMDemoBlockDevice(FILE *file) NPDRMDemoBlockDevice::~NPDRMDemoBlockDevice() { - fclose(f); delete [] table; delete [] tempBuf; delete [] blockBuf; @@ -535,14 +471,12 @@ bool NPDRMDemoBlockDevice::ReadBlock(int blockNumber, u8 *outPtr) return false; } - fseek(f, psarOffset+table[block].offset, SEEK_SET); - if(table[block].sizeReadAt(psarOffset+table[block].offset, 1, table[block].size, readBuf); if(readSize != (size_t)table[block].size){ if((u32)block==(numBlocks-1)) return true; diff --git a/Core/FileSystems/BlockDevices.h b/Core/FileSystems/BlockDevices.h index 3223c2bad6..a715514a9f 100644 --- a/Core/FileSystems/BlockDevices.h +++ b/Core/FileSystems/BlockDevices.h @@ -26,6 +26,8 @@ #include "Common/CommonTypes.h" #include "Core/ELF/PBPReader.h" +class FileLoader; + class BlockDevice { public: @@ -48,14 +50,14 @@ public: class CISOFileBlockDevice : public BlockDevice { public: - CISOFileBlockDevice(FILE *file); + CISOFileBlockDevice(FileLoader *fileLoader); ~CISOFileBlockDevice(); bool ReadBlock(int blockNumber, u8 *outPtr) override; bool ReadBlocks(u32 minBlock, int count, u8 *outPtr) override; u32 GetNumBlocks() { return numBlocks;} private: - FILE *f; + FileLoader *fileLoader_; u32 *index; u8 *readBuffer; u8 *zlibBuffer; @@ -71,18 +73,15 @@ private: class FileBlockDevice : public BlockDevice { public: - FileBlockDevice(FILE *file); + FileBlockDevice(FileLoader *fileLoader); ~FileBlockDevice(); bool ReadBlock(int blockNumber, u8 *outPtr) override; bool ReadBlocks(u32 minBlock, int count, u8 *outPtr) override; - u32 GetNumBlocks() override {return (u32)(filesize / GetBlockSize());} + u32 GetNumBlocks() override {return (u32)(filesize_ / GetBlockSize());} private: -#ifdef ANDROID - int fd; -#endif - FILE *f; - u64 filesize; + FileLoader *fileLoader_; + u64 filesize_; }; @@ -99,14 +98,14 @@ struct table_info { class NPDRMDemoBlockDevice : public BlockDevice { public: - NPDRMDemoBlockDevice(FILE *file); + NPDRMDemoBlockDevice(FileLoader *fileLoader); ~NPDRMDemoBlockDevice(); bool ReadBlock(int blockNumber, u8 *outPtr) override; u32 GetNumBlocks() override {return (u32)lbaSize;} private: - FILE *f; + FileLoader *fileLoader_; u32 lbaSize; u32 psarOffset; @@ -140,4 +139,4 @@ private: }; -BlockDevice *constructBlockDevice(const char *filename); +BlockDevice *constructBlockDevice(FileLoader *fileLoader); diff --git a/Core/HLE/sceUmd.cpp b/Core/HLE/sceUmd.cpp index 1e54d38d85..22f3761dee 100644 --- a/Core/HLE/sceUmd.cpp +++ b/Core/HLE/sceUmd.cpp @@ -20,6 +20,7 @@ #include "file/file_util.h" #include "Common/ChunkFile.h" +#include "Core/Loaders.h" #include "Core/MemMap.h" #include "Core/System.h" #include "Core/CoreTiming.h" @@ -460,20 +461,28 @@ u32 sceUmdGetErrorStat() } void __UmdReplace(std::string filepath) { + // TODO: This should really go through Loaders, no? What if it's an invalid file? + // Only get system from disc0 seems have been enough. IFileSystem* currentUMD = pspFileSystem.GetSystem("disc0:"); IFileSystem* currentISOBlock = pspFileSystem.GetSystem("umd0:"); if (!currentUMD) return; + FileLoader *loadedFile = new LocalFileLoader(filepath); + IFileSystem* umd2; FileInfo info; - if (!getFileInfo(filepath.c_str(), &info)) // This shouldn't happen, but for safety. + if (!loadedFile->Exists()) { + delete loadedFile; return; + } + UpdateLoadedFile(loadedFile); + if (info.isDirectory) { umd2 = new VirtualDiscFileSystem(&pspFileSystem, filepath); } else { - auto bd = constructBlockDevice(filepath.c_str()); + auto bd = constructBlockDevice(loadedFile); if (!bd) return; umd2 = new ISOFileSystem(&pspFileSystem, bd); diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index c6c4341c9c..381d3ee4c9 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -47,6 +47,8 @@ bool LocalFileLoader::Reopen(const std::string &filename) { } #ifdef ANDROID + // Android NDK does not support 64-bit file I/O using C streams + // so we fall back onto syscalls fd_ = fileno(f_); off64_t off = lseek64(fd_, 0, SEEK_END); @@ -282,7 +284,7 @@ bool LoadFile(FileLoader *fileLoader, std::string *error_string) { if (ebootType == FILETYPE_PSP_ISO_NP) { InitMemoryForGameISO(fileLoader); pspFileSystem.SetStartingDirectory("disc0:/PSP_GAME/USRDIR"); - return Load_PSP_ISO(filename.c_str(), error_string); + return Load_PSP_ISO(fileLoader, error_string); } else if (ebootType == FILETYPE_PSP_PS1_PBP) { *error_string = "PS1 EBOOTs are not supported by PPSSPP."; @@ -292,7 +294,7 @@ bool LoadFile(FileLoader *fileLoader, std::string *error_string) { size_t pos = path.find("/PSP/GAME/"); if (pos != std::string::npos) pspFileSystem.SetStartingDirectory("ms0:" + path.substr(pos)); - return Load_PSP_ELF_PBP(fileLoader->Path().c_str(), error_string); + return Load_PSP_ELF_PBP(fileLoader, error_string); } else { *error_string = "No EBOOT.PBP, misidentified game"; return false; @@ -303,14 +305,14 @@ bool LoadFile(FileLoader *fileLoader, std::string *error_string) { case FILETYPE_PSP_ELF: { INFO_LOG(LOADER,"File is an ELF or loose PBP!"); - return Load_PSP_ELF_PBP(fileLoader->Path().c_str(), error_string); + return Load_PSP_ELF_PBP(fileLoader, error_string); } case FILETYPE_PSP_ISO: case FILETYPE_PSP_ISO_NP: case FILETYPE_PSP_DISC_DIRECTORY: // behaves the same as the mounting is already done by now pspFileSystem.SetStartingDirectory("disc0:/PSP_GAME/USRDIR"); - return Load_PSP_ISO(fileLoader->Path().c_str(), error_string); + return Load_PSP_ISO(fileLoader, error_string); case FILETYPE_PSP_PS1_PBP: *error_string = "PS1 EBOOTs are not supported by PPSSPP."; diff --git a/Core/PSPLoaders.cpp b/Core/PSPLoaders.cpp index e020d4865d..2831d4daaf 100644 --- a/Core/PSPLoaders.cpp +++ b/Core/PSPLoaders.cpp @@ -68,7 +68,7 @@ void InitMemoryForGameISO(FileLoader *fileLoader) { } else { - auto bd = constructBlockDevice(fileLoader->Path().c_str()); + auto bd = constructBlockDevice(fileLoader); // Can't init anything without a block device... if (!bd) return; @@ -151,7 +151,7 @@ static const char *altBootNames[] = { "disc0:/PSP_GAME/SYSDIR/ss.RAW", }; -bool Load_PSP_ISO(const char *filename, std::string *error_string) +bool Load_PSP_ISO(FileLoader *fileLoader, std::string *error_string) { // Mounting stuff relocated to InitMemoryForGameISO due to HD Remaster restructuring of code. @@ -224,12 +224,12 @@ static std::string NormalizePath(const std::string &path) return buf; } -bool Load_PSP_ELF_PBP(const char *filename, std::string *error_string) +bool Load_PSP_ELF_PBP(FileLoader *fileLoader, std::string *error_string) { // This is really just for headless, might need tweaking later. - if (!PSP_CoreParameter().mountIso.empty()) + if (PSP_CoreParameter().mountIsoLoader != nullptr) { - auto bd = constructBlockDevice(PSP_CoreParameter().mountIso.c_str()); + auto bd = constructBlockDevice(PSP_CoreParameter().mountIsoLoader); if (bd != NULL) { ISOFileSystem *umd2 = new ISOFileSystem(&pspFileSystem, bd); @@ -239,7 +239,7 @@ bool Load_PSP_ELF_PBP(const char *filename, std::string *error_string) } } - std::string full_path = filename; + std::string full_path = fileLoader->Path(); std::string path, file, extension; SplitPath(ReplaceAll(full_path, "\\", "/"), &path, &file, &extension); #ifdef _WIN32 diff --git a/Core/PSPLoaders.h b/Core/PSPLoaders.h index dc788772e4..3dce913c08 100644 --- a/Core/PSPLoaders.h +++ b/Core/PSPLoaders.h @@ -21,6 +21,6 @@ class FileLoader; -bool Load_PSP_ISO(const char *filename, std::string *error_string); -bool Load_PSP_ELF_PBP(const char *filename, std::string *error_string); +bool Load_PSP_ISO(FileLoader *fileLoader, std::string *error_string); +bool Load_PSP_ELF_PBP(FileLoader *fileLoader, std::string *error_string); void InitMemoryForGameISO(FileLoader *fileLoader); diff --git a/Core/System.cpp b/Core/System.cpp index 924463c9e6..d60a42e44f 100644 --- a/Core/System.cpp +++ b/Core/System.cpp @@ -185,6 +185,11 @@ void CPU_Init() { loadedFile = new LocalFileLoader(filename); IdentifiedFileType type = Identify_File(loadedFile); + // TODO: Put this somewhere better? + if (coreParameter.mountIso != "") { + coreParameter.mountIsoLoader = new LocalFileLoader(coreParameter.mountIso); + } + MIPSAnalyst::Reset(); Replacement_Init(); @@ -249,7 +254,16 @@ void CPU_Shutdown() { Memory::Shutdown(); delete loadedFile; - loadedFile = 0; + loadedFile = nullptr; + + delete coreParameter.mountIsoLoader; + coreParameter.mountIsoLoader = nullptr; +} + +// TODO: Maybe loadedFile doesn't even belong here... +void UpdateLoadedFile(FileLoader *fileLoader) { + delete loadedFile; + loadedFile = fileLoader; } void CPU_RunLoop() { diff --git a/Core/System.h b/Core/System.h index 336ac59ff6..651aac0733 100644 --- a/Core/System.h +++ b/Core/System.h @@ -63,6 +63,8 @@ void Audio_Init(); bool IsOnSeparateCPUThread(); bool IsAudioInitialised(); +void UpdateLoadedFile(FileLoader *fileLoader); + std::string GetSysDirectory(PSPDirectories directoryType); #ifdef _WIN32 void InitSysDirectories(); diff --git a/UI/GameInfoCache.cpp b/UI/GameInfoCache.cpp index 872eeeebc6..81734392ac 100644 --- a/UI/GameInfoCache.cpp +++ b/UI/GameInfoCache.cpp @@ -378,7 +378,7 @@ handleELF: // Let's assume it's an ISO. // TODO: This will currently read in the whole directory tree. Not really necessary for just a // few files. - BlockDevice *bd = constructBlockDevice(gamePath_.c_str()); + BlockDevice *bd = constructBlockDevice(fileLoader.get()); if (!bd) return; // nothing to do here.. ISOFileSystem umd(&handles, bd, "/PSP_GAME"); From fd2d7406d17fac787ab5f9b3cdce85a09dfc131b Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Nov 2014 14:02:35 -0800 Subject: [PATCH 03/16] Make LocalFileLoader essentially private. --- Core/HLE/sceUmd.cpp | 2 +- Core/Loaders.cpp | 28 ++++++++++++++++++++++++++++ Core/Loaders.h | 24 +----------------------- Core/System.cpp | 4 ++-- UI/GameInfoCache.cpp | 2 +- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/Core/HLE/sceUmd.cpp b/Core/HLE/sceUmd.cpp index 22f3761dee..b5688260ce 100644 --- a/Core/HLE/sceUmd.cpp +++ b/Core/HLE/sceUmd.cpp @@ -469,7 +469,7 @@ void __UmdReplace(std::string filepath) { if (!currentUMD) return; - FileLoader *loadedFile = new LocalFileLoader(filepath); + FileLoader *loadedFile = ConstructFileLoader(filepath); IFileSystem* umd2; FileInfo info; diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index 381d3ee4c9..f64c57279a 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -31,6 +31,34 @@ #include "Core/ELF/PBPReader.h" #include "Core/ELF/ParamSFO.h" +class LocalFileLoader : public FileLoader { +public: + LocalFileLoader(const std::string &filename); + virtual ~LocalFileLoader(); + + virtual bool Reopen(const std::string &filename); + + virtual bool Exists() override; + virtual bool IsDirectory() override; + virtual s64 FileSize() override; + virtual std::string Path() const override; + + virtual void Seek(s64 absolutePos) override; + virtual size_t Read(size_t bytes, size_t count, void *data) override; + virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) override; + +private: + // First only used by Android, but we can keep it here for everyone. + int fd_; + FILE *f_; + u64 filesize_; + std::string filename_; +}; + +FileLoader *ConstructFileLoader(const std::string &filename) { + return new LocalFileLoader(filename); +} + LocalFileLoader::LocalFileLoader(const std::string &filename) { Reopen(filename); } diff --git a/Core/Loaders.h b/Core/Loaders.h index c39fc471d6..86069914f3 100644 --- a/Core/Loaders.h +++ b/Core/Loaders.h @@ -78,29 +78,7 @@ public: } }; -class LocalFileLoader : public FileLoader { -public: - LocalFileLoader(const std::string &filename); - virtual ~LocalFileLoader(); - - virtual bool Reopen(const std::string &filename); - - virtual bool Exists() override; - virtual bool IsDirectory() override; - virtual s64 FileSize() override; - virtual std::string Path() const override; - - virtual void Seek(s64 absolutePos) override; - virtual size_t Read(size_t bytes, size_t count, void *data) override; - virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) override; - -private: - // First only used by Android, but we can keep it here for everyone. - int fd_; - FILE *f_; - u64 filesize_; - std::string filename_; -}; +FileLoader *ConstructFileLoader(const std::string &filename); // This can modify the string, for example for stripping off the "/EBOOT.PBP" // for a FILETYPE_PSP_PBP_DIRECTORY. diff --git a/Core/System.cpp b/Core/System.cpp index d60a42e44f..da52d185a1 100644 --- a/Core/System.cpp +++ b/Core/System.cpp @@ -182,12 +182,12 @@ void CPU_Init() { Memory::g_PSPModel = g_Config.iPSPModel; std::string filename = coreParameter.fileToStart; - loadedFile = new LocalFileLoader(filename); + loadedFile = ConstructFileLoader(filename); IdentifiedFileType type = Identify_File(loadedFile); // TODO: Put this somewhere better? if (coreParameter.mountIso != "") { - coreParameter.mountIsoLoader = new LocalFileLoader(coreParameter.mountIso); + coreParameter.mountIsoLoader = ConstructFileLoader(coreParameter.mountIso); } MIPSAnalyst::Reset(); diff --git a/UI/GameInfoCache.cpp b/UI/GameInfoCache.cpp index 81734392ac..a8d18e5cc4 100644 --- a/UI/GameInfoCache.cpp +++ b/UI/GameInfoCache.cpp @@ -247,7 +247,7 @@ public: return; std::string filename = gamePath_; - std::unique_ptr fileLoader(new LocalFileLoader(filename)); + std::unique_ptr fileLoader(ConstructFileLoader(filename)); info_->path = gamePath_; info_->fileType = Identify_File(fileLoader.get()); // Fallback title From b620070fa6a10ce09af8d64c9a10486b461dabdb Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Nov 2014 14:08:14 -0800 Subject: [PATCH 04/16] Kill FileLoader::Reopen()... Was a bad idea. --- Core/Loaders.cpp | 24 +++++++----------------- Core/Loaders.h | 5 +---- Core/System.cpp | 2 +- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index f64c57279a..c3f8869fdb 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -36,8 +36,6 @@ public: LocalFileLoader(const std::string &filename); virtual ~LocalFileLoader(); - virtual bool Reopen(const std::string &filename); - virtual bool Exists() override; virtual bool IsDirectory() override; virtual s64 FileSize() override; @@ -59,19 +57,11 @@ FileLoader *ConstructFileLoader(const std::string &filename) { return new LocalFileLoader(filename); } -LocalFileLoader::LocalFileLoader(const std::string &filename) { - Reopen(filename); -} - -bool LocalFileLoader::Reopen(const std::string &filename) { - fd_ = 0; - f_ = nullptr; - filesize_ = 0; - filename_ = filename; - +LocalFileLoader::LocalFileLoader(const std::string &filename) + : fd_(0), f_(nullptr), filesize_(0), filename_(filename) { f_ = File::OpenCFile(filename, "rb"); if (!f_) { - return false; + return; } #ifdef ANDROID @@ -87,8 +77,6 @@ bool LocalFileLoader::Reopen(const std::string &filename) { filesize_ = ftello(f_); fseek(f_, 0, SEEK_SET); #endif - - return true; } LocalFileLoader::~LocalFileLoader() { @@ -295,7 +283,8 @@ IdentifiedFileType Identify_File(FileLoader *fileLoader) return FILETYPE_UNKNOWN; } -bool LoadFile(FileLoader *fileLoader, std::string *error_string) { +bool LoadFile(FileLoader **fileLoaderPtr, std::string *error_string) { + FileLoader *&fileLoader = *fileLoaderPtr; // Note that this can modify filename! switch (Identify_File(fileLoader)) { case FILETYPE_PSP_PBP_DIRECTORY: @@ -304,7 +293,8 @@ bool LoadFile(FileLoader *fileLoader, std::string *error_string) { std::string ebootFilename = filename + "/EBOOT.PBP"; // Switch fileLoader to the EBOOT. - fileLoader->Reopen(ebootFilename); + delete fileLoader; + fileLoader = ConstructFileLoader(ebootFilename); if (fileLoader->Exists()) { INFO_LOG(LOADER, "File is a PBP in a directory!"); diff --git a/Core/Loaders.h b/Core/Loaders.h index 86069914f3..f3b56668f3 100644 --- a/Core/Loaders.h +++ b/Core/Loaders.h @@ -50,9 +50,6 @@ class FileLoader { public: virtual ~FileLoader() {} - // Needed when we switch from a directory to a PBP, etc. - virtual bool Reopen(const std::string &filename) = 0; - virtual bool Exists() = 0; virtual bool IsDirectory() = 0; virtual s64 FileSize() = 0; @@ -85,4 +82,4 @@ FileLoader *ConstructFileLoader(const std::string &filename); IdentifiedFileType Identify_File(FileLoader *fileLoader); // Can modify the string filename, as it calls IdentifyFile above. -bool LoadFile(FileLoader *fileLoader, std::string *error_string); +bool LoadFile(FileLoader **fileLoaderPtr, std::string *error_string); diff --git a/Core/System.cpp b/Core/System.cpp index da52d185a1..ece502bad4 100644 --- a/Core/System.cpp +++ b/Core/System.cpp @@ -220,7 +220,7 @@ void CPU_Init() { // TODO: Check Game INI here for settings, patches and cheats, and modify coreParameter accordingly // Why did we check for CORE_POWERDOWN here? - if (!LoadFile(loadedFile, &coreParameter.errorString)) { + if (!LoadFile(&loadedFile, &coreParameter.errorString)) { CPU_Shutdown(); coreParameter.fileToStart = ""; CPU_SetState(CPU_THREAD_NOT_RUNNING); From 6dca6ad9bdf1dfee7f3066f6fc3a35acc916eaf0 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Nov 2014 16:14:11 -0800 Subject: [PATCH 05/16] Add an HTTP transport for loading isos. Not exposed in UI yet, but you can use command line. --- Core/Loaders.cpp | 158 +++++++++++++++++++++++++++++++++++++++++++++++ native | 2 +- 2 files changed, 159 insertions(+), 1 deletion(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index c3f8869fdb..7f23a02a58 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -15,7 +15,11 @@ // Official git repository and contact information can be found at // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. +#include "base/stringutil.h" #include "file/file_util.h" +#include "net/http_client.h" +#include "net/resolve.h" +#include "net/url.h" #include #include "Common/FileUtil.h" @@ -53,7 +57,41 @@ private: std::string filename_; }; +class HTTPFileLoader : public FileLoader { +public: + HTTPFileLoader(const std::string &filename); + virtual ~HTTPFileLoader(); + + virtual bool Exists() override; + virtual bool IsDirectory() override; + virtual s64 FileSize() override; + virtual std::string Path() const override; + + virtual void Seek(s64 absolutePos) override; + virtual size_t Read(size_t bytes, size_t count, void *data) override { + return ReadAt(filepos_, bytes, count, data); + } + virtual size_t Read(size_t bytes, void *data) override { + return ReadAt(filepos_, bytes, data); + } + virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) override { + return ReadAt(absolutePos, bytes * count, data) / bytes; + } + virtual size_t ReadAt(s64 absolutePos, size_t bytes, void *data) override; + +private: + // TODO: Caching, etc. + s64 filesize_; + s64 filepos_; + Url url_; + net::AutoInit netInit_; + http::Client client_; + std::string filename_; +}; + FileLoader *ConstructFileLoader(const std::string &filename) { + if (filename.find("http://") == 0 || filename.find("https://") == 0) + return new HTTPFileLoader(filename); return new LocalFileLoader(filename); } @@ -125,6 +163,126 @@ size_t LocalFileLoader::ReadAt(s64 absolutePos, size_t bytes, size_t count, void return Read(bytes, count, data); } +HTTPFileLoader::HTTPFileLoader(const std::string &filename) + : filesize_(0), url_(filename), filename_(filename) { + if (!client_.Resolve(url_.Host().c_str(), 80)) { + return; + } + + // TODO: Keepalive, etc. + client_.Connect(); + int err = client_.SendRequest("HEAD", url_.Resource().c_str()); + if (err < 0) { + return; + } + + Buffer readbuf; + std::vector responseHeaders; + int code = client_.ReadResponseHeaders(&readbuf, responseHeaders); + if (code != 200) { + // Leave size at 0, invalid. + return; + } + + // TODO: Expire cache via ETag, etc. + for (std::string header : responseHeaders) { + if (startsWith(header, "Content-Length:")) { + size_t size_pos = header.find_first_of(' '); + if (size_pos != header.npos) { + size_pos = header.find_first_not_of(' ', size_pos); + } + if (size_pos != header.npos) { + filesize_ = atoll(&header[size_pos]); + } + } + } + + client_.Disconnect(); + + // If we didn't end up with a filesize_ (e.g. chunked response), give up. File invalid. +} + +HTTPFileLoader::~HTTPFileLoader() { +} + +bool HTTPFileLoader::Exists() { + // TODO + return url_.Valid() && filesize_ > 0; +} + +bool HTTPFileLoader::IsDirectory() { + // Only files. + return false; +} + +s64 HTTPFileLoader::FileSize() { + return filesize_; +} + +std::string HTTPFileLoader::Path() const { + return filename_; +} + +void HTTPFileLoader::Seek(s64 absolutePos) { + filepos_ = absolutePos; +} + +size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { + s64 absoluteEnd = absolutePos + bytes; + + // TODO: Keepalive, etc. + client_.Connect(); + + char requestHeaders[4096]; + // Note that the Range header is *inclusive*. + snprintf(requestHeaders, sizeof(requestHeaders), + "Range: bytes=%lld-%lld\r\n", absolutePos, absoluteEnd - 1); + + int err = client_.SendRequest("GET", url_.Resource().c_str(), requestHeaders); + if (err < 0) { + return 0; + } + + Buffer readbuf; + std::vector responseHeaders; + int code = client_.ReadResponseHeaders(&readbuf, responseHeaders); + if (code != 206) { + ERROR_LOG(LOADER, "HTTP server does not support range requests."); + return 0; + } + + // TODO: Expire cache via ETag, etc. + // We don't support multipart/byteranges responses. + bool supportedResponse = false; + for (std::string header : responseHeaders) { + if (startsWith(header, "Content-Range:")) { + // TODO: More correctness. Whitespace can be missing or different. + s64 first = -1, last = -1, total = -1; + if (sscanf(header.c_str(), "Content-Range: bytes %lld-%lld/%lld", &first, &last, &total) >= 2) { + if (first == absolutePos && last == absoluteEnd - 1) { + supportedResponse = true; + } + } + } + } + + // TODO: Would be nice to read directly. + Buffer output; + client_.ReadResponseEntity(&readbuf, responseHeaders, &output); + + if (!supportedResponse) { + ERROR_LOG(LOADER, "HTTP server did not respond with the range we wanted."); + return 0; + } + + client_.Disconnect(); + + size_t readBytes = output.size(); + output.Take(readBytes, (char *)data); + filepos_ = absolutePos + readBytes; + return readBytes; +} + // TODO : improve, look in the file more IdentifiedFileType Identify_File(FileLoader *fileLoader) { diff --git a/native b/native index 92be7b0b1b..73d5ad3b5c 160000 --- a/native +++ b/native @@ -1 +1 @@ -Subproject commit 92be7b0b1bb73b01b4322d787b9c54d03ed7c270 +Subproject commit 73d5ad3b5c937afceb95569dd55ce8603da17357 From 4e11f7c1828e755427d94c7dfd027a34579dfe69 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Nov 2014 18:39:24 -0800 Subject: [PATCH 06/16] Implement a simple recently used cache for HTTP. --- Core/Loaders.cpp | 235 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 231 insertions(+), 4 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index 7f23a02a58..df646c894d 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -60,7 +60,7 @@ private: class HTTPFileLoader : public FileLoader { public: HTTPFileLoader(const std::string &filename); - virtual ~HTTPFileLoader(); + virtual ~HTTPFileLoader() override; virtual bool Exists() override; virtual bool IsDirectory() override; @@ -80,7 +80,6 @@ public: virtual size_t ReadAt(s64 absolutePos, size_t bytes, void *data) override; private: - // TODO: Caching, etc. s64 filesize_; s64 filepos_; Url url_; @@ -89,9 +88,68 @@ private: std::string filename_; }; +class CachingFileLoader : public FileLoader { +public: + CachingFileLoader(FileLoader *backend); + virtual ~CachingFileLoader() override; + + virtual bool Exists() override; + virtual bool IsDirectory() override; + virtual s64 FileSize() override; + virtual std::string Path() const override; + + virtual void Seek(s64 absolutePos) override; + virtual size_t Read(size_t bytes, size_t count, void *data) override { + return ReadAt(filepos_, bytes, count, data); + } + virtual size_t Read(size_t bytes, void *data) override { + return ReadAt(filepos_, bytes, data); + } + virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) override { + return ReadAt(absolutePos, bytes * count, data) / bytes; + } + virtual size_t ReadAt(s64 absolutePos, size_t bytes, void *data) override; + +private: + void InitCache(); + void ShutdownCache(); + size_t ReadFromCache(s64 pos, size_t bytes, void *data); + // Guaranteed to read at least one block into the cache. + void SaveIntoCache(s64 pos, size_t bytes); + void MakeCacheSpaceFor(size_t blocks); + + enum { + BLOCK_SIZE = 65536, + BLOCK_SHIFT = 16, + MAX_BLOCKS_PER_READ = 16, + MAX_BLOCKS_CACHED = 4096, // 256 MB + }; + + s64 filesize_; + s64 filepos_; + FileLoader *backend_; + int exists_; + int isDirectory_; + u64 generation_; + u64 oldestGeneration_; + size_t cacheSize_; + + struct BlockInfo { + u8 *ptr; + u64 generation; + + BlockInfo() : ptr(nullptr), generation(0) { + } + BlockInfo(u8 *p) : ptr(p), generation(0) { + } + }; + + std::map blocks_; +}; + FileLoader *ConstructFileLoader(const std::string &filename) { if (filename.find("http://") == 0 || filename.find("https://") == 0) - return new HTTPFileLoader(filename); + return new CachingFileLoader(new HTTPFileLoader(filename)); return new LocalFileLoader(filename); } @@ -164,7 +222,7 @@ size_t LocalFileLoader::ReadAt(s64 absolutePos, size_t bytes, size_t count, void } HTTPFileLoader::HTTPFileLoader(const std::string &filename) - : filesize_(0), url_(filename), filename_(filename) { + : filesize_(0), filepos_(0), url_(filename), filename_(filename) { if (!client_.Resolve(url_.Host().c_str(), 80)) { return; } @@ -283,6 +341,175 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { return readBytes; } +// Takes ownership of backend. +CachingFileLoader::CachingFileLoader(FileLoader *backend) + : filesize_(0), filepos_(0), backend_(backend), exists_(-1), isDirectory_(-1) { + filesize_ = backend->FileSize(); + if (filesize_ > 0) { + InitCache(); + } +} + +CachingFileLoader::~CachingFileLoader() { + if (filesize_ > 0) { + ShutdownCache(); + } + // Takes ownership. + delete backend_; +} + +bool CachingFileLoader::Exists() { + if (exists_ == -1) { + exists_ = backend_->Exists() ? 1 : 0; + } + return exists_ == 1; +} + +bool CachingFileLoader::IsDirectory() { + if (isDirectory_ == -1) { + isDirectory_ = backend_->IsDirectory() ? 1 : 0; + } + return isDirectory_ == 1; +} + +s64 CachingFileLoader::FileSize() { + return filesize_; +} + +std::string CachingFileLoader::Path() const { + return backend_->Path(); +} + +void CachingFileLoader::Seek(s64 absolutePos) { + filepos_ = absolutePos; +} + +size_t CachingFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { + size_t readSize = ReadFromCache(absolutePos, bytes, data); + // While in case the cache size is too small for the entire read. + while (readSize < bytes) { + SaveIntoCache(absolutePos + readSize, bytes - readSize); + readSize += ReadFromCache(absolutePos + readSize, bytes - readSize, (u8 *)data + readSize); + } + + filepos_ = absolutePos + readSize; + return readSize; +} + +void CachingFileLoader::InitCache() { + cacheSize_ = 0; + oldestGeneration_ = 0; + generation_ = 0; +} + +void CachingFileLoader::ShutdownCache() { + for (auto block : blocks_) { + delete [] block.second.ptr; + } + blocks_.clear(); + cacheSize_ = 0; +} + +size_t CachingFileLoader::ReadFromCache(s64 pos, size_t bytes, void *data) { + s64 cacheStartPos = pos >> BLOCK_SHIFT; + s64 cacheEndPos = (pos + bytes - 1) >> BLOCK_SHIFT; + // TODO: Smarter. + size_t readSize = 0; + size_t offset = (size_t)(pos - (cacheStartPos << BLOCK_SHIFT)); + u8 *p = (u8 *)data; + for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) { + auto block = blocks_.find(i); + if (block == blocks_.end()) { + return readSize; + } + block->second.generation = generation_; + + size_t toRead = std::min(bytes - readSize, (size_t)BLOCK_SIZE - offset); + memcpy(p + readSize, block->second.ptr + offset, toRead); + readSize += toRead; + + // Don't need an offset after the first read. + offset = 0; + } + return readSize; +} + +void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes) { + s64 cacheStartPos = pos >> BLOCK_SHIFT; + s64 cacheEndPos = (pos + bytes - 1) >> BLOCK_SHIFT; + + size_t blocksToRead = 0; + for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) { + auto block = blocks_.find(i); + if (block != blocks_.end()) { + break; + } + ++blocksToRead; + if (blocksToRead >= MAX_BLOCKS_PER_READ) { + break; + } + } + + MakeCacheSpaceFor(blocksToRead); + + if (blocksToRead == 0) { + ERROR_LOG(LOADER, "No blocks to read into cache?"); + } else if (blocksToRead == 1) { + u8 *buf = new u8[BLOCK_SIZE]; + backend_->ReadAt(cacheStartPos << BLOCK_SHIFT, BLOCK_SIZE, buf); + blocks_[cacheStartPos] = BlockInfo(buf); + } else { + u8 *wholeRead = new u8[blocksToRead << BLOCK_SHIFT]; + backend_->ReadAt(cacheStartPos << BLOCK_SHIFT, blocksToRead << BLOCK_SHIFT, wholeRead); + for (size_t i = 0; i < blocksToRead; ++i) { + u8 *buf = new u8[BLOCK_SIZE]; + memcpy(buf, wholeRead + (i << BLOCK_SHIFT), BLOCK_SIZE); + blocks_[cacheStartPos + i] = BlockInfo(buf); + } + delete wholeRead; + } + + cacheSize_ += blocksToRead; + ++generation_; +} + +void CachingFileLoader::MakeCacheSpaceFor(size_t blocks) { + size_t goal = MAX_BLOCKS_CACHED - blocks; + + while (cacheSize_ > goal) { + u64 minGeneration = generation_; + + // We increment the iterator inside because we delete things inside. + for (auto it = blocks_.begin(); it != blocks_.end(); ) { + // Check for the minimum seen generation. + // TODO: Do this smarter? + if (it->second.generation < minGeneration) { + minGeneration = it->second.generation; + } + + if (it->second.generation == oldestGeneration_) { + s64 pos = it->first; + delete it->second.ptr; + blocks_.erase(it); + --cacheSize_; + + // Our iterator is invalid now. Keep going? + if (cacheSize_ > goal) { + // This finds the one at that position. + it = blocks_.lower_bound(pos); + } else { + break; + } + } else { + ++it; + } + } + + // If we didn't find any, update to the lowest we did find. + oldestGeneration_ = minGeneration; + } +} + // TODO : improve, look in the file more IdentifiedFileType Identify_File(FileLoader *fileLoader) { From 49a5394b9389bd9d9dbbcbd69c1150d63f00f206 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Mon, 24 Nov 2014 00:19:13 -0800 Subject: [PATCH 07/16] Oops, don't try to read past end of file. --- Core/Loaders.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index df646c894d..77ea03b3f3 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -286,7 +286,7 @@ void HTTPFileLoader::Seek(s64 absolutePos) { } size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { - s64 absoluteEnd = absolutePos + bytes; + s64 absoluteEnd = std::min(absolutePos + (s64)bytes, filesize_); // TODO: Keepalive, etc. client_.Connect(); @@ -319,7 +319,11 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { if (sscanf(header.c_str(), "Content-Range: bytes %lld-%lld/%lld", &first, &last, &total) >= 2) { if (first == absolutePos && last == absoluteEnd - 1) { supportedResponse = true; + } else { + ERROR_LOG(LOADER, "Unexpected HTTP range: got %lld-%lld, wanted %lld-%lld.", first, last, absolutePos, absoluteEnd - 1); } + } else { + ERROR_LOG(LOADER, "Unexpected HTTP range response: %s", header.c_str()); } } } From 824e8cf17c480f10b8a9a36e2da13acc076a1e11 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Mon, 24 Nov 2014 01:08:30 -0800 Subject: [PATCH 08/16] Add a readahead cache for HTTP. So the only other thing it could do is idle backfill... --- Core/Loaders.cpp | 92 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 9 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index 77ea03b3f3..b546b9d154 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -15,7 +15,10 @@ // Official git repository and contact information can be found at // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. +#include "thread/thread.h" +#include "base/mutex.h" #include "base/stringutil.h" +#include "base/timeutil.h" #include "file/file_util.h" #include "net/http_client.h" #include "net/resolve.h" @@ -115,14 +118,16 @@ private: void ShutdownCache(); size_t ReadFromCache(s64 pos, size_t bytes, void *data); // Guaranteed to read at least one block into the cache. - void SaveIntoCache(s64 pos, size_t bytes); - void MakeCacheSpaceFor(size_t blocks); + void SaveIntoCache(s64 pos, size_t bytes, bool readingAhead = false); + bool MakeCacheSpaceFor(size_t blocks, bool readingAhead); + void StartReadAhead(s64 pos); enum { BLOCK_SIZE = 65536, BLOCK_SHIFT = 16, MAX_BLOCKS_PER_READ = 16, MAX_BLOCKS_CACHED = 4096, // 256 MB + BLOCK_READAHEAD = 4, }; s64 filesize_; @@ -145,6 +150,9 @@ private: }; std::map blocks_; + recursive_mutex blocksMutex_; + mutable recursive_mutex backendMutex_; + bool aheadThread_; }; FileLoader *ConstructFileLoader(const std::string &filename) { @@ -347,7 +355,7 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { // Takes ownership of backend. CachingFileLoader::CachingFileLoader(FileLoader *backend) - : filesize_(0), filepos_(0), backend_(backend), exists_(-1), isDirectory_(-1) { + : filesize_(0), filepos_(0), backend_(backend), exists_(-1), isDirectory_(-1), aheadThread_(false) { filesize_ = backend->FileSize(); if (filesize_ > 0) { InitCache(); @@ -364,6 +372,7 @@ CachingFileLoader::~CachingFileLoader() { bool CachingFileLoader::Exists() { if (exists_ == -1) { + lock_guard guard(backendMutex_); exists_ = backend_->Exists() ? 1 : 0; } return exists_ == 1; @@ -371,6 +380,7 @@ bool CachingFileLoader::Exists() { bool CachingFileLoader::IsDirectory() { if (isDirectory_ == -1) { + lock_guard guard(backendMutex_); isDirectory_ = backend_->IsDirectory() ? 1 : 0; } return isDirectory_ == 1; @@ -381,6 +391,7 @@ s64 CachingFileLoader::FileSize() { } std::string CachingFileLoader::Path() const { + lock_guard guard(backendMutex_); return backend_->Path(); } @@ -396,6 +407,8 @@ size_t CachingFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { readSize += ReadFromCache(absolutePos + readSize, bytes - readSize, (u8 *)data + readSize); } + StartReadAhead(absolutePos + readSize); + filepos_ = absolutePos + readSize; return readSize; } @@ -407,6 +420,14 @@ void CachingFileLoader::InitCache() { } void CachingFileLoader::ShutdownCache() { + // TODO: Maybe add some hint that deletion is coming soon? + // We can't delete while the thread is running, so have to wait. + // This should only happen from the menu. + while (aheadThread_) { + sleep_ms(1); + } + + lock_guard guard(blocksMutex_); for (auto block : blocks_) { delete [] block.second.ptr; } @@ -421,6 +442,8 @@ size_t CachingFileLoader::ReadFromCache(s64 pos, size_t bytes, void *data) { size_t readSize = 0; size_t offset = (size_t)(pos - (cacheStartPos << BLOCK_SHIFT)); u8 *p = (u8 *)data; + + lock_guard guard(blocksMutex_); for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) { auto block = blocks_.find(i); if (block == blocks_.end()) { @@ -438,10 +461,11 @@ size_t CachingFileLoader::ReadFromCache(s64 pos, size_t bytes, void *data) { return readSize; } -void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes) { +void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes, bool readingAhead) { s64 cacheStartPos = pos >> BLOCK_SHIFT; s64 cacheEndPos = (pos + bytes - 1) >> BLOCK_SHIFT; + lock_guard guard(blocksMutex_); size_t blocksToRead = 0; for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) { auto block = blocks_.find(i); @@ -454,17 +478,29 @@ void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes) { } } - MakeCacheSpaceFor(blocksToRead); + if (!MakeCacheSpaceFor(blocksToRead, readingAhead) || blocksToRead == 0) { + return; + } + + if (blocksToRead == 1) { + blocksMutex_.unlock(); - if (blocksToRead == 0) { - ERROR_LOG(LOADER, "No blocks to read into cache?"); - } else if (blocksToRead == 1) { u8 *buf = new u8[BLOCK_SIZE]; + backendMutex_.lock(); backend_->ReadAt(cacheStartPos << BLOCK_SHIFT, BLOCK_SIZE, buf); + backendMutex_.unlock(); + + blocksMutex_.lock(); blocks_[cacheStartPos] = BlockInfo(buf); } else { + blocksMutex_.unlock(); + u8 *wholeRead = new u8[blocksToRead << BLOCK_SHIFT]; + backendMutex_.lock(); backend_->ReadAt(cacheStartPos << BLOCK_SHIFT, blocksToRead << BLOCK_SHIFT, wholeRead); + backendMutex_.unlock(); + + blocksMutex_.lock(); for (size_t i = 0; i < blocksToRead; ++i) { u8 *buf = new u8[BLOCK_SIZE]; memcpy(buf, wholeRead + (i << BLOCK_SHIFT), BLOCK_SIZE); @@ -477,9 +513,14 @@ void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes) { ++generation_; } -void CachingFileLoader::MakeCacheSpaceFor(size_t blocks) { +bool CachingFileLoader::MakeCacheSpaceFor(size_t blocks, bool readingAhead) { size_t goal = MAX_BLOCKS_CACHED - blocks; + if (readingAhead && cacheSize_ > goal) { + return false; + } + + lock_guard guard(blocksMutex_); while (cacheSize_ > goal) { u64 minGeneration = generation_; @@ -512,6 +553,39 @@ void CachingFileLoader::MakeCacheSpaceFor(size_t blocks) { // If we didn't find any, update to the lowest we did find. oldestGeneration_ = minGeneration; } + + return true; +} + +void CachingFileLoader::StartReadAhead(s64 pos) { + lock_guard guard(blocksMutex_); + if (aheadThread_) { + // Already going. + return; + } + if (cacheSize_ + BLOCK_READAHEAD > MAX_BLOCKS_CACHED) { + // Not enough space to readahead. + return; + } + + aheadThread_ = true; + std::thread th([this, pos] { + lock_guard guard(blocksMutex_); + s64 cacheStartPos = pos >> BLOCK_SHIFT; + s64 cacheEndPos = cacheStartPos + BLOCK_READAHEAD - 1; + + for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) { + auto block = blocks_.find(i); + if (block == blocks_.end()) { + blocksMutex_.unlock(); + SaveIntoCache(i << BLOCK_SHIFT, BLOCK_SIZE * BLOCK_READAHEAD, true); + break; + } + } + + aheadThread_ = false; + }); + th.detach(); } // TODO : improve, look in the file more From c259c6708e66d3a0a7fa367f2056113b8af7bb93 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Mon, 24 Nov 2014 07:50:24 -0800 Subject: [PATCH 09/16] Update native with HTTP fix. --- native | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native b/native index 73d5ad3b5c..c85fb55502 160000 --- a/native +++ b/native @@ -1 +1 @@ -Subproject commit 73d5ad3b5c937afceb95569dd55ce8603da17357 +Subproject commit c85fb55502feaea7c8a36004957229df4dc2b334 From 27eb38df8d6ee5e3660d60825459bc389598b824 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Mon, 24 Nov 2014 14:08:21 -0800 Subject: [PATCH 10/16] Fix a possible leak. Can definitely see this happening. --- Core/Loaders.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index b546b9d154..08e42a7b4d 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -491,7 +491,13 @@ void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes, bool readingAhead) backendMutex_.unlock(); blocksMutex_.lock(); - blocks_[cacheStartPos] = BlockInfo(buf); + // While blocksMutex_ was unlocked, another thread may have read. + // If so, free the one we just read. + if (blocks_.find(cacheStartPos) == blocks_.end()) { + blocks_[cacheStartPos] = BlockInfo(buf); + } else { + delete [] buf; + } } else { blocksMutex_.unlock(); @@ -502,6 +508,10 @@ void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes, bool readingAhead) blocksMutex_.lock(); for (size_t i = 0; i < blocksToRead; ++i) { + if (blocks_.find(cacheStartPos + i) != blocks_.end()) { + // Written while we were busy, just skip it. Keep the existing block. + continue; + } u8 *buf = new u8[BLOCK_SIZE]; memcpy(buf, wholeRead + (i << BLOCK_SHIFT), BLOCK_SIZE); blocks_[cacheStartPos + i] = BlockInfo(buf); From df5020c439b58fb5d8dfefdb8118e5ec9ca19df9 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Mon, 24 Nov 2014 14:10:48 -0800 Subject: [PATCH 11/16] Treat generation 0 a bit specially. I guess we could always check gen 0 first, that'd be readahead/backfill. --- Core/Loaders.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index 08e42a7b4d..5729f5c92c 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -538,11 +538,12 @@ bool CachingFileLoader::MakeCacheSpaceFor(size_t blocks, bool readingAhead) { for (auto it = blocks_.begin(); it != blocks_.end(); ) { // Check for the minimum seen generation. // TODO: Do this smarter? - if (it->second.generation < minGeneration) { + if (it->second.generation != 0 && it->second.generation < minGeneration) { minGeneration = it->second.generation; } - if (it->second.generation == oldestGeneration_) { + // 0 means it was never used yet or was the first read (e.g. block descriptor.) + if (it->second.generation == oldestGeneration_ || it->second.generation == 0) { s64 pos = it->first; delete it->second.ptr; blocks_.erase(it); From 219d546b7946acb80b49a53048754938177fa93c Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 00:30:48 -0800 Subject: [PATCH 12/16] Handle case in HTTP headers better. --- Core/Loaders.cpp | 8 +++++--- native | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index 5729f5c92c..07181e1796 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -252,7 +252,7 @@ HTTPFileLoader::HTTPFileLoader(const std::string &filename) // TODO: Expire cache via ETag, etc. for (std::string header : responseHeaders) { - if (startsWith(header, "Content-Length:")) { + if (startsWithNoCase(header, "Content-Length:")) { size_t size_pos = header.find_first_of(' '); if (size_pos != header.npos) { size_pos = header.find_first_not_of(' ', size_pos); @@ -321,10 +321,12 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { // We don't support multipart/byteranges responses. bool supportedResponse = false; for (std::string header : responseHeaders) { - if (startsWith(header, "Content-Range:")) { + if (startsWithNoCase(header, "Content-Range:")) { // TODO: More correctness. Whitespace can be missing or different. s64 first = -1, last = -1, total = -1; - if (sscanf(header.c_str(), "Content-Range: bytes %lld-%lld/%lld", &first, &last, &total) >= 2) { + std::string lowerHeader = header; + std::transform(lowerHeader.begin(), lowerHeader.end(), lowerHeader.begin(), tolower); + if (sscanf(lowerHeader.c_str(), "content-range: bytes %lld-%lld/%lld", &first, &last, &total) >= 2) { if (first == absolutePos && last == absoluteEnd - 1) { supportedResponse = true; } else { diff --git a/native b/native index c85fb55502..3de556b248 160000 --- a/native +++ b/native @@ -1 +1 @@ -Subproject commit c85fb55502feaea7c8a36004957229df4dc2b334 +Subproject commit 3de556b248f0c83405451e46ce5e56168d0ed0d1 From 6d9ff8fac47ae8861a622c4cb729b0f25ded72ef Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 00:33:23 -0800 Subject: [PATCH 13/16] Fix disconnect order. --- Core/Loaders.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index 07181e1796..f40a6b3d12 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -342,13 +342,13 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { Buffer output; client_.ReadResponseEntity(&readbuf, responseHeaders, &output); + client_.Disconnect(); + if (!supportedResponse) { ERROR_LOG(LOADER, "HTTP server did not respond with the range we wanted."); return 0; } - client_.Disconnect(); - size_t readBytes = output.size(); output.Take(readBytes, (char *)data); filepos_ = absolutePos + readBytes; From 99b422aa0aec4cbc01b3f1da83338342409ef11c Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 08:59:29 -0800 Subject: [PATCH 14/16] Add more verbose logging in case of error. --- Core/Loaders.cpp | 19 ++++++++++++++++++- native | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index f40a6b3d12..bd6f6ffc70 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -247,10 +247,12 @@ HTTPFileLoader::HTTPFileLoader(const std::string &filename) int code = client_.ReadResponseHeaders(&readbuf, responseHeaders); if (code != 200) { // Leave size at 0, invalid. + ERROR_LOG(LOADER, "HTTP request failed, got %03d for %s", code, filename.c_str()); return; } // TODO: Expire cache via ETag, etc. + bool acceptsRange = false; for (std::string header : responseHeaders) { if (startsWithNoCase(header, "Content-Length:")) { size_t size_pos = header.find_first_of(' '); @@ -261,10 +263,25 @@ HTTPFileLoader::HTTPFileLoader(const std::string &filename) filesize_ = atoll(&header[size_pos]); } } + if (startsWithNoCase(header, "Accept-Ranges:")) { + std::string lowerHeader = header; + std::transform(lowerHeader.begin(), lowerHeader.end(), lowerHeader.begin(), tolower); + // TODO: Delimited. + if (lowerHeader.find("bytes") != lowerHeader.npos) { + acceptsRange = true; + } + } } client_.Disconnect(); + if (!acceptsRange) { + WARN_LOG(LOADER, "HTTP server did not advertise support for range requests."); + } + if (filesize_ == 0) { + ERROR_LOG(LOADER, "Could not determine file size for %s", filename.c_str()); + } + // If we didn't end up with a filesize_ (e.g. chunked response), give up. File invalid. } @@ -313,7 +330,7 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { std::vector responseHeaders; int code = client_.ReadResponseHeaders(&readbuf, responseHeaders); if (code != 206) { - ERROR_LOG(LOADER, "HTTP server does not support range requests."); + ERROR_LOG(LOADER, "HTTP server did not respond with range, received code=%03d", code); return 0; } diff --git a/native b/native index 3de556b248..1150598ce4 160000 --- a/native +++ b/native @@ -1 +1 @@ -Subproject commit 3de556b248f0c83405451e46ce5e56168d0ed0d1 +Subproject commit 1150598ce463c99fbb08ba4177a7937692fe9bc0 From 53e6d49568cb8fb07c1c3aee7ab9a1c04f4fb824 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 09:16:17 -0800 Subject: [PATCH 15/16] Shortcut a read outside the file. This happened to me. --- Core/Loaders.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index bd6f6ffc70..df81db2204 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -312,6 +312,10 @@ void HTTPFileLoader::Seek(s64 absolutePos) { size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { s64 absoluteEnd = std::min(absolutePos + (s64)bytes, filesize_); + if (absolutePos >= filesize_ || bytes == 0) { + // Read outside of the file or no read at all, just fail immediately. + return 0; + } // TODO: Keepalive, etc. client_.Connect(); From 0551773599bc172423984c50d4d08a2b4955861f Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 09:16:47 -0800 Subject: [PATCH 16/16] Add an interface to retry HTTP requests. They could fail / be spotty in some situations. A little retry can go a long way. --- Core/Loaders.cpp | 86 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index df81db2204..adb3cad1c8 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -91,6 +91,37 @@ private: std::string filename_; }; +class RetryingFileLoader : public FileLoader { +public: + RetryingFileLoader(FileLoader *backend); + virtual ~RetryingFileLoader() override; + + virtual bool Exists() override; + virtual bool IsDirectory() override; + virtual s64 FileSize() override; + virtual std::string Path() const override; + + virtual void Seek(s64 absolutePos) override; + virtual size_t Read(size_t bytes, size_t count, void *data) override { + return ReadAt(filepos_, bytes, count, data); + } + virtual size_t Read(size_t bytes, void *data) override { + return ReadAt(filepos_, bytes, data); + } + virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data) override { + return ReadAt(absolutePos, bytes * count, data) / bytes; + } + virtual size_t ReadAt(s64 absolutePos, size_t bytes, void *data) override; + +private: + enum { + MAX_RETRIES = 3, + }; + + s64 filepos_; + FileLoader *backend_; +}; + class CachingFileLoader : public FileLoader { public: CachingFileLoader(FileLoader *backend); @@ -157,7 +188,7 @@ private: FileLoader *ConstructFileLoader(const std::string &filename) { if (filename.find("http://") == 0 || filename.find("https://") == 0) - return new CachingFileLoader(new HTTPFileLoader(filename)); + return new CachingFileLoader(new RetryingFileLoader(new HTTPFileLoader(filename))); return new LocalFileLoader(filename); } @@ -622,6 +653,59 @@ void CachingFileLoader::StartReadAhead(s64 pos) { th.detach(); } +// Takes ownership of backend. +RetryingFileLoader::RetryingFileLoader(FileLoader *backend) + : filepos_(0), backend_(backend) { +} + +RetryingFileLoader::~RetryingFileLoader() { + // Takes ownership. + delete backend_; +} + +bool RetryingFileLoader::Exists() { + if (!backend_->Exists()) { + // Retry once, immediately. + return backend_->Exists(); + } + return true; +} + +bool RetryingFileLoader::IsDirectory() { + // Can't tell if it's an error either way. + return backend_->IsDirectory(); +} + +s64 RetryingFileLoader::FileSize() { + s64 filesize = backend_->FileSize(); + if (filesize == 0) { + return backend_->FileSize(); + } + return filesize; +} + +std::string RetryingFileLoader::Path() const { + return backend_->Path(); +} + +void RetryingFileLoader::Seek(s64 absolutePos) { + filepos_ = absolutePos; +} + +size_t RetryingFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data) { + size_t readSize = backend_->ReadAt(absolutePos, bytes, data); + + int retries = 0; + while (readSize < bytes && retries < MAX_RETRIES) { + u8 *p = (u8 *)data; + readSize += backend_->ReadAt(absolutePos + readSize, bytes - readSize, p + readSize); + ++retries; + } + + filepos_ = absolutePos + readSize; + return readSize; +} + // TODO : improve, look in the file more IdentifiedFileType Identify_File(FileLoader *fileLoader) {