From 97cf5f8d81ec8221e2c074d2dee553efe4e48e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Mon, 6 Mar 2023 22:46:05 +0100 Subject: [PATCH] More renaming, initial implementation of manual file access for both VFS backends --- Common/File/VFS/DirectoryReader.cpp | 44 +++++++---- Common/File/VFS/DirectoryReader.h | 13 ++-- Common/File/VFS/VFS.h | 17 +++-- Common/File/VFS/ZipFileReader.cpp | 114 ++++++++++++++++++++++------ Common/File/VFS/ZipFileReader.h | 18 +++-- 5 files changed, 148 insertions(+), 58 deletions(-) diff --git a/Common/File/VFS/DirectoryReader.cpp b/Common/File/VFS/DirectoryReader.cpp index 332bb0bb48..e4da6a712d 100644 --- a/Common/File/VFS/DirectoryReader.cpp +++ b/Common/File/VFS/DirectoryReader.cpp @@ -41,27 +41,45 @@ public: }; VFSFileReference *DirectoryReader::GetFile(const char *path) { - return nullptr; + DirectoryReaderFileReference *reference = new DirectoryReaderFileReference(); + reference->path = path_ / path; + return reference; } -void DirectoryReader::ReleaseFile(VFSFileReference *reference) { - DirectoryReaderFileReference *file = (DirectoryReaderFileReference *)reference; +bool DirectoryReader::GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) { + DirectoryReaderFileReference *reference = (DirectoryReaderFileReference *)vfsReference; + return File::GetFileInfo(reference->path, fileInfo); } -VFSOpenFile *DirectoryReader::OpenFileForRead(VFSFileReference *reference) { - DirectoryReaderFileReference *file = (DirectoryReaderFileReference *)reference; - return nullptr; +void DirectoryReader::ReleaseFile(VFSFileReference *vfsReference) { + DirectoryReaderFileReference *reference = (DirectoryReaderFileReference *)vfsReference; + delete reference; } -void DirectoryReader::Rewind(VFSOpenFile *openFile) { - DirectoryReaderOpenFile *file = (DirectoryReaderOpenFile *)openFile; +VFSOpenFile *DirectoryReader::OpenFileForRead(VFSFileReference *vfsReference) { + DirectoryReaderFileReference *reference = (DirectoryReaderFileReference *)vfsReference; + FILE *file = File::OpenCFile(reference->path, "rb"); + if (!file) { + return nullptr; + } + + DirectoryReaderOpenFile *openFile = new DirectoryReaderOpenFile(); + openFile->file = file; + return openFile; } -size_t DirectoryReader::Read(VFSOpenFile *openFile, uint8_t *buffer, size_t length) { - DirectoryReaderOpenFile *file = (DirectoryReaderOpenFile *)openFile; - return 0; +void DirectoryReader::Rewind(VFSOpenFile *vfsOpenFile) { + DirectoryReaderOpenFile *openFile = (DirectoryReaderOpenFile *)vfsOpenFile; + fseek(openFile->file, 0, SEEK_SET); } -void DirectoryReader::CloseFile(VFSOpenFile *openFile) { - DirectoryReaderOpenFile *file = (DirectoryReaderOpenFile *)openFile; +size_t DirectoryReader::Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) { + DirectoryReaderOpenFile *openFile = (DirectoryReaderOpenFile *)vfsOpenFile; + return fread(buffer, 1, length, openFile->file); +} + +void DirectoryReader::CloseFile(VFSOpenFile *vfsOpenFile) { + DirectoryReaderOpenFile *openFile = (DirectoryReaderOpenFile *)vfsOpenFile; + fclose(openFile->file); + delete openFile; } diff --git a/Common/File/VFS/DirectoryReader.h b/Common/File/VFS/DirectoryReader.h index 00cb8f7878..cc6d971371 100644 --- a/Common/File/VFS/DirectoryReader.h +++ b/Common/File/VFS/DirectoryReader.h @@ -10,13 +10,14 @@ public: // use delete[] on the returned value. uint8_t *ReadFile(const char *path, size_t *size) override; - virtual VFSFileReference *GetFile(const char *path) override; - virtual void ReleaseFile(VFSFileReference *reference) override; + VFSFileReference *GetFile(const char *path) override; + bool GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) override; + void ReleaseFile(VFSFileReference *vfsReference) override; - virtual VFSOpenFile *OpenFileForRead(VFSFileReference *reference) override; - virtual void Rewind(VFSOpenFile *openFile) override; - virtual size_t Read(VFSOpenFile *openFile, uint8_t *buffer, size_t length) override; - virtual void CloseFile(VFSOpenFile *openFile) override; + VFSOpenFile *OpenFileForRead(VFSFileReference *vfsReference) override; + void Rewind(VFSOpenFile *vfsOpenFile) override; + size_t Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) override; + void CloseFile(VFSOpenFile *vfsOpenFile) override; bool GetFileListing(const char *path, std::vector *listing, const char *filter) override; bool GetFileInfo(const char *path, File::FileInfo *info) override; diff --git a/Common/File/VFS/VFS.h b/Common/File/VFS/VFS.h index 6c94bfbecb..22a242c903 100644 --- a/Common/File/VFS/VFS.h +++ b/Common/File/VFS/VFS.h @@ -30,13 +30,14 @@ public: virtual ~VFSOpenFile() {} }; -// Common inteface parts between VFSBackend and VFS. +// Common interface parts between VFSBackend and VFS. // Sometimes you don't need the VFS multiplexing and only have a VFSBackend *, sometimes you do need it, // and it would be cool to be able to use the same interface, like when loading INI files. class VFSInterface { public: virtual ~VFSInterface() {} virtual uint8_t *ReadFile(const char *path, size_t *size) = 0; + virtual bool GetFileListing(const char *path, std::vector *listing, const char *filter = nullptr) = 0; }; class VFSBackend : public VFSInterface { @@ -44,15 +45,15 @@ public: // use delete[] to release the returned memory. virtual VFSFileReference *GetFile(const char *path) = 0; - virtual void ReleaseFile(VFSFileReference *file) = 0; + virtual bool GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) = 0; + virtual void ReleaseFile(VFSFileReference *vfsReference) = 0; - virtual VFSOpenFile *OpenFileForRead(VFSFileReference *reference) = 0; - virtual void Rewind(VFSOpenFile *file) = 0; - virtual size_t Read(VFSOpenFile *file, uint8_t *buffer, size_t length) = 0; - virtual void CloseFile(VFSOpenFile *file) = 0; + virtual VFSOpenFile *OpenFileForRead(VFSFileReference *vfsReference) = 0; + virtual void Rewind(VFSOpenFile *vfsOpenFile) = 0; + virtual size_t Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) = 0; + virtual void CloseFile(VFSOpenFile *vfsOpenFile) = 0; // Filter support is optional but nice to have - virtual bool GetFileListing(const char *path, std::vector *listing, const char *filter = 0) = 0; virtual bool GetFileInfo(const char *path, File::FileInfo *info) = 0; virtual std::string toString() const = 0; }; @@ -67,8 +68,8 @@ public: // Always allocates an extra zero byte at the end, so that it // can be used for text like shader sources. uint8_t *ReadFile(const char *filename, size_t *size) override; - bool GetFileListing(const char *path, std::vector *listing, const char *filter = 0); bool GetFileInfo(const char *filename, File::FileInfo *fileInfo); + bool GetFileListing(const char *path, std::vector *listing, const char *filter = nullptr) override; private: struct VFSEntry { diff --git a/Common/File/VFS/ZipFileReader.cpp b/Common/File/VFS/ZipFileReader.cpp index 2c63a9c4db..1ba7e5bf9f 100644 --- a/Common/File/VFS/ZipFileReader.cpp +++ b/Common/File/VFS/ZipFileReader.cpp @@ -34,18 +34,29 @@ static uint8_t *ReadFromZip(zip *archive, const char* filename, size_t *size) { return contents; } -ZipFileReader::ZipFileReader(const Path &zipFile, const char *inZipPath) { +ZipFileReader *ZipFileReader::Create(const Path &zipFile, const char *inZipPath) { int error = 0; + zip *zip_file; if (zipFile.Type() == PathType::CONTENT_URI) { int fd = File::OpenFD(zipFile, File::OPEN_READ); - zip_file_ = zip_fdopen(fd, 0, &error); + if (!fd) { + ERROR_LOG(IO, "Failed to open FD for %s as zip file", zipFile.c_str()); + return nullptr; + } + zip_file = zip_fdopen(fd, 0, &error); } else { - zip_file_ = zip_open(zipFile.c_str(), 0, &error); + zip_file = zip_open(zipFile.c_str(), 0, &error); } - truncate_cpy(inZipPath_, inZipPath); - if (!zip_file_) { + + if (!zip_file) { ERROR_LOG(IO, "Failed to open %s as a zip file", zipFile.c_str()); + return nullptr; } + + ZipFileReader *reader = new ZipFileReader(); + reader->zip_file_ = zip_file; + truncate_cpy(reader->inZipPath_, inZipPath); + return reader; } ZipFileReader::~ZipFileReader() { @@ -78,6 +89,7 @@ bool ZipFileReader::GetFileListing(const char *orig_path, std::vectorexists = false; - info->size = 0; - return false; + + { + std::lock_guard guard(lock_); + if (0 != zip_stat(zip_file_, temp_path, ZIP_FL_NOCASE | ZIP_FL_UNCHANGED, &zstat)) { + // ZIP files do not have real directories, so we'll end up here if we + // try to stat one. For now that's fine. + info->exists = false; + info->size = 0; + return false; + } } info->fullName = Path(path); @@ -167,34 +183,84 @@ bool ZipFileReader::GetFileInfo(const char *path, File::FileInfo *info) { class ZipFileReaderFileReference : public VFSFileReference { public: + int zi; }; class ZipFileReaderOpenFile : public VFSOpenFile { public: + ZipFileReaderFileReference *reference; + zip_file_t *zf; }; +static constexpr zip_uint64_t INVALID_ZIP_SIZE = 0xFFFFFFFFFFFFFFFFULL; + VFSFileReference *ZipFileReader::GetFile(const char *path) { - return nullptr; + std::lock_guard guard(lock_); + int zi = zip_name_locate(zip_file_, path, ZIP_FL_NOCASE); + if (zi < 0) { + // Not found. + return nullptr; + } + ZipFileReaderFileReference *ref = new ZipFileReaderFileReference(); + ref->zi = zi; + return ref; } -void ZipFileReader::ReleaseFile(VFSFileReference *reference) { - ZipFileReaderFileReference *file = (ZipFileReaderFileReference *)reference; +bool ZipFileReader::GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) { + ZipFileReaderFileReference *reference = (ZipFileReaderFileReference *)vfsReference; + // If you crash here, you called this while having the lock held by having the file open. + // Don't do that, check the info before you open the file. + std::lock_guard guard(lock_); + zip_stat_t zstat; + if (zip_stat_index(zip_file_, reference->zi, 0, &zstat) != 0) + return false; + *fileInfo = File::FileInfo{}; + fileInfo->size = 0; + if (zstat.valid & ZIP_STAT_SIZE) + fileInfo->size = zstat.size; + return zstat.size; } -VFSOpenFile *ZipFileReader::OpenFileForRead(VFSFileReference *reference) { - ZipFileReaderFileReference *file = (ZipFileReaderFileReference *)reference; - return nullptr; +void ZipFileReader::ReleaseFile(VFSFileReference *vfsReference) { + ZipFileReaderFileReference *reference = (ZipFileReaderFileReference *)vfsReference; + // Don't do anything other than deleting it. + delete reference; } -void ZipFileReader::Rewind(VFSOpenFile *openFile) { - ZipFileReaderOpenFile *file = (ZipFileReaderOpenFile *)openFile; +VFSOpenFile *ZipFileReader::OpenFileForRead(VFSFileReference *vfsReference) { + ZipFileReaderFileReference *reference = (ZipFileReaderFileReference *)vfsReference; + ZipFileReaderOpenFile *openFile = new ZipFileReaderOpenFile(); + openFile->reference = reference; + // We only allow one file to be open for read concurrently. It's possible that this can be improved, + // especially if we only access by index like this. + lock_.lock(); + openFile->zf = zip_fopen_index(zip_file_, reference->zi, 0); + + if (!openFile->zf) { + WARN_LOG(G3D, "File with index %d not found in zip", reference->zi); + lock_.unlock(); + return nullptr; + } + + return openFile; } -size_t ZipFileReader::Read(VFSOpenFile *openFile, uint8_t *buffer, size_t length) { - ZipFileReaderOpenFile *file = (ZipFileReaderOpenFile *)openFile; - return 0; +void ZipFileReader::Rewind(VFSOpenFile *vfsOpenFile) { + ZipFileReaderOpenFile *openFile = (ZipFileReaderOpenFile *)vfsOpenFile; + // Close and re-open. + zip_fclose(openFile->zf); + openFile->zf = zip_fopen_index(zip_file_, openFile->reference->zi, 0); } -void ZipFileReader::CloseFile(VFSOpenFile *openFile) { - ZipFileReaderOpenFile *file = (ZipFileReaderOpenFile *)openFile; +size_t ZipFileReader::Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) { + ZipFileReaderOpenFile *file = (ZipFileReaderOpenFile *)vfsOpenFile; + return zip_fread(file->zf, buffer, length); +} + +void ZipFileReader::CloseFile(VFSOpenFile *vfsOpenFile) { + ZipFileReaderOpenFile *file = (ZipFileReaderOpenFile *)vfsOpenFile; + _dbg_assert_(file->zf != nullptr); + zip_fclose(file->zf); + lock_.unlock(); + delete file; } diff --git a/Common/File/VFS/ZipFileReader.h b/Common/File/VFS/ZipFileReader.h index 841491fb44..e3c50b38bf 100644 --- a/Common/File/VFS/ZipFileReader.h +++ b/Common/File/VFS/ZipFileReader.h @@ -16,18 +16,22 @@ class ZipFileReader : public VFSBackend { public: - ZipFileReader(const Path &zipFile, const char *inZipPath); + static ZipFileReader *Create(const Path &zipFile, const char *inZipPath); ~ZipFileReader(); + + bool IsValid() const { return zip_file_ != nullptr; } + // use delete[] on the returned value. uint8_t *ReadFile(const char *path, size_t *size) override; VFSFileReference *GetFile(const char *path) override; - void ReleaseFile(VFSFileReference *reference) override; + bool GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) override; + void ReleaseFile(VFSFileReference *vfsReference) override; - VFSOpenFile *OpenFileForRead(VFSFileReference *reference) override; - void Rewind(VFSOpenFile *openFile) override; - size_t Read(VFSOpenFile *openFile, uint8_t *buffer, size_t length) override; - void CloseFile(VFSOpenFile *openFile) override; + VFSOpenFile *OpenFileForRead(VFSFileReference *vfsReference) override; + void Rewind(VFSOpenFile *vfsOpenFile) override; + size_t Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) override; + void CloseFile(VFSOpenFile *vfsOpenFile) override; bool GetFileListing(const char *path, std::vector *listing, const char *filter) override; bool GetFileInfo(const char *path, File::FileInfo *info) override; @@ -38,7 +42,7 @@ public: private: void GetZipListings(const char *path, std::set &files, std::set &directories); - zip *zip_file_; + zip *zip_file_ = nullptr; std::mutex lock_; char inZipPath_[256]; };