Add a ZipFileLoader, which can let us load any single-file file type from a zip.

Useful for loading framedumps from github without manually having to
unzip each one, for example.
This commit is contained in:
Henrik Rydgård
2025-04-12 20:21:35 +02:00
parent 200edfaa96
commit fffbed53bc
19 changed files with 317 additions and 5 deletions
+2
View File
@@ -2378,6 +2378,8 @@ add_library(${CoreLibName} ${CoreLinkType}
Core/FileLoaders/RamCachingFileLoader.h
Core/FileLoaders/RetryingFileLoader.cpp
Core/FileLoaders/RetryingFileLoader.h
Core/FileLoaders/ZipFileLoader.cpp
Core/FileLoaders/ZipFileLoader.h
Core/MIPS/MIPS.cpp
Core/MIPS/MIPS.h
Core/MIPS/MIPSAnalyst.cpp
+9
View File
@@ -266,6 +266,15 @@ std::string_view KeepAfterLast(std::string_view s, char c) {
}
}
std::string_view KeepIncludingLast(std::string_view s, char c) {
size_t pos = s.rfind(c);
if (pos != std::string_view::npos) {
return s.substr(pos);
} else {
return s;
}
}
void SkipSpace(const char **ptr) {
while (**ptr && isspace(**ptr)) {
(*ptr)++;
+1
View File
@@ -99,6 +99,7 @@ std::string StripSpaces(const std::string &s);
std::string StripQuotes(const std::string &s);
std::string_view KeepAfterLast(std::string_view s, char c);
std::string_view KeepIncludingLast(std::string_view s, char c);
std::string_view StripSpaces(std::string_view s);
std::string_view StripQuotes(std::string_view s);
+2
View File
@@ -565,6 +565,7 @@
<ClCompile Include="Debugger\WebSocket\SteppingSubscriber.cpp" />
<ClCompile Include="Debugger\WebSocket\WebSocketUtils.cpp" />
<ClCompile Include="Dialog\PSPOskConstants.cpp" />
<ClCompile Include="FileLoaders\ZipFileLoader.cpp" />
<ClCompile Include="FileSystems\BlobFileSystem.cpp" />
<ClCompile Include="FrameTiming.cpp" />
<ClCompile Include="HLE\AtracCtx.cpp" />
@@ -1186,6 +1187,7 @@
<ClInclude Include="Debugger\WebSocket\LogBroadcaster.h" />
<ClInclude Include="Debugger\WebSocket\SteppingBroadcaster.h" />
<ClInclude Include="Dialog\PSPOskConstants.h" />
<ClInclude Include="FileLoaders\ZipFileLoader.h" />
<ClInclude Include="FileSystems\BlobFileSystem.h" />
<ClInclude Include="FrameTiming.h" />
<ClInclude Include="HLE\AtracCtx.h" />
+6
View File
@@ -1351,6 +1351,9 @@
<ClCompile Include="HLE\sceReg.cpp">
<Filter>HLE\Libraries</Filter>
</ClCompile>
<ClCompile Include="FileLoaders\ZipFileLoader.cpp">
<Filter>FileLoaders</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="ELF\ElfReader.h">
@@ -2184,6 +2187,9 @@
<ClInclude Include="HLE\sceReg.h">
<Filter>HLE\Libraries</Filter>
</ClInclude>
<ClInclude Include="FileLoaders\ZipFileLoader.h">
<Filter>FileLoaders</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\LICENSE.TXT" />
+1
View File
@@ -21,6 +21,7 @@
#include "Common/CommonTypes.h"
#include "Common/File/Path.h"
#include "Common/StringUtils.h"
#include "Core/Loaders.h"
#ifdef _WIN32
+154
View File
@@ -0,0 +1,154 @@
#pragma once
#include <ext/libzip/zip.h>
#include "Core/FileLoaders/LocalFileLoader.h"
#include "Core/FileLoaders/ZipFileLoader.h"
ZipFileLoader::ZipFileLoader(FileLoader *sourceLoader)
: ProxiedFileLoader(sourceLoader), zipArchive_(nullptr) {
if (!backend_ || !backend_->Exists() || backend_->IsDirectory()) {
// bad
}
zip_error_t error{};
zip_source_t* zipSource = zip_source_function_create([](void* userdata, void* data, zip_uint64_t len, zip_source_cmd_t cmd) -> zip_int64_t {
ZipFileLoader *loader = (ZipFileLoader *)userdata;
return loader->ZipSourceCallback(data, len, cmd);
}, this, &error);
if (!zipSource) {
ERROR_LOG(Log::IO, "Failed to create ZIP source: %s", zip_error_strerror(&error));
return;
}
zipArchive_ = zip_open_from_source(zipSource, ZIP_RDONLY, &error);
if (!zipArchive_) {
ERROR_LOG(Log::IO, "Failed to open ZIP archive: %s", zip_error_strerror(&error));
zip_source_free(zipSource);
}
}
ZipFileLoader::~ZipFileLoader() {
if (dataFile_) {
zip_fclose(dataFile_);
}
if (zipArchive_) {
zip_discard(zipArchive_);
}
if (data_) {
free(data_);
}
}
bool ZipFileLoader::Initialize(int fileIndex) {
_dbg_assert_(!data_);
struct zip_stat zstat;
int retval = zip_stat_index(zipArchive_, fileIndex, ZIP_FL_NOCASE | ZIP_FL_UNCHANGED, &zstat);
if (retval < 0) {
return false;
}
const char *name = zip_get_name(zipArchive_, fileIndex, ZIP_FL_NOCASE | ZIP_FL_UNCHANGED);
fileExtension_ = KeepIncludingLast(name, '.');
_dbg_assert_(zstat.index == fileIndex);
dataFileSize_ = zstat.size;
dataFile_ = zip_fopen_index(zipArchive_, zstat.index, ZIP_FL_UNCHANGED);
data_ = (u8 *)malloc(dataFileSize_);
return data_ != nullptr;
}
size_t ZipFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data, Flags flags) {
if (!dataFile_ || absolutePos < 0 || absolutePos >= dataFileSize_) {
return 0;
}
if (absolutePos + bytes > dataFileSize_) {
// TODO: This could go negative..
bytes = dataFileSize_ - absolutePos;
}
// Decompress until the requested point, filling up data_ as we go. TODO: Do on thread.
while (dataReadPos_ < absolutePos + bytes) {
int remaining = BLOCK_SIZE;
if (dataReadPos_ + remaining > dataFileSize_) {
remaining = dataFileSize_ - dataReadPos_;
}
zip_int64_t retval = zip_fread(dataFile_, data_ + dataReadPos_, remaining);
_dbg_assert_(retval == remaining);
dataReadPos_ += retval;
}
// Perform the read.
memcpy(data, data_ + absolutePos, bytes);
return bytes;
}
zip_int64_t ZipFileLoader::ZipSourceCallback(void *data, zip_uint64_t len, zip_source_cmd_t cmd) {
switch (cmd) {
case ZIP_SOURCE_OPEN:
{
zipReadPos_ = 0;
return 0;
}
case ZIP_SOURCE_READ:
{
size_t readBytes = static_cast<zip_int64_t>(backend_->ReadAt(zipReadPos_, len, data));
zipReadPos_ += readBytes;
return readBytes;
}
case ZIP_SOURCE_SEEK:
{
struct SeekData {
zip_int64_t offset;
int whence;
};
if (len < sizeof(SeekData)) {
return -1; // Invalid argument size
}
const SeekData *seekData = static_cast<const SeekData*>(data);
zip_int64_t new_offset;
switch (seekData->whence) {
case SEEK_SET:
new_offset = seekData->offset;
break;
case SEEK_CUR:
new_offset = zipReadPos_ + seekData->offset;
break;
case SEEK_END:
new_offset = backend_->FileSize() + seekData->offset;
break;
default:
return -1; // Invalid 'whence' value
}
if (new_offset < 0 || new_offset > backend_->FileSize()) {
return -1; // Offset out of bounds
}
zipReadPos_ = new_offset;
return 0;
}
case ZIP_SOURCE_TELL:
return zipReadPos_;
case ZIP_SOURCE_CLOSE:
return 0;
case ZIP_SOURCE_STAT:
{
if (len < sizeof(zip_stat_t)) {
return -1;
}
zip_stat_t* st = static_cast<zip_stat_t*>(data);
zip_stat_init(st);
st->valid = ZIP_STAT_SIZE;
st->size = static_cast<zip_uint64_t>(backend_->FileSize());
return sizeof(zip_stat_t);
}
case ZIP_SOURCE_ERROR:
return -1;
case ZIP_SOURCE_FREE:
return 0;
case ZIP_SOURCE_SUPPORTS:
return zip_source_make_command_bitmap(ZIP_SOURCE_READ, ZIP_SOURCE_SEEK, ZIP_SOURCE_TELL, ZIP_SOURCE_OPEN, ZIP_SOURCE_CLOSE, ZIP_SOURCE_STAT);
default:
return -1;
}
}
+65
View File
@@ -0,0 +1,65 @@
#pragma once
#include <mutex>
#include <ext/libzip/zip.h>
#include "Common/CommonTypes.h"
#include "Common/Log.h"
#include "Common/File/Path.h"
#include "Common/StringUtils.h"
#include "Core/Loaders.h"
// Exposes a single (chosen) file from a zip file as another file loader.
// Useful in a bunch of possible chains.
class ZipFileLoader : public ProxiedFileLoader {
public:
ZipFileLoader(FileLoader *sourceLoader);
~ZipFileLoader() override;
zip_t *GetZip() const {
return zipArchive_;
}
bool Initialize(int fileIndex);
bool Exists() override {
return dataFile_ != nullptr;
}
bool IsDirectory() override {
return false;
}
s64 FileSize() override {
return dataFileSize_;
}
Path GetPath() const override {
return backend_->GetPath();
}
size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data, Flags flags = Flags::NONE) override {
return ReadAt(absolutePos, bytes * count, data, flags) / bytes;
}
size_t ReadAt(s64 absolutePos, size_t bytes, void *data, Flags flags = Flags::NONE) override;
std::string GetFileExtension() const {
return fileExtension_;
}
private:
zip_int64_t ZipSourceCallback(void* data, zip_uint64_t len, zip_source_cmd_t cmd);
enum {
BLOCK_SIZE = 65536,
};
zip_t *zipArchive_ = nullptr;
s64 zipReadPos_ = 0;
zip_file_t *dataFile_ = nullptr;
uint8_t *data_ = nullptr; // malloc/free
s64 dataReadPos_ = 0;
s64 dataFileSize_ = 0;
std::string fileExtension_;
};
+4
View File
@@ -17,6 +17,10 @@
#pragma once
// TODO: Somehow unify FileSystem, VFS and FileLoaders. Actually, maybe FileSystem and VFS have the most in common
// but file systems should be able to contain FileLoader as files. Then we can do stuff like playing homebrew directly
// out of zip files, and similar tricks.
#include <vector>
#include <string>
#include <cstring>
+27 -1
View File
@@ -18,11 +18,13 @@
#include "Common/File/FileUtil.h"
#include "Common/File/Path.h"
#include "Common/StringUtils.h"
#include "Common/Data/Text/I18n.h"
#include "Core/FileLoaders/CachingFileLoader.h"
#include "Core/FileLoaders/DiskCachingFileLoader.h"
#include "Core/FileLoaders/HTTPFileLoader.h"
#include "Core/FileLoaders/LocalFileLoader.h"
#include "Core/FileLoaders/RetryingFileLoader.h"
#include "Core/FileLoaders/ZipFileLoader.h"
#include "Core/FileSystems/MetaFileSystem.h"
#include "Core/PSPLoaders.h"
#include "Core/MemMap.h"
@@ -31,6 +33,7 @@
#include "Core/System.h"
#include "Core/ELF/PBPReader.h"
#include "Core/ELF/ParamSFO.h"
#include "Core/Util/GameManager.h"
FileLoader *ConstructFileLoader(const Path &filename) {
if (filename.Type() == PathType::HTTP) {
@@ -62,7 +65,7 @@ IdentifiedFileType Identify_File(FileLoader *fileLoader, std::string *errorStrin
return IdentifiedFileType::ERROR_IDENTIFYING;
}
std::string extension = fileLoader->GetPath().GetFileExtension();
std::string extension = fileLoader->GetFileExtension();
if (extension == ".iso") {
// may be a psx iso, they have 2352 byte sectors. You never know what some people try to open
if ((fileLoader->FileSize() % 2352) == 0) {
@@ -229,6 +232,29 @@ FileLoader *ResolveFileLoaderTarget(FileLoader *fileLoader) {
delete fileLoader;
fileLoader = ConstructFileLoader(ebootFilename);
}
} else if (type == IdentifiedFileType::ARCHIVE_ZIP) {
// Handle zip files, take automatic action depending on contents.
// Can also return nullptr.
ZipFileLoader *zipLoader = new ZipFileLoader(fileLoader);
ZipFileInfo zipFileInfo{};
DetectZipFileContents(zipLoader->GetZip(), &zipFileInfo);
switch (zipFileInfo.contents) {
case ZipFileContents::ISO_FILE:
case ZipFileContents::FRAME_DUMP:
{
zipLoader->Initialize(zipFileInfo.isoFileIndex);
return zipLoader;
}
default:
{
// Nothing runnable in file. Take the original loader back and return it.
fileLoader = zipLoader->Steal();
delete zipLoader;
return fileLoader;
}
}
}
return fileLoader;
}
+9 -2
View File
@@ -57,6 +57,7 @@ enum class IdentifiedFileType {
};
// NB: It is a REQUIREMENT that implementations of this class are entirely thread safe!
// TOOD: actually, is it really?
class FileLoader {
public:
enum class Flags {
@@ -77,7 +78,9 @@ public:
virtual bool IsDirectory() = 0;
virtual s64 FileSize() = 0;
virtual Path GetPath() const = 0;
virtual std::string GetFileExtension() const {
return GetPath().GetFileExtension();
}
virtual size_t ReadAt(s64 absolutePos, size_t bytes, size_t count, void *data, Flags flags = Flags::NONE) = 0;
virtual size_t ReadAt(s64 absolutePos, size_t bytes, void *data, Flags flags = Flags::NONE) {
return ReadAt(absolutePos, 1, bytes, data, flags);
@@ -98,7 +101,6 @@ public:
// Takes ownership.
delete backend_;
}
bool IsRemote() override {
return backend_->IsRemote();
}
@@ -129,6 +131,11 @@ public:
size_t ReadAt(s64 absolutePos, size_t bytes, void *data, Flags flags = Flags::NONE) override {
return backend_->ReadAt(absolutePos, bytes, data, flags);
}
FileLoader *Steal() {
FileLoader *backend = backend_;
backend_ = nullptr;
return backend;
}
protected:
FileLoader *backend_;
+8
View File
@@ -256,6 +256,7 @@ bool CanExtractWithoutOverwrite(struct zip *z, const Path &destination, int maxO
void DetectZipFileContents(struct zip *z, ZipFileInfo *info) {
int numFiles = zip_get_num_files(z);
_dbg_assert_(numFiles >= 0);
// Verify that this is a PSP zip file with the correct layout. We also try
// to detect simple zipped ISO files, those we'll just "install" to the current
@@ -263,6 +264,7 @@ void DetectZipFileContents(struct zip *z, ZipFileInfo *info) {
bool isPSPMemstickGame = false;
bool isZippedISO = false;
bool isTexturePack = false;
bool isFrameDump = false;
int stripChars = 0;
int isoFileIndex = -1;
int stripCharsTexturePack = -1;
@@ -323,6 +325,10 @@ void DetectZipFileContents(struct zip *z, ZipFileInfo *info) {
isTexturePack = true;
textureIniIndex = i;
}
} else if (endsWith(zippedName, ".ppdmp")) {
isFrameDump = true;
isoFileIndex = i;
info->contentName = zippedName;
} else if (endsWith(zippedName, "/param.sfo")) {
// Get the game name so we can display it.
std::string paramSFOContents;
@@ -368,6 +374,8 @@ void DetectZipFileContents(struct zip *z, ZipFileInfo *info) {
} else if (stripChars == 0 && filesInRoot == 0 && hasParamSFO && hasIcon0PNG) {
// As downloaded from GameFAQs, for example.
info->contents = ZipFileContents::SAVE_DATA;
} else if (isFrameDump) {
info->contents = ZipFileContents::FRAME_DUMP;
} else {
info->contents = ZipFileContents::UNKNOWN;
}
+1
View File
@@ -41,6 +41,7 @@ enum class ZipFileContents {
ISO_FILE,
TEXTURE_PACK,
SAVE_DATA,
FRAME_DUMP,
};
struct ZipFileInfo {
+15 -1
View File
@@ -31,6 +31,7 @@
#include "UI/MainScreen.h"
#include "UI/OnScreenDisplay.h"
#include "UI/SavedataScreen.h"
#include "UI/EmuScreen.h"
InstallZipScreen::InstallZipScreen(const Path &zipPath) : zipPath_(zipPath) {
g_GameManager.ResetInstallError();
@@ -64,8 +65,8 @@ void InstallZipScreen::CreateViews() {
bool showDeleteCheckbox = false;
returnToHomebrew_ = false;
installChoice_ = nullptr;
playChoice_ = nullptr;
doneView_ = nullptr;
installChoice_ = nullptr;
existingSaveView_ = nullptr;
destFolders_.clear();
@@ -101,6 +102,9 @@ void InstallZipScreen::CreateViews() {
installChoice_ = rightColumnItems->Add(new Choice(iz->T("Install")));
installChoice_->OnClick.Handle(this, &InstallZipScreen::OnInstall);
playChoice_ = rightColumnItems->Add(new Choice(ga->T("Play")));
playChoice_->OnClick.Handle(this, &InstallZipScreen::OnPlay);
returnToHomebrew_ = true;
showDeleteCheckbox = true;
} else if (zipFileInfo_.contents == ZipFileContents::TEXTURE_PACK) {
@@ -160,6 +164,11 @@ void InstallZipScreen::CreateViews() {
doneView_ = leftColumn->Add(new TextView(""));
showDeleteCheckbox = true;
} else if (zipFileInfo_.contents == ZipFileContents::FRAME_DUMP) {
leftColumn->Add(new TextView(zipFileInfo_.contentName));
// It's a frame dump, add a play button!
playChoice_ = rightColumnItems->Add(new Choice(ga->T("Play")));
playChoice_->OnClick.Handle(this, &InstallZipScreen::OnPlay);
} else {
leftColumn->Add(new TextView(iz->T("Zip file does not contain PSP software"), ALIGN_LEFT, false, new AnchorLayoutParams(10, 10, NONE, NONE)));
}
@@ -212,6 +221,11 @@ UI::EventReturn InstallZipScreen::OnInstall(UI::EventParams &params) {
return UI::EVENT_DONE;
}
UI::EventReturn InstallZipScreen::OnPlay(UI::EventParams &params) {
screenManager()->switchScreen(new EmuScreen(zipPath_));
return UI::EVENT_DONE;
}
void InstallZipScreen::update() {
auto iz = GetI18NCategory(I18NCat::INSTALLZIP);
+2
View File
@@ -41,8 +41,10 @@ protected:
private:
UI::EventReturn OnInstall(UI::EventParams &params);
UI::EventReturn OnPlay(UI::EventParams &params);
UI::Choice *installChoice_ = nullptr;
UI::Choice *playChoice_ = nullptr;
UI::Choice *backChoice_ = nullptr;
UI::TextView *doneView_ = nullptr;
SavedataView *existingSaveView_ = nullptr;
+3 -1
View File
@@ -165,6 +165,7 @@
<ClInclude Include="..\..\Core\FileLoaders\LocalFileLoader.h" />
<ClInclude Include="..\..\Core\FileLoaders\RamCachingFileLoader.h" />
<ClInclude Include="..\..\Core\FileLoaders\RetryingFileLoader.h" />
<ClInclude Include="..\..\Core\FileLoaders\ZipFileLoader.h" />
<ClInclude Include="..\..\Core\FileSystems\BlobFileSystem.h" />
<ClInclude Include="..\..\Core\FileSystems\BlockDevices.h" />
<ClInclude Include="..\..\Core\FileSystems\DirectoryFileSystem.h" />
@@ -429,6 +430,7 @@
<ClCompile Include="..\..\Core\FileLoaders\LocalFileLoader.cpp" />
<ClCompile Include="..\..\Core\FileLoaders\RamCachingFileLoader.cpp" />
<ClCompile Include="..\..\Core\FileLoaders\RetryingFileLoader.cpp" />
<ClCompile Include="..\..\Core\FileLoaders\ZipFileLoader.cpp" />
<ClCompile Include="..\..\Core\FileSystems\BlobFileSystem.cpp" />
<ClCompile Include="..\..\Core\FileSystems\BlockDevices.cpp" />
<ClCompile Include="..\..\Core\FileSystems\DirectoryFileSystem.cpp" />
@@ -1048,4 +1050,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>
+6
View File
@@ -312,6 +312,9 @@
<ClCompile Include="..\..\Core\FileLoaders\RetryingFileLoader.cpp">
<Filter>FileLoaders</Filter>
</ClCompile>
<ClCompile Include="..\..\Core\FileLoaders\ZipFileLoader.cpp">
<Filter>FileLoaders</Filter>
</ClCompile>
<ClCompile Include="..\..\Core\HLE\__sceAudio.cpp">
<Filter>HLE</Filter>
</ClCompile>
@@ -1406,6 +1409,9 @@
<ClInclude Include="..\..\Core\FileLoaders\RetryingFileLoader.h">
<Filter>FileLoaders</Filter>
</ClInclude>
<ClInclude Include="..\..\Core\FileLoaders\ZipFileLoader.h">
<Filter>FileLoaders</Filter>
</ClInclude>
<ClInclude Include="..\..\Core\HLE\__sceAudio.h">
<Filter>HLE</Filter>
</ClInclude>
+1
View File
@@ -611,6 +611,7 @@ EXEC_AND_LIB_FILES := \
$(SRC)/Core/FileLoaders/LocalFileLoader.cpp \
$(SRC)/Core/FileLoaders/RamCachingFileLoader.cpp \
$(SRC)/Core/FileLoaders/RetryingFileLoader.cpp \
$(SRC)/Core/FileLoaders/ZipFileLoader.cpp \
$(SRC)/Core/MemFault.cpp \
$(SRC)/Core/MemMap.cpp \
$(SRC)/Core/MemMapFunctions.cpp \
+1
View File
@@ -660,6 +660,7 @@ SOURCES_CXX += \
$(COREDIR)/FileLoaders/RetryingFileLoader.cpp \
$(COREDIR)/FileLoaders/RamCachingFileLoader.cpp \
$(COREDIR)/FileLoaders/LocalFileLoader.cpp \
$(COREDIR)/FileLoaders/ZipFileLoader.cpp \
$(COREDIR)/CoreTiming.cpp \
$(COREDIR)/CwCheat.cpp \
$(COREDIR)/HDRemaster.cpp \