From 7f877bb16b388dd7d382c333eb35fe28b70e85a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Thu, 23 Jan 2025 11:59:55 +0100 Subject: [PATCH] Implement simple file caching. --- Common/Buffer.cpp | 6 +++-- Common/Buffer.h | 4 +-- Common/Net/HTTPClient.cpp | 11 +++++--- Common/Net/HTTPClient.h | 4 +-- Common/Net/HTTPNaettRequest.cpp | 3 ++- Common/Net/HTTPRequest.cpp | 46 +++++++++++++++++++++++++++++++-- Common/Net/HTTPRequest.h | 15 ++++++++++- Core/HLE/sceNet.cpp | 8 ++++-- UI/NativeApp.cpp | 2 ++ 9 files changed, 83 insertions(+), 16 deletions(-) diff --git a/Common/Buffer.cpp b/Common/Buffer.cpp index 54ca26f226..9c83a27400 100644 --- a/Common/Buffer.cpp +++ b/Common/Buffer.cpp @@ -115,7 +115,7 @@ void Buffer::Printf(const char *fmt, ...) { memcpy(ptr, buffer, retval); } -bool Buffer::FlushToFile(const Path &filename) { +bool Buffer::FlushToFile(const Path &filename, bool clear) { FILE *f = File::OpenCFile(filename, "wb"); if (!f) return false; @@ -124,7 +124,9 @@ bool Buffer::FlushToFile(const Path &filename) { data_.iterate_blocks([=](const char *blockData, size_t blockSize) { return fwrite(blockData, 1, blockSize, f) == blockSize; }); - data_.clear(); + if (clear) { + data_.clear(); + } } fclose(f); return true; diff --git a/Common/Buffer.h b/Common/Buffer.h index 79875bfd92..892f496e7f 100644 --- a/Common/Buffer.h +++ b/Common/Buffer.h @@ -66,9 +66,9 @@ public: // Simple I/O. // Writes the entire buffer to the file descriptor. Also resets the - // size to zero. On failure, data remains in buffer and nothing is + // size to zero, if the clear flag is true. On failure, data remains in buffer and nothing is // written. - bool FlushToFile(const Path &filename); + bool FlushToFile(const Path &filename, bool clear); // Utilities. Try to avoid checking for size. size_t size() const { return data_.size(); } diff --git a/Common/Net/HTTPClient.cpp b/Common/Net/HTTPClient.cpp index ad3d1c7002..f3608ebe04 100644 --- a/Common/Net/HTTPClient.cpp +++ b/Common/Net/HTTPClient.cpp @@ -494,7 +494,9 @@ HTTPRequest::HTTPRequest(RequestMethod method, std::string_view url, std::string HTTPRequest::~HTTPRequest() { g_OSD.RemoveProgressBar(url_, !failed_, 0.5f); - _assert_msg_(joined_, "Download destructed without join"); + if (thread_.joinable()) { + _dbg_assert_msg_(false, "Download destructed without join"); + } } void HTTPRequest::Start() { @@ -502,11 +504,11 @@ void HTTPRequest::Start() { } void HTTPRequest::Join() { - if (joined_) { + if (!thread_.joinable()) { ERROR_LOG(Log::HTTP, "Already joined thread!"); + _dbg_assert_(false); } thread_.join(); - joined_ = true; } void HTTPRequest::SetFailed(int code) { @@ -599,7 +601,8 @@ void HTTPRequest::Do() { if (resultCode == 200) { INFO_LOG(Log::HTTP, "Completed requesting %s (storing result to %s)", url_.c_str(), outfile_.empty() ? "memory" : outfile_.c_str()); - if (!outfile_.empty() && !buffer_.FlushToFile(outfile_)) { + bool clear = !(flags_ & RequestFlags::KeepInMemory); + if (!outfile_.empty() && !buffer_.FlushToFile(outfile_, clear)) { ERROR_LOG(Log::HTTP, "Failed writing download to '%s'", outfile_.c_str()); } } else { diff --git a/Common/Net/HTTPClient.h b/Common/Net/HTTPClient.h index 877621c138..ab685e31cc 100644 --- a/Common/Net/HTTPClient.h +++ b/Common/Net/HTTPClient.h @@ -35,7 +35,6 @@ protected: private: uintptr_t sock_ = -1; - }; } // namespace net @@ -116,10 +115,11 @@ private: std::string postMime_; bool completed_ = false; bool failed_ = false; - bool joined_ = false; }; // Fake request for cache hits. +// The download manager uses this when caching was requested, and a new-enough file was present in the cache directory. +// This is simply a finished request, that can still be queried like a normal one so users don't know it came from the cache. class CachedRequest : public Request { public: CachedRequest(RequestMethod method, std::string_view url, std::string_view name, bool *cancelled, RequestFlags flags, std::string_view responseData) diff --git a/Common/Net/HTTPNaettRequest.cpp b/Common/Net/HTTPNaettRequest.cpp index ab630aa428..4e62bd83b2 100644 --- a/Common/Net/HTTPNaettRequest.cpp +++ b/Common/Net/HTTPNaettRequest.cpp @@ -107,7 +107,8 @@ bool HTTPSRequest::Done() { failed_ = true; progress_.Update(bodyLength, bodyLength, true); } else if (resultCode_ == 200) { - if (!outfile_.empty() && !buffer_.FlushToFile(outfile_)) { + bool clear = !(flags_ & RequestFlags::KeepInMemory); + if (!outfile_.empty() && !buffer_.FlushToFile(outfile_, clear)) { ERROR_LOG(Log::IO, "Failed writing download to '%s'", outfile_.c_str()); } progress_.Update(bodyLength, bodyLength, true); diff --git a/Common/Net/HTTPRequest.cpp b/Common/Net/HTTPRequest.cpp index 18c1e3d87d..044a516ea0 100644 --- a/Common/Net/HTTPRequest.cpp +++ b/Common/Net/HTTPRequest.cpp @@ -2,6 +2,7 @@ #include "Common/Net/HTTPClient.h" #include "Common/Net/HTTPNaettRequest.h" #include "Common/TimeUtil.h" +#include "Common/File/FileUtil.h" #include "Common/StringUtils.h" #include "Common/Log.h" #include "Common/System/OSD.h" @@ -39,25 +40,66 @@ static bool IsHttpsUrl(std::string_view url) { return startsWith(url, "https:"); } +Path UrlToCachePath(const Path &cacheDir, std::string_view url) { + std::string fn = "DLCACHE_"; + for (auto c : url) { + if (isalnum(c) || c == '.' || c == '-' || c == '_') { + fn.push_back(tolower(c)); + } else { + fn.push_back('_'); + } + } + return cacheDir / fn; +} + std::shared_ptr CreateRequest(RequestMethod method, std::string_view url, std::string_view postdata, std::string_view postMime, const Path &outfile, RequestFlags flags, std::string_view name) { if (IsHttpsUrl(url) && System_GetPropertyBool(SYSPROP_SUPPORTS_HTTPS)) { #ifndef HTTPS_NOT_AVAILABLE - return std::shared_ptr(new HTTPSRequest(method, url, postdata, postMime, outfile, flags, name)); + return std::make_shared(method, url, postdata, postMime, outfile, flags, name); #else return std::shared_ptr(); #endif } else { - return std::shared_ptr(new HTTPRequest(method, url, postdata, postMime, outfile, flags, name)); + return std::make_shared(method, url, postdata, postMime, outfile, flags, name); } } std::shared_ptr RequestManager::StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime) { std::shared_ptr dl = CreateRequest(RequestMethod::GET, url, "", "", outfile, flags, ""); + if (!cacheDir_.empty() && (flags & RequestFlags::Cached24H)) { + _dbg_assert_(outfile.empty()); // It's automatically replaced below + + // Come up with a cache file path. + Path cacheFile = UrlToCachePath(cacheDir_, url); + + // TODO: This should be done on the thread, maybe. But let's keep it simple for now. + time_t cacheFileTime; + if (File::GetModifTimeT(cacheFile, &cacheFileTime)) { + time_t now = (time_t)time_now_unix_utc(); + if (cacheFileTime > now - 24 * 60 * 60) { + // The file is new enough. Let's construct a fake, already finished download so we don't need + // to modify the calling code. + std::string contents; + if (File::ReadBinaryFileToString(cacheFile, &contents)) { + // All is well, but we've indented a bit much here. + dl.reset(new CachedRequest(RequestMethod::GET, url, "", nullptr, flags, contents)); + newDownloads_.push_back(dl); + return dl; + } + } + } + + // OK, didn't get it from cache, so let's continue with the download, putting it in the cache. + dl->OverrideOutFile(cacheFile); + dl->AddFlag(RequestFlags::KeepInMemory); + } + if (!userAgent_.empty()) dl->SetUserAgent(userAgent_); if (acceptMime) dl->SetAccept(acceptMime); + newDownloads_.push_back(dl); dl->Start(); return dl; diff --git a/Common/Net/HTTPRequest.h b/Common/Net/HTTPRequest.h index 35706b66c8..db0525e19e 100644 --- a/Common/Net/HTTPRequest.h +++ b/Common/Net/HTTPRequest.h @@ -18,7 +18,8 @@ enum class RequestFlags { Default = 0, ProgressBar = 1, ProgressBarDelayed = 2, - // Cached = 4 etc + Cached24H = 4, + KeepInMemory = 8, }; ENUM_CLASS_BITOPS(RequestFlags); @@ -59,6 +60,12 @@ public: std::string url() const { return url_; } const Path &OutFile() const { return outfile_; } + void OverrideOutFile(const Path &path) { + outfile_ = path; + } + void AddFlag(RequestFlags flag) { + flags_ |= flag; + } void Cancel() { cancelled_ = true; } bool IsCancelled() const { return cancelled_; } @@ -97,6 +104,7 @@ public: CancelAll(); } + // NOTE: This is the only version that supports the cache flag (for now). std::shared_ptr StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime = nullptr); std::shared_ptr StartDownloadWithCallback( @@ -123,6 +131,10 @@ public: userAgent_ = userAgent; } + void SetCacheDir(const Path &path) { + cacheDir_ = path; + } + private: std::vector> downloads_; // These get copied to downloads_ in Update(). It's so that callbacks can add new downloads @@ -130,6 +142,7 @@ private: std::vector> newDownloads_; std::string userAgent_; + Path cacheDir_; }; inline const char *RequestMethodToString(RequestMethod method) { diff --git a/Core/HLE/sceNet.cpp b/Core/HLE/sceNet.cpp index da833a60a6..fb7be6ada3 100644 --- a/Core/HLE/sceNet.cpp +++ b/Core/HLE/sceNet.cpp @@ -328,7 +328,7 @@ void StartInfraJsonDownload() { INFO_LOG(Log::sceNet, "json is already being downloaded"); } const char *acceptMime = "application/json, text/*; q=0.9, */*; q=0.8"; - g_infraDL = g_DownloadManager.StartDownload("http://metadata.ppsspp.org/infra-dns.json", Path(), http::RequestFlags::Default, acceptMime); + g_infraDL = g_DownloadManager.StartDownload("http://metadata.ppsspp.org/infra-dns.json", Path(), http::RequestFlags::Cached24H, acceptMime); } bool PollInfraJsonDownload(std::string *jsonOutput) { @@ -359,9 +359,13 @@ bool PollInfraJsonDownload(std::string *jsonOutput) { } // OK, we actually got data. Load it! - INFO_LOG(Log::sceNet, "json downloaded"); g_infraDL->buffer().TakeAll(jsonOutput); + if (jsonOutput->empty()) { + _dbg_assert_msg_(false, "Json output is empty!"); + ERROR_LOG(Log::sceNet, "JSON output is empty! Something went wrong."); + } + LoadAutoDNS(*jsonOutput); return true; } diff --git a/UI/NativeApp.cpp b/UI/NativeApp.cpp index 04257f98cc..7c26bb348e 100644 --- a/UI/NativeApp.cpp +++ b/UI/NativeApp.cpp @@ -724,6 +724,8 @@ void NativeInit(int argc, const char *argv[], const char *savegame_dir, const ch } } + g_DownloadManager.SetCacheDir(GetSysDirectory(DIRECTORY_APP_CACHE)); + DEBUG_LOG(Log::System, "ScreenManager!"); g_screenManager = new ScreenManager(); if (g_Config.memStickDirectory.empty()) {