HTTPDownloader: Convert to namespace

No virtual calls, and we can notify the host when the state changes to
reduce request latency.

src/core/achievements.cpp

@@ -615,12 +615,6 @@ bool Achievements::CreateClient(std::unique_lock<std::recursive_mutex>& lock, bo
 {
   Assert(!s_state.client);
 
-  if (!HTTPCache::GetDownloader())
-  {
-    Host::ReportErrorAsync("Achievements Error", "Failed to create HTTPDownloader, cannot use achievements");
-    return false;
-  }
-
   s_state.client = rc_client_create(ClientReadMemory, ClientServerCall);
   if (!s_state.client)
   {
@@ -638,7 +632,7 @@ bool Achievements::CreateClient(std::unique_lock<std::recursive_mutex>& lock, bo
   // Populate user-agent.
   char rc_client_user_agent[128];
   rc_client_get_user_agent_clause(s_state.client, rc_client_user_agent, std::size(rc_client_user_agent));
-  s_state.http_user_agent_header = fmt::format("User-Agent: {} {}", HTTPCache::GetUserAgent(), rc_client_user_agent);
+  s_state.http_user_agent_header = fmt::format("User-Agent: {} {}", Host::GetHTTPUserAgent(), rc_client_user_agent);
   VERBOSE_LOG(s_state.http_user_agent_header);
 
   // Allow custom host to be overridden through config.
@@ -881,9 +875,9 @@ uint32_t Achievements::ClientReadMemory(uint32_t address, uint8_t* buffer, uint3
 void Achievements::ClientServerCall(const rc_api_request_t* request, rc_client_server_callback_t callback,
                                     void* callback_data, rc_client_t* client)
 {
-  HTTPDownloader::Request::Callback hd_callback = [callback, callback_data](s32 status_code, Error& error,
-                                                                            std::string& content_type,
-                                                                            HTTPDownloader::Request::Data& data) {
+  HTTPDownloader::RequestCallback hd_callback = [callback, callback_data](s32 status_code, Error& error,
+                                                                          std::string& content_type,
+                                                                          HTTPDownloader::RequestData& data) {
     if (status_code != HTTPDownloader::HTTP_STATUS_OK)
       ERROR_LOG("Server call failed: {}", error.GetDescription());
 
@@ -892,20 +886,18 @@ void Achievements::ClientServerCall(const rc_api_request_t* request, rc_client_s
     callback(&rr, callback_data);
   };
 
-  HTTPDownloader* const downloader = HTTPCache::GetDownloader();
-  DebugAssert(downloader);
-
   const std::array<const char* const, 1> headers = {s_state.http_user_agent_header.c_str()};
   if (request->post_data)
   {
     // const auto pd = std::string_view(request->post_data);
     // Log_DevFmt("Server POST: {}", pd.substr(0, std::min<size_t>(pd.length(), 10)));
-    downloader->CreatePostRequest(request->url, request->post_data, &s_state, std::move(hd_callback), nullptr, headers,
-                                  SERVER_CALL_TIMEOUT);
+    HTTPDownloader::CreatePostRequest(request->url, request->post_data, &s_state, std::move(hd_callback), nullptr,
+                                      headers, SERVER_CALL_TIMEOUT);
   }
   else
   {
-    downloader->CreateRequest(request->url, &s_state, std::move(hd_callback), nullptr, headers, SERVER_CALL_TIMEOUT);
+    HTTPDownloader::CreateRequest(request->url, &s_state, std::move(hd_callback), nullptr, headers,
+                                  SERVER_CALL_TIMEOUT);
   }
 }
 
@@ -945,7 +937,8 @@ rc_api_server_response_t Achievements::MakeRCAPIServerResponse(s32 status_code,
 
 void Achievements::WaitForServerCallsWithYield(std::unique_lock<std::recursive_mutex>& lock)
 {
-  HTTPCache::WaitForAllRequestsFromOwnerWithYield(&s_state, [&lock]() { lock.unlock(); }, [&lock]() { lock.lock(); });
+  HTTPDownloader::WaitForAllRequestsFromOwnerWithYield(
+    &s_state, [&lock]() { lock.unlock(); }, [&lock]() { lock.lock(); });
 }
 
 void Achievements::IdleUpdate()
@@ -2344,7 +2337,7 @@ bool Achievements::DownloadGameIcons(ProgressCallback* progress, Error* error)
   progress->SetProgressRange(badges_to_download);
   progress->FormatStatusText(TRANSLATE_FS("Achievements", "Downloading {} game icons..."), badges_to_download);
   lock.unlock();
-  HTTPCache::WaitForAllRequests();
+  HTTPCache::WaitForAllPrefetchRequests();
   return true;
 }
 

src/core/core.cpp

@@ -15,6 +15,7 @@
 
 #include "util/gpu_device.h"
 #include "util/http_cache.h"
+#include "util/http_downloader.h"
 #include "util/ini_settings_interface.h"
 #include "util/input_manager.h"
 
@@ -558,6 +559,11 @@ void Core::SetInputSettingsLayer(SettingsInterface* sif, std::unique_lock<std::m
   s_locals.layered_settings_interface.SetLayer(LayeredSettingsInterface::LAYER_INPUT, sif);
 }
 
+std::string Host::GetHTTPUserAgent()
+{
+  return fmt::format("DuckStation for {} ({}) {}", TARGET_OS_STR, CPU_ARCH_STR, g_scm_tag_str);
+}
+
 bool Core::PerformEarlyHardwareChecks(Error* error)
 {
   // This shouldn't fail... if it does, just hope for the best.
@@ -750,6 +756,7 @@ void Core::CoreThreadShutdown()
 
   InputManager::CloseSources();
 
+  HTTPDownloader::Shutdown();
   HTTPCache::Shutdown();
 
   VideoThread::ProcessShutdown();
@@ -794,7 +801,7 @@ void Core::IdleUpdate()
   DiscordPresence::Poll();
 #endif
 
-  HTTPCache::PollRequests();
+  HTTPDownloader::PollRequests();
 
   Achievements::IdleUpdate();
 

src/core/game_list.cpp

@@ -1852,13 +1852,12 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
     }
 
     // we could actually do a few in parallel here...
+    // TODO: if we do, need to make sure to not write the same file at the same time, and to update progress correctly.
     std::string filename = Path::URLDecode(url);
-    if (HTTPDownloader* const downloader = HTTPCache::GetDownloader(error))
-    {
-      downloader->CreateRequest(
-        std::move(url), &s_state,
-        [use_serial, &save_callback, entry_path = std::move(entry_path), filename = std::move(filename)](
-          s32 status_code, Error& error, std::string& content_type, HTTPDownloader::Request::Data& data) {
+    HTTPDownloader::CreateRequest(
+      std::move(url), &s_state,
+      [use_serial, &save_callback, entry_path = std::move(entry_path), filename = std::move(filename)](
+        s32 status_code, Error& error, std::string& content_type, HTTPDownloader::RequestData& data) {
         if (status_code != HTTPDownloader::HTTP_STATUS_OK || data.empty())
         {
           ERROR_LOG("Download for {} failed: {}", Path::GetFileName(filename), error.GetDescription());
@@ -1892,8 +1891,7 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
         if (FileSystem::WriteBinaryFile(write_path.c_str(), data.data(), data.size()) && save_callback)
           save_callback(entry, std::move(write_path));
       });
-    }
-    HTTPCache::WaitForAllRequests();
+    HTTPDownloader::WaitForAllRequestsFromOwner(&s_state);
     progress->IncrementProgressValue();
   }
 

src/core/system.cpp

@@ -47,7 +47,7 @@
 #include "util/cd_image.h"
 #include "util/compress_helpers.h"
 #include "util/gpu_device.h"
-#include "util/http_cache.h"
+#include "util/http_downloader.h"
 #include "util/imgui_manager.h"
 #include "util/ini_settings_interface.h"
 #include "util/input_manager.h"
@@ -2001,7 +2001,7 @@ void System::FrameDone()
 
     Cheats::ApplyFrameEndCodes();
 
-    HTTPCache::PollRequests();
+    HTTPDownloader::PollRequests();
 
     if (Achievements::IsActive())
       Achievements::FrameUpdate();

src/duckstation-qt/asynchttprequest.cpp

@@ -25,22 +25,10 @@ void AsyncHTTPRequest::get(std::string url, const void* owner, ProgressCallback*
                            HTTPDownloader::HeaderList additional_headers /* =  */,
                            std::optional<u16> timeout_seconds /* = */)
 {
-  HTTPDownloader* const downloader = HTTPCache::GetDownloader(&m_error);
-  if (!downloader)
-  {
-    emit requestComplete(HTTPDownloader::HTTP_STATUS_ERROR, m_error, m_content_type, m_data);
-    deleteLater();
-    return;
-  }
-
-  downloader->CreateRequest(
+  HTTPDownloader::CreateRequest(
     std::move(url), owner,
-    [this](s32 status_code, Error& error, std::string& content_type, HTTPDownloader::Request::Data& data) {
-      m_status_code = status_code;
-      m_error = std::move(error);
-      m_content_type = std::move(content_type);
-      m_data = std::move(data);
-      QMetaObject::invokeMethod(this, &AsyncHTTPRequest::finishRequest, Qt::QueuedConnection);
+    [this](s32 status_code, Error& error, std::string& content_type, HTTPDownloader::RequestData& data) {
+      handleResponse(status_code, error, content_type, data);
     },
     progress, additional_headers, timeout_seconds);
 }
@@ -50,26 +38,24 @@ void AsyncHTTPRequest::post(std::string url, std::string post_data, const void*
                             HTTPDownloader::HeaderList additional_headers /* =  */,
                             std::optional<u16> timeout_seconds /* = */)
 {
-  HTTPDownloader* const downloader = HTTPCache::GetDownloader(&m_error);
-  if (!downloader)
-  {
-    emit requestComplete(HTTPDownloader::HTTP_STATUS_ERROR, m_error, m_content_type, m_data);
-    deleteLater();
-    return;
-  }
-
-  downloader->CreatePostRequest(
+  HTTPDownloader::CreatePostRequest(
     std::move(url), std::move(post_data), owner,
-    [this](s32 status_code, Error& error, std::string& content_type, HTTPDownloader::Request::Data& data) {
-      m_status_code = status_code;
-      m_error = std::move(error);
-      m_content_type = std::move(content_type);
-      m_data = std::move(data);
-      QMetaObject::invokeMethod(this, &AsyncHTTPRequest::finishRequest, Qt::QueuedConnection);
+    [this](s32 status_code, Error& error, std::string& content_type, HTTPDownloader::RequestData& data) {
+      handleResponse(status_code, error, content_type, data);
     },
     progress, additional_headers, timeout_seconds);
 }
 
+ALWAYS_INLINE_RELEASE void AsyncHTTPRequest::handleResponse(s32 status_code, Error& error, std::string& content_type,
+                                                            HTTPDownloader::RequestData& data)
+{
+  m_status_code = status_code;
+  m_error = std::move(error);
+  m_content_type = std::move(content_type);
+  m_data = std::move(data);
+  QMetaObject::invokeMethod(this, &AsyncHTTPRequest::finishRequest, Qt::QueuedConnection);
+}
+
 void AsyncHTTPRequest::finishRequest()
 {
   emit requestComplete(m_status_code, m_error, m_content_type, m_data);

src/duckstation-qt/asynchttprequest.h

@@ -5,6 +5,8 @@
 
 #include "util/http_downloader.h"
 
+#include "common/error.h"
+
 #include <string>
 #include <string_view>
 
@@ -27,13 +29,14 @@ public:
 
 Q_SIGNALS:
   void requestComplete(qint32 status_code, Error& error_message, std::string& content_type,
-                       HTTPDownloader::Request::Data& data);
+                       HTTPDownloader::RequestData& data);
 
 private:
+  void handleResponse(s32 status_code, Error& error, std::string& content_type, HTTPDownloader::RequestData& data);
   void finishRequest();
 
   s32 m_status_code = HTTPDownloader::HTTP_STATUS_ERROR;
   Error m_error;
   std::string m_content_type;
-  HTTPDownloader::Request::Data m_data;
+  HTTPDownloader::RequestData m_data;
 };

src/duckstation-qt/autoupdaterdialog.cpp

@@ -12,7 +12,6 @@
 
 #include "core/core.h"
 
-#include "util/http_cache.h"
 #include "util/http_downloader.h"
 #include "util/translation.h"
 
@@ -129,7 +128,7 @@ AutoUpdaterDialog::AutoUpdaterDialog(QWidget* const parent) : QDialog(parent)
 AutoUpdaterDialog::~AutoUpdaterDialog()
 {
   // Ensure all requests have finished.
-  HTTPCache::CancelRequestsForOwner(this);
+  HTTPDownloader::CancelRequestsForOwner(this);
 }
 
 void AutoUpdaterDialog::warnAboutUnofficialBuild()
@@ -300,9 +299,7 @@ void AutoUpdaterDialog::cancel()
   if (m_updates_available)
     return;
 
-  HTTPDownloader* const downloader = HTTPCache::GetDownloader();
-  if (downloader)
-    downloader->CancelRequestsForOwner(this);
+  HTTPDownloader::CancelRequestsForOwner(this);
 }
 
 bool AutoUpdaterDialog::handleCancelledRequest(s32 status_code)

src/duckstation-qt/autoupdaterdialog.h

@@ -12,7 +12,6 @@
 #include <vector>
 
 class Error;
-class HTTPDownloader;
 class QtProgressCallback;
 
 class CoreThread;

src/duckstation-qt/qthost.cpp

@@ -742,30 +742,26 @@ void QtHost::DownloadFile(QWidget* parent, std::string url, std::string path,
      completion_callback = std::move(completion_callback)](ProgressCallback* const progress) mutable {
       Error error;
       bool result = false;
-      if (HTTPDownloader* const downloader = HTTPCache::GetDownloader(&error))
-      {
-        result = true;
-        downloader->CreateRequest(
-          std::move(url), parent,
-          [&result, &error, &path](s32 status_code, Error& http_error, std::string&, std::vector<u8>& hdata) {
-            if (status_code != HTTPDownloader::HTTP_STATUS_OK)
-            {
-              error.SetString(http_error.GetDescription());
-              return;
-            }
-            else if (hdata.empty())
-            {
-              error.SetStringView(TRANSLATE_SV("QtHost", "Download failed: Data is empty."));
-              return;
-            }
+      HTTPDownloader::CreateRequest(
+        std::move(url), parent,
+        [&result, &error, &path](s32 status_code, Error& http_error, std::string&, std::vector<u8>& hdata) {
+          if (status_code != HTTPDownloader::HTTP_STATUS_OK)
+          {
+            error.SetString(http_error.GetDescription());
+            return;
+          }
+          else if (hdata.empty())
+          {
+            error.SetStringView(TRANSLATE_SV("QtHost", "Download failed: Data is empty."));
+            return;
+          }
 
-            result = FileSystem::WriteBinaryFile(path.c_str(), hdata, &error);
-          },
-          progress);
-      }
+          result = FileSystem::WriteBinaryFile(path.c_str(), hdata, &error);
+        },
+        progress);
 
       // Block until completion.
-      HTTPCache::WaitForAllRequestsFromOwner(parent);
+      HTTPDownloader::WaitForAllRequestsFromOwner(parent);
 
       QtAsyncTaskWithProgressDialog::CompletionCallback ret;
       if (completion_callback)
@@ -2112,7 +2108,7 @@ int CoreThread::getBackgroundControllerPollInterval() const
 
   if (m_video_thread_run_idle)
     return FULLSCREEN_UI_CONTROLLER_POLLING_INTERVAL;
-  else if (HTTPCache::IsDownloaderActive())
+  else if (m_http_downloader_active)
     return DOWNLOAD_CONTROLLER_POLLING_INTERVAL;
   else if (InputManager::GetPollableDeviceCount() > 0)
     return BACKGROUND_CONTROLLER_POLLING_INTERVAL_WITH_DEVICES;
@@ -2120,6 +2116,19 @@ int CoreThread::getBackgroundControllerPollInterval() const
     return BACKGROUND_CONTROLLER_POLLING_INTERVAL_WITHOUT_DEVICES;
 }
 
+void CoreThread::setHTTPDownloaderActive(bool active)
+{
+  if (!isCurrentThread())
+  {
+    QMetaObject::invokeMethod(this, &CoreThread::setHTTPDownloaderActive, Qt::QueuedConnection, active);
+    return;
+  }
+
+  DEV_LOG("HTTP Downloader now {}", active ? "active" : "inactive");
+  m_http_downloader_active = active;
+  updateBackgroundControllerPollInterval();
+}
+
 void CoreThread::setVideoThreadRunIdle(bool active)
 {
   if (!isCurrentThread())
@@ -2154,9 +2163,9 @@ void CoreThread::updateFullscreenUITheme()
     VideoThread::RunOnThread(&FullscreenUI::UpdateTheme);
 }
 
-void Host::OnHTTPCacheDownloaderActiveChanged(bool active)
+void Host::OnHTTPDownloaderActiveChanged(bool active)
 {
-  g_core_thread->updateBackgroundControllerPollInterval();
+  g_core_thread->setHTTPDownloaderActive(active);
 }
 
 void CoreThread::stop()

src/duckstation-qt/qthost.h

@@ -179,6 +179,7 @@ public:
   void captureGPUFrameDump();
   void startControllerTest();
   void openGamePropertiesForCurrentGame(const QString& category = {});
+  void setHTTPDownloaderActive(bool active);
   void setVideoThreadRunIdle(bool active);
   void updateFullscreenUITheme();
   void runOnCoreThread(const std::function<void()>& callback);
@@ -213,17 +214,19 @@ private:
   std::unique_ptr<InputDeviceListModel> m_input_device_list_model;
 
   bool m_shutdown_flag = false;
+  bool m_http_downloader_active = false;
   bool m_video_thread_run_idle = false;
   bool m_is_fullscreen_ui_started = false;
   bool m_was_paused_by_focus_loss = false;
 
+  RenderAPI m_last_render_api = RenderAPI::None;
+  bool m_last_hardware_renderer = false;
+
   float m_last_speed = std::numeric_limits<float>::infinity();
   float m_last_game_fps = std::numeric_limits<float>::infinity();
   float m_last_video_fps = std::numeric_limits<float>::infinity();
   u32 m_last_render_width = std::numeric_limits<u32>::max();
   u32 m_last_render_height = std::numeric_limits<u32>::max();
-  RenderAPI m_last_render_api = RenderAPI::None;
-  bool m_last_hardware_renderer = false;
 };
 
 class InputDeviceListModel final : public QAbstractListModel

src/duckstation-regtest/regtest_host.cpp

@@ -24,7 +24,7 @@
 
 #include "util/cd_image.h"
 #include "util/gpu_device.h"
-#include "util/http_cache.h"
+#include "util/http_downloader.h"
 #include "util/imgui_manager.h"
 #include "util/input_manager.h"
 #include "util/translation.h"
@@ -313,7 +313,7 @@ void Host::OnMediaCaptureStopped()
   //
 }
 
-void Host::OnHTTPCacheDownloaderActiveChanged(bool active)
+void Host::OnHTTPDownloaderActiveChanged(bool active)
 {
   //
 }

src/util/CMakeLists.txt

@@ -238,12 +238,7 @@ elseif(NOT ANDROID)
 endif()
 
 if(NOT ANDROID)
-  target_sources(util PRIVATE
-    http_downloader_curl.cpp
-  )
-  target_link_libraries(util PRIVATE
-    CURL::libcurl
-  )
+  target_link_libraries(util PRIVATE CURL::libcurl)
 endif()
 
 function(add_util_resources target)

src/util/http_cache.cpp

@@ -13,8 +13,6 @@
 #include "common/path.h"
 #include "common/string_util.h"
 
-#include "scmversion/scmversion.h"
-
 #include <array>
 #include <deque>
 #include <functional>
@@ -32,19 +30,16 @@ static constexpr u32 CACHE_VERSION = 1;
 
 static bool QueueDownload(std::string_view url, FetchCallback callback, Error* error);
 static void DownloadCallback(const std::string& url, s32 status_code, const Error& error,
-                             const std::string& content_type, const HTTPDownloader::Request::Data& data);
+                             const std::string& content_type, const HTTPDownloader::RequestData& data);
 
 namespace {
 
 struct Locals
 {
-  std::unique_ptr<HTTPDownloader> downloader;
-  bool downloader_was_active = false;
-  bool tried_initialize_cache_archive = false;
   ObjectArchive cache_archive;
   std::deque<std::pair<std::string, FetchCallback>> pending_downloads;
   std::mutex cache_mutex;
-  std::once_flag downloader_once_flag;
+  bool tried_initialize_cache_archive = false;
 };
 
 } // namespace
@@ -77,118 +72,23 @@ std::string_view HTTPCache::GetURLFilename(std::string_view url)
   return (pos != std::string_view::npos) ? url.substr(pos + 1) : url;
 }
 
-std::string HTTPCache::GetUserAgent()
-{
-  return fmt::format("DuckStation for {} ({}) {}", TARGET_OS_STR, CPU_ARCH_STR, g_scm_tag_str);
-}
-
 void HTTPCache::Shutdown()
 {
   // awkward situation where a request callback could create another downloader...
-  std::unique_ptr<HTTPDownloader> downloader;
-  {
-    const std::unique_lock cache_lock(s_locals.cache_mutex);
-    for (auto iter = s_locals.pending_downloads.begin(); iter != s_locals.pending_downloads.end();)
-    {
-      if (iter->second)
-        iter->second({});
-      iter = s_locals.pending_downloads.erase(iter);
-    }
-
-    // Hope nothing else is calling in here... we'll use the cache mutex as a guard.
-    downloader = std::move(s_locals.downloader);
-  }
-
-  if (downloader)
-  {
-    downloader->CancelAllRequests();
-    downloader.reset();
-  }
+  HTTPDownloader::CancelRequestsForOwner(&s_locals);
 
   const std::unique_lock cache_lock(s_locals.cache_mutex);
+  for (auto iter = s_locals.pending_downloads.begin(); iter != s_locals.pending_downloads.end();)
+  {
+    if (iter->second)
+      iter->second({});
+    iter = s_locals.pending_downloads.erase(iter);
+  }
+
   if (s_locals.cache_archive.IsOpen())
     s_locals.cache_archive.Close();
 }
 
-bool HTTPCache::IsDownloaderActive()
-{
-  return s_locals.downloader_was_active;
-}
-
-void HTTPCache::PollRequests()
-{
-  // Racey read, but worst case we just miss some requests until the next update
-  if (!s_locals.downloader)
-    return;
-
-  const bool active = s_locals.downloader->PollRequests();
-  if (s_locals.downloader_was_active != active)
-  {
-    s_locals.downloader_was_active = active;
-    Host::OnHTTPCacheDownloaderActiveChanged(active);
-  }
-}
-
-void HTTPCache::WaitForAllRequests()
-{
-  if (s_locals.downloader)
-    s_locals.downloader->WaitForAllRequests();
-}
-
-void HTTPCache::WaitForAllRequestsFromOwner(const void* owner)
-{
-  if (s_locals.downloader)
-    s_locals.downloader->WaitForAllRequestsFromOwner(owner);
-}
-
-void HTTPCache::WaitForAllRequestsWithYield(std::function<void()> before_sleep_cb, std::function<void()> after_sleep_cb)
-{
-  if (s_locals.downloader)
-    s_locals.downloader->WaitForAllRequestsWithYield(std::move(before_sleep_cb), std::move(after_sleep_cb));
-}
-
-void HTTPCache::WaitForAllRequestsFromOwnerWithYield(const void* owner, std::function<void()> before_sleep_cb,
-                                                     std::function<void()> after_sleep_cb)
-{
-  if (s_locals.downloader)
-  {
-    s_locals.downloader->WaitForAllRequestsFromOwnerWithYield(owner, std::move(before_sleep_cb),
-                                                              std::move(after_sleep_cb));
-  }
-}
-
-void HTTPCache::CancelRequestsForOwner(const void* owner)
-{
-  if (!s_locals.downloader)
-    return;
-
-  s_locals.downloader->CancelRequestsForOwner(owner);
-}
-
-HTTPDownloader* HTTPCache::GetDownloader(Error* create_error)
-{
-  if (s_locals.downloader) [[likely]]
-    return s_locals.downloader.get();
-
-  Error::Clear(create_error);
-  std::call_once(s_locals.downloader_once_flag, [&create_error]() {
-    Error error;
-    s_locals.downloader = HTTPDownloader::Create(GetUserAgent(), create_error ? create_error : &error);
-    if (!s_locals.downloader)
-      ERROR_LOG("Failed to create HTTPDownloader: {}", (create_error ? create_error : &error)->GetDescription());
-  });
-
-  if (!s_locals.downloader)
-  {
-    if (create_error && !create_error->IsValid())
-      create_error->SetStringView("Previous HTTPDownloader creation failed");
-
-    return nullptr;
-  }
-
-  return s_locals.downloader.get();
-}
-
 HTTPCache::CacheArchivePtr HTTPCache::GetCacheArchive()
 {
   std::unique_lock lock(s_locals.cache_mutex);
@@ -262,14 +162,6 @@ bool HTTPCache::QueueDownload(std::string_view url, FetchCallback callback, Erro
 {
   // NOTE: Assumes that the lock is held
 
-  HTTPDownloader* const downloader = GetDownloader(error);
-  if (!downloader) [[unlikely]]
-  {
-    if (callback)
-      callback({});
-    return false;
-  }
-
   // do we already have a request?
   const bool has_request =
     std::ranges::any_of(s_locals.pending_downloads, [url](const auto& pair) { return pair.first == url; });
@@ -285,17 +177,17 @@ bool HTTPCache::QueueDownload(std::string_view url, FetchCallback callback, Erro
 
   DEV_LOG("Cache miss for URL '{}', downloading...", url);
 
-  downloader->CreateRequest(std::string(url), &s_locals,
-                            [url = std::string(url)](s32 status_code, Error& error, std::string& content_type,
-                                                     HTTPDownloader::Request::Data& data) {
-                              DownloadCallback(url, status_code, error, content_type, std::move(data));
-                            });
+  HTTPDownloader::CreateRequest(std::string(url), &s_locals,
+                                [url = std::string(url)](s32 status_code, Error& error, std::string& content_type,
+                                                         HTTPDownloader::RequestData& data) {
+                                  DownloadCallback(url, status_code, error, content_type, std::move(data));
+                                });
 
   return true;
 }
 
 void HTTPCache::DownloadCallback(const std::string& url, s32 status_code, const Error& error,
-                                 const std::string& content_type, const HTTPDownloader::Request::Data& data)
+                                 const std::string& content_type, const HTTPDownloader::RequestData& data)
 {
   const bool success = (status_code == HTTPDownloader::HTTP_STATUS_OK);
   if (!success)
@@ -348,10 +240,6 @@ void HTTPCache::Prefetch(std::string_view url)
   if (!cache->IsOpen() || cache->Contains(url)) [[unlikely]]
     return;
 
-  HTTPDownloader* const downloader = GetDownloader();
-  if (!downloader) [[unlikely]]
-    return;
-
   // queue a download with no callback, which will cause it to be cached when it completes
   QueueDownload(url, {}, nullptr);
 }
@@ -373,17 +261,15 @@ void HTTPCache::Prefetch(std::string_view url, PrefetchCallback callback)
     return;
   }
 
-  HTTPDownloader* const downloader = GetDownloader();
-  if (!downloader) [[unlikely]]
-  {
-    callback(false);
-    return;
-  }
-
   // queue a download with no callback, which will cause it to be cached when it completes
   QueueDownload(url, [callback = std::move(callback)](std::span<const u8> data) { callback(!data.empty()); }, nullptr);
 }
 
+void HTTPCache::WaitForAllPrefetchRequests()
+{
+  HTTPDownloader::WaitForAllRequestsFromOwner(&s_locals);
+}
+
 bool HTTPCache::Clear(Error* error)
 {
   return GetCacheArchive()->Clear(error);

src/util/http_cache.h

@@ -13,7 +13,6 @@
 #include <string_view>
 
 class Error;
-class HTTPDownloader;
 class ObjectArchive;
 
 namespace HTTPCache {
@@ -47,35 +46,9 @@ bool IsHTTPURL(std::string_view url);
 /// Returns the filename portion of @p url, stripping any query string or fragment.
 std::string_view GetURLFilename(std::string_view url);
 
-/// Returns the user agent to use for HTTP requests.
-std::string GetUserAgent();
-
-/// Shuts down the HTTP cache, releasing the downloader and cache archive.
+/// Shuts down the HTTP cache, releasing the cache archive.
 void Shutdown();
 
-/// Returns a pointer to the shared HTTP downloader, creating it on first use.
-/// If @p create_error is non-null, creation details are written there on failure; if null, creation
-/// is skipped after the first failed attempt.
-HTTPDownloader* GetDownloader(Error* create_error = nullptr);
-
-/// Returns true if idle updates are necessary (e.g. outstanding requests).
-bool IsDownloaderActive();
-
-/// Processes completed HTTP requests and invokes their callbacks. Should be called regularly on the main thread.
-void PollRequests();
-
-/// Waits for all requests to finish.
-void WaitForAllRequests();
-void WaitForAllRequestsFromOwner(const void* owner);
-
-/// Waits for all requests to finish, periodically yielding to allow other tasks to run.
-void WaitForAllRequestsWithYield(std::function<void()> before_sleep_cb, std::function<void()> after_sleep_cb);
-void WaitForAllRequestsFromOwnerWithYield(const void* owner, std::function<void()> before_sleep_cb,
-                                          std::function<void()> after_sleep_cb);
-
-/// Cancels all outstanding requests for the specified owner.
-void CancelRequestsForOwner(const void* owner);
-
 /// Returns a locked pointer to the shared cache archive, opening it on first use.
 CacheArchivePtr GetCacheArchive();
 
@@ -101,15 +74,10 @@ void Prefetch(std::string_view url);
 /// Use this to warm the cache ahead of an anticipated lookup.
 void Prefetch(std::string_view url, PrefetchCallback callback);
 
+/// Waits for all prefetches to complete.
+void WaitForAllPrefetchRequests();
+
 /// Removes all entries currently in the cache.
 bool Clear(Error* error);
 
 } // namespace HTTPCache
-
-namespace Host {
-
-/// Called by the HTTPDownloader implementation when the active state of the downloader changes.
-/// active is set if there are any requests in-progress.
-void OnHTTPCacheDownloaderActiveChanged(bool active);
-
-} // namespace Host

src/util/http_downloader.h

@@ -1,123 +1,132 @@
-// SPDX-FileCopyrightText: 2019-2025 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2026 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: CC-BY-NC-ND-4.0
 
 #pragma once
 
-#include "common/error.h"
 #include "common/types.h"
 
-#include <atomic>
 #include <functional>
-#include <memory>
-#include <mutex>
+#include <optional>
 #include <span>
 #include <string>
-#include <string_view>
 #include <vector>
 
+class Error;
 class ProgressCallback;
 
-class HTTPDownloader
+/// Global HTTP downloading subsystem. Provides a platform-specific backend
+/// (WinHTTP on Windows, libcurl elsewhere) managed as a singleton. All public
+/// functions are thread-safe: requests may be created from any thread, and
+/// callbacks are dispatched on whichever thread calls PollRequests().
+namespace HTTPDownloader {
+
+/// Span of NUL-terminated C-string HTTP header lines, e.g. {"X-Foo: bar"}.
+using HeaderList = std::span<const char* const>;
+
+/// Raw response body data.
+using RequestData = std::vector<u8>;
+
+/// Callback fired when a request completes, times out, or is cancelled.
+/// Invoked on the thread that calls PollRequests(), with no internal locks held.
+///
+/// @param status_code  HTTP status code, or one of the negative HTTP_STATUS_* sentinels on failure.
+/// @param error        Populated with a description when status_code < HTTP_STATUS_OK.
+/// @param content_type Value of the response Content-Type header; empty if unavailable.
+/// @param data         Response body; empty if the request did not succeed.
+using RequestCallback =
+  std::function<void(s32 status_code, Error& error, std::string& content_type, RequestData& data)>;
+
+/// Synthetic status codes used in place of a real HTTP status on failure.
+enum : s32
 {
-public:
-  using HeaderList = std::span<const char* const>;
-
-  static constexpr u64 WAIT_FOR_ALL_REQUESTS_POLL_INTERVAL_MS = 16;
-  static constexpr u64 WAIT_FOR_ALL_REQUESTS_POLL_INTERVAL_NS = WAIT_FOR_ALL_REQUESTS_POLL_INTERVAL_MS * 1000000;
-
-  enum : s32
-  {
-    HTTP_STATUS_CANCELLED = -3,
-    HTTP_STATUS_TIMEOUT = -2,
-    HTTP_STATUS_ERROR = -1,
-    HTTP_STATUS_OK = 200
-  };
-
-  struct Request
-  {
-    using Data = std::vector<u8>;
-    using Callback = std::function<void(s32 status_code, Error& error, std::string& content_type, Data& data)>;
-
-    enum class Type : u8
-    {
-      Get,
-      Post,
-    };
-
-    enum class State : u8
-    {
-      Pending,
-      Cancelled,
-      Started,
-      Receiving,
-      Complete,
-    };
-
-    Request(HTTPDownloader* parent, const void* owner, Type type, std::string url, std::string post_data,
-            Callback callback, ProgressCallback* progress, u16 timeout_seconds);
-    virtual ~Request();
-
-    HTTPDownloader* parent;
-    const void* owner;
-    Callback callback;
-    ProgressCallback* progress;
-    std::string url;
-    std::string post_data;
-    std::string content_type;
-    Data data;
-    Error error;
-    u64 start_time = 0;
-    u64 last_update_time = 0;
-    s32 status_code = 0;
-    u32 content_length = 0;
-    u32 last_progress_update = 0;
-    Type type = Type::Get;
-    std::atomic<State> state{State::Pending};
-    u16 timeout_seconds = 0;
-  };
-
-  HTTPDownloader();
-  virtual ~HTTPDownloader();
-
-  static std::unique_ptr<HTTPDownloader> Create(std::string user_agent, Error* error = nullptr);
-  static std::string GetExtensionForContentType(const std::string& content_type);
-
-  void SetDefaultTimeout(u16 timeout_seconds);
-  void SetMaxActiveRequests(u32 max_active_requests);
-
-  void CreateRequest(std::string url, const void* owner, Request::Callback callback,
-                     ProgressCallback* progress = nullptr, HeaderList additional_headers = {},
-                     std::optional<u16> timeout_seconds = {});
-  void CreatePostRequest(std::string url, std::string post_data, const void* owner, Request::Callback callback,
-                         ProgressCallback* progress = nullptr, HeaderList additional_headers = {},
-                         std::optional<u16> timeout_seconds = {});
-  bool PollRequests();
-  void WaitForAllRequests();
-  void WaitForAllRequestsWithYield(std::function<void()> before_sleep_cb, std::function<void()> after_sleep_cb);
-  void WaitForAllRequestsFromOwner(const void* owner);
-  void WaitForAllRequestsFromOwnerWithYield(const void* owner, std::function<void()> before_sleep_cb,
-                                            std::function<void()> after_sleep_cb);
-  bool HasAnyRequests();
-  bool HasAnyRequestsFromOwner(const void* owner);
-  void CancelAllRequests();
-  void CancelRequestsForOwner(const void* owner);
-
-protected:
-  virtual Request* InternalCreateRequest(Request::Type type, std::string url, std::string post_data, const void* owner,
-                                         Request::Callback callback, ProgressCallback* progress, u16 timeout_seconds,
-                                         HeaderList additional_headers) = 0;
-
-  virtual bool StartRequest(Request* request) = 0;
-  virtual void CloseRequest(Request* request) = 0;
-
-  void LockedAddRequest(Request* request);
-  void StartOrAddRequest(Request* request);
-  u32 LockedGetActiveRequestCount();
-  void LockedPollRequests(std::unique_lock<std::mutex>& lock);
-
-  u16 m_default_timeout;
-  u32 m_max_active_requests;
-
-  std::mutex m_pending_http_request_lock;
-  std::vector<Request*> m_pending_http_requests;
+  HTTP_STATUS_CANCELLED = -3, ///< Request was cancelled via CancelRequests*().
+  HTTP_STATUS_TIMEOUT = -2,   ///< Request exceeded its timeout duration.
+  HTTP_STATUS_ERROR = -1,     ///< Network or protocol-level error; see the error argument.
+  HTTP_STATUS_OK = 200        ///< Standard HTTP 200 OK.
 };
+
+/// Returns the file extension (without leading dot) for the given MIME type,
+/// or an empty string if the type is not recognised.
+std::string GetExtensionForContentType(const std::string& content_type);
+
+/// Sets the default timeout applied to new requests when none is explicitly provided.
+/// The initial default is 30 seconds.
+void SetDefaultTimeout(u16 timeout_seconds);
+
+/// Sets the maximum number of concurrently active (in-flight) requests.
+/// Additional requests are queued and promoted as active slots become free.
+/// Must be greater than zero. The initial default is 10.
+void SetMaxActiveRequests(u16 max_active_requests);
+
+/// Cancels all outstanding requests and tears down the HTTP backend.
+/// Each cancelled request still has its callback invoked before this returns.
+/// Safe to call even if the downloader was never initialised.
+void Shutdown();
+
+/// Queues an HTTP GET request. The callback is invoked from the thread that calls
+/// PollRequests() when the request completes, times out, or is cancelled.
+///
+/// @param owner  Opaque pointer identifying the owner, used to cancel or wait on groups
+///               of requests via CancelRequestsForOwner() / WaitForAllRequestsFromOwner().
+///               May be null. The pointer is never dereferenced.
+void CreateRequest(std::string url, const void* owner, RequestCallback callback, ProgressCallback* progress = nullptr,
+                   HeaderList additional_headers = {}, std::optional<u16> timeout_seconds = {});
+
+/// Queues an HTTP POST request with a URL-encoded body.
+/// See CreateRequest() for parameter documentation.
+void CreatePostRequest(std::string url, std::string post_data, const void* owner, RequestCallback callback,
+                       ProgressCallback* progress = nullptr, HeaderList additional_headers = {},
+                       std::optional<u16> timeout_seconds = {});
+
+/// Checks all in-flight requests for completion, timeout, or cancellation, and
+/// fires callbacks for any that have finished. Must be called periodically from
+/// the thread that should receive callbacks.
+void PollRequests();
+
+/// Blocks the calling thread, polling until all queued and in-flight requests
+/// have completed.
+void WaitForAllRequests();
+
+/// Like WaitForAllRequests(), but invokes before_sleep_cb / after_sleep_cb around
+/// each sleep interval. The internal lock is released before before_sleep_cb and
+/// re-acquired inside after_sleep_cb, allowing the caller to yield to other work.
+void WaitForAllRequestsWithYield(std::function<void()> before_sleep_cb, std::function<void()> after_sleep_cb);
+
+/// Blocks until all requests associated with owner have completed.
+/// Passing nullptr is equivalent to WaitForAllRequests().
+void WaitForAllRequestsFromOwner(const void* owner);
+
+/// Like WaitForAllRequestsFromOwner(), but invokes yield callbacks around each sleep.
+void WaitForAllRequestsFromOwnerWithYield(const void* owner, std::function<void()> before_sleep_cb,
+                                          std::function<void()> after_sleep_cb);
+
+/// Returns true if there are any requests currently queued or in-flight.
+bool HasAnyRequests();
+
+/// Returns true if there are any requests associated with owner queued or in-flight.
+/// Passing nullptr is equivalent to HasAnyRequests().
+bool HasAnyRequestsFromOwner(const void* owner);
+
+/// Cancels all outstanding requests. Equivalent to CancelRequestsForOwner(nullptr).
+void CancelAllRequests();
+
+/// Cancels all outstanding requests belonging to owner.
+/// Passing nullptr cancels every pending request (same as CancelAllRequests()).
+/// Each cancelled request still has its callback invoked with HTTP_STATUS_CANCELLED.
+/// The function loops until no matching requests remain, because a cancellation
+/// callback may itself queue new requests for the same owner.
+void CancelRequestsForOwner(const void* owner);
+
+} // namespace HTTPDownloader
+
+namespace Host {
+
+/// Returns the user agent to use for HTTP requests.
+std::string GetHTTPUserAgent();
+
+/// Called by the HTTPDownloader implementation when the active state of the downloader changes.
+/// The active parameter is set if there are any requests in-progress.
+/// NOTE: Can be called from any thread, since any thread can queue requests.
+void OnHTTPDownloaderActiveChanged(bool active);
+
+} // namespace Host

src/util/http_downloader_curl.cpp

@@ -1,374 +0,0 @@
-// SPDX-FileCopyrightText: 2019-2025 Connor McLaughlin <stenzek@gmail.com>
-// SPDX-License-Identifier: CC-BY-NC-ND-4.0
-
-#include "http_downloader.h"
-
-#include "common/assert.h"
-#include "common/log.h"
-#include "common/string_util.h"
-#include "common/threading.h"
-#include "common/timer.h"
-
-#include <algorithm>
-#include <curl/curl.h>
-#include <deque>
-#include <functional>
-#include <limits>
-#include <pthread.h>
-#include <signal.h>
-#include <thread>
-
-LOG_CHANNEL(HTTPDownloader);
-
-namespace {
-class HTTPDownloaderCurl final : public HTTPDownloader
-{
-public:
-  HTTPDownloaderCurl();
-  ~HTTPDownloaderCurl() override;
-
-  bool Initialize(std::string user_agent, Error* error);
-
-protected:
-  Request* InternalCreateRequest(Request::Type type, std::string url, std::string post_data, const void* owner,
-                                 Request::Callback callback, ProgressCallback* progress, u16 timeout_seconds,
-                                 HeaderList additional_headers) override;
-  bool StartRequest(HTTPDownloader::Request* request) override;
-  void CloseRequest(HTTPDownloader::Request* request) override;
-
-private:
-  enum class QueueAction
-  {
-    Add,
-    RemoveAndDelete
-  };
-
-  struct Request : HTTPDownloader::Request
-  {
-    using HTTPDownloader::Request::Request;
-    ~Request() override;
-
-    CURL* handle = nullptr;
-    curl_slist* header_list = nullptr;
-  };
-
-  static size_t WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata);
-
-  void WorkerThreadEntryPoint();
-
-  void ProcessQueuedActions();
-  void ReadMultiResults();
-
-  CURLM* m_multi_handle = nullptr;
-  std::string m_user_agent;
-
-  // Background thread for curl operations
-  std::thread m_worker_thread;
-  std::atomic_bool m_worker_thread_shutdown{false};
-
-  // Worker thread queue
-  std::deque<std::pair<QueueAction, Request*>> m_worker_queue;
-  std::mutex m_worker_queue_mutex;
-};
-} // namespace
-
-HTTPDownloaderCurl::HTTPDownloaderCurl() : HTTPDownloader()
-{
-}
-
-HTTPDownloaderCurl::~HTTPDownloaderCurl()
-{
-  // Signal worker thread to shutdown
-  if (m_worker_thread.joinable())
-  {
-    {
-      const std::unique_lock lock(m_worker_queue_mutex);
-      m_worker_thread_shutdown.store(true, std::memory_order_release);
-
-      // Should break the curl_multi_poll wait.
-      if (m_multi_handle)
-        curl_multi_wakeup(m_multi_handle);
-    }
-
-    m_worker_thread.join();
-  }
-
-  if (m_multi_handle)
-    curl_multi_cleanup(m_multi_handle);
-}
-
-std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(std::string user_agent, Error* error)
-{
-  std::unique_ptr<HTTPDownloaderCurl> instance = std::make_unique<HTTPDownloaderCurl>();
-  if (!instance->Initialize(std::move(user_agent), error))
-    instance.reset();
-
-  return instance;
-}
-
-static bool s_curl_initialized = false;
-static std::once_flag s_curl_initialized_once_flag;
-
-bool HTTPDownloaderCurl::Initialize(std::string user_agent, Error* error)
-{
-  if (!s_curl_initialized)
-  {
-    std::call_once(s_curl_initialized_once_flag, []() {
-      s_curl_initialized = curl_global_init(CURL_GLOBAL_ALL) == CURLE_OK;
-      if (s_curl_initialized)
-      {
-        std::atexit([]() {
-          curl_global_cleanup();
-          s_curl_initialized = false;
-        });
-      }
-    });
-    if (!s_curl_initialized)
-    {
-      Error::SetStringView(error, "curl_global_init() failed");
-      return false;
-    }
-  }
-
-  m_multi_handle = curl_multi_init();
-  if (!m_multi_handle)
-  {
-    Error::SetStringView(error, "curl_multi_init() failed");
-    return false;
-  }
-
-  m_user_agent = std::move(user_agent);
-
-  // Start the worker thread
-  m_worker_thread = std::thread(&HTTPDownloaderCurl::WorkerThreadEntryPoint, this);
-  return true;
-}
-
-size_t HTTPDownloaderCurl::WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata)
-{
-  Request* req = static_cast<Request*>(userdata);
-  const size_t current_size = req->data.size();
-  const size_t transfer_size = size * nmemb;
-  const size_t new_size = current_size + transfer_size;
-  req->data.resize(new_size);
-  req->last_update_time = Timer::GetCurrentValue();
-  std::memcpy(&req->data[current_size], ptr, transfer_size);
-
-  if (req->content_length == 0)
-  {
-    curl_off_t length;
-    if (curl_easy_getinfo(req->handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length) == CURLE_OK)
-      req->content_length = static_cast<u32>(length);
-  }
-
-  return transfer_size;
-}
-
-HTTPDownloaderCurl::Request::~Request()
-{
-  if (header_list)
-    curl_slist_free_all(header_list);
-}
-
-HTTPDownloader::Request* HTTPDownloaderCurl::InternalCreateRequest(Request::Type type, std::string url,
-                                                                   std::string post_data, const void* owner,
-                                                                   Request::Callback callback,
-                                                                   ProgressCallback* progress, u16 timeout_seconds,
-                                                                   HeaderList additional_headers)
-{
-  Request* req = new Request(this, owner, type, std::move(url), std::move(post_data), std::move(callback), progress,
-                             timeout_seconds);
-
-  for (const char* header : additional_headers)
-    req->header_list = curl_slist_append(req->header_list, header);
-
-  return req;
-}
-
-void HTTPDownloaderCurl::WorkerThreadEntryPoint()
-{
-  Threading::SetNameOfCurrentThread("HTTPDownloaderCurl Worker Thread");
-
-  // Apparently OpenSSL can fire SIGPIPE...
-  sigset_t block_mask = {};
-  sigemptyset(&block_mask);
-  sigaddset(&block_mask, SIGPIPE);
-  if (pthread_sigmask(SIG_BLOCK, &block_mask, nullptr) != 0)
-    WARNING_LOG("Failed to block SIGPIPE");
-
-  while (!m_worker_thread_shutdown.load(std::memory_order_acquire))
-  {
-    // Wait for activity with curl_multi_poll
-    CURLMcode err = curl_multi_poll(m_multi_handle, nullptr, 0, std::numeric_limits<int>::max(), nullptr);
-    if (err != CURLM_OK)
-      ERROR_LOG("curl_multi_poll() returned {}", static_cast<int>(err));
-
-    // Process any queued actions
-    ProcessQueuedActions();
-
-    // Perform curl operations
-    int running_handles;
-    err = curl_multi_perform(m_multi_handle, &running_handles);
-    if (err != CURLM_OK)
-      ERROR_LOG("curl_multi_perform() returned {}", static_cast<int>(err));
-
-    // Read any results
-    ReadMultiResults();
-  }
-}
-
-void HTTPDownloaderCurl::ProcessQueuedActions()
-{
-  const std::unique_lock lock(m_worker_queue_mutex);
-  while (!m_worker_queue.empty())
-  {
-    const auto& [action, request] = m_worker_queue.front();
-    switch (action)
-    {
-      case QueueAction::Add:
-      {
-        const CURLMcode err = curl_multi_add_handle(m_multi_handle, request->handle);
-        if (err != CURLM_OK)
-        {
-          ERROR_LOG("curl_multi_add_handle() returned {}", static_cast<int>(err));
-          request->error.SetStringFmt("curl_multi_add_handle() failed: {}", curl_multi_strerror(err));
-          request->state.store(Request::State::Complete, std::memory_order_release);
-        }
-      }
-      break;
-
-      case QueueAction::RemoveAndDelete:
-      {
-        curl_multi_remove_handle(m_multi_handle, request->handle);
-        curl_easy_cleanup(request->handle);
-        delete request;
-      }
-      break;
-    }
-
-    m_worker_queue.pop_front();
-  }
-}
-
-void HTTPDownloaderCurl::ReadMultiResults()
-{
-  for (;;)
-  {
-    int msgq;
-    struct CURLMsg* msg = curl_multi_info_read(m_multi_handle, &msgq);
-    if (!msg)
-      break;
-
-    if (msg->msg != CURLMSG_DONE)
-    {
-      WARNING_LOG("Unexpected multi message {}", static_cast<int>(msg->msg));
-      continue;
-    }
-
-    Request* req;
-    if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &req) != CURLE_OK)
-    {
-      ERROR_LOG("curl_easy_getinfo() failed");
-      continue;
-    }
-
-    if (msg->data.result == CURLE_OK)
-    {
-      long response_code = 0;
-      curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &response_code);
-      req->status_code = static_cast<s32>(response_code);
-
-      char* content_type = nullptr;
-      if (curl_easy_getinfo(req->handle, CURLINFO_CONTENT_TYPE, &content_type) == CURLE_OK && content_type)
-        req->content_type = content_type;
-
-      DEV_LOG("Request for '{}' returned status code {} and {} bytes", req->url, req->status_code, req->data.size());
-    }
-    else
-    {
-      ERROR_LOG("Request for '{}' returned error {}", req->url, static_cast<int>(msg->data.result));
-      req->error.SetStringFmt("Request failed: {}", curl_easy_strerror(msg->data.result));
-      req->status_code = HTTP_STATUS_ERROR;
-    }
-
-    req->state.store(Request::State::Complete, std::memory_order_release);
-  }
-}
-
-bool HTTPDownloaderCurl::StartRequest(HTTPDownloader::Request* request)
-{
-  Request* req = static_cast<Request*>(request);
-
-  req->handle = curl_easy_init();
-  if (!req->handle)
-  {
-    ERROR_LOG("curl_easy_init() failed");
-    req->error.SetStringView("curl_easy_init() failed");
-    req->callback(HTTP_STATUS_ERROR, req->error, req->content_type, req->data);
-    delete req;
-    return false;
-  }
-
-  curl_easy_setopt(req->handle, CURLOPT_URL, request->url.c_str());
-  curl_easy_setopt(req->handle, CURLOPT_WRITEFUNCTION, &HTTPDownloaderCurl::WriteCallback);
-  curl_easy_setopt(req->handle, CURLOPT_WRITEDATA, req);
-  curl_easy_setopt(req->handle, CURLOPT_NOSIGNAL, 1L);
-  curl_easy_setopt(req->handle, CURLOPT_PRIVATE, req);
-  curl_easy_setopt(req->handle, CURLOPT_FOLLOWLOCATION, 1L);
-
-  if (req->header_list)
-  {
-    curl_easy_setopt(req->handle, CURLOPT_HTTPHEADER, req->header_list);
-
-    // Only set the default user agent if the header list doesn't contain a User-Agent override.
-    bool has_user_agent = false;
-    for (const curl_slist* p = req->header_list; p; p = p->next)
-    {
-      if (StringUtil::Strncasecmp(p->data, "User-Agent:", 11) == 0)
-      {
-        has_user_agent = true;
-        break;
-      }
-    }
-
-    if (!has_user_agent)
-      curl_easy_setopt(req->handle, CURLOPT_USERAGENT, m_user_agent.c_str());
-  }
-  else
-  {
-    curl_easy_setopt(req->handle, CURLOPT_USERAGENT, m_user_agent.c_str());
-  }
-
-  if (request->type == Request::Type::Post)
-  {
-    curl_easy_setopt(req->handle, CURLOPT_POST, 1L);
-    curl_easy_setopt(req->handle, CURLOPT_POSTFIELDS, request->post_data.c_str());
-  }
-
-  DEV_LOG("Started HTTP request for '{}'", req->url);
-  req->state.store(Request::State::Started, std::memory_order_release);
-  req->start_time = Timer::GetCurrentValue();
-  req->last_update_time = req->start_time;
-
-  // Add to action queue for worker thread to process
-  const std::unique_lock lock(m_worker_queue_mutex);
-  m_worker_queue.emplace_back(QueueAction::Add, req);
-
-  // Wake up worker thread
-  curl_multi_wakeup(m_multi_handle);
-
-  return true;
-}
-
-void HTTPDownloaderCurl::CloseRequest(HTTPDownloader::Request* request)
-{
-  Request* req = static_cast<Request*>(request);
-  DebugAssert(req->handle);
-
-  // Add to action queue for worker thread to process
-  const std::unique_lock lock(m_worker_queue_mutex);
-  m_worker_queue.emplace_back(QueueAction::RemoveAndDelete, req);
-
-  // Wake up worker thread
-  curl_multi_wakeup(m_multi_handle);
-}

src/util/http_downloader_winhttp.cpp

@@ -1,406 +0,0 @@
-// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
-// SPDX-License-Identifier: CC-BY-NC-ND-4.0
-
-#include "http_downloader.h"
-
-#include "common/assert.h"
-#include "common/log.h"
-#include "common/string_util.h"
-#include "common/timer.h"
-
-#include <algorithm>
-
-#include "common/windows_headers.h"
-
-#include <winhttp.h>
-
-namespace {
-class HTTPDownloaderWinHttp final : public HTTPDownloader
-{
-public:
-  HTTPDownloaderWinHttp();
-  ~HTTPDownloaderWinHttp() override;
-
-  bool Initialize(std::string user_agent, Error* error);
-
-protected:
-  Request* InternalCreateRequest(Request::Type type, std::string url, std::string post_data, const void* owner,
-                                 Request::Callback callback, ProgressCallback* progress, u16 timeout_seconds,
-                                 HeaderList additional_headers) override;
-  bool StartRequest(HTTPDownloader::Request* request) override;
-  void CloseRequest(HTTPDownloader::Request* request) override;
-
-private:
-  struct Request : HTTPDownloader::Request
-  {
-    using HTTPDownloader::Request::Request;
-
-    std::wstring object_name;
-    std::wstring additional_headers;
-    HINTERNET hConnection = NULL;
-    HINTERNET hRequest = NULL;
-    u32 io_position = 0;
-  };
-
-  static void CALLBACK HTTPStatusCallback(HINTERNET hInternet, DWORD_PTR dwContext, DWORD dwInternetStatus,
-                                          LPVOID lpvStatusInformation, DWORD dwStatusInformationLength);
-
-  HINTERNET m_hSession = NULL;
-};
-} // namespace
-
-LOG_CHANNEL(HTTPDownloader);
-
-HTTPDownloaderWinHttp::HTTPDownloaderWinHttp() : HTTPDownloader()
-{
-}
-
-HTTPDownloaderWinHttp::~HTTPDownloaderWinHttp()
-{
-  if (m_hSession)
-  {
-    WinHttpSetStatusCallback(m_hSession, nullptr, WINHTTP_CALLBACK_FLAG_ALL_NOTIFICATIONS, NULL);
-    WinHttpCloseHandle(m_hSession);
-  }
-}
-
-std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(std::string user_agent, Error* error)
-{
-  std::unique_ptr<HTTPDownloaderWinHttp> instance(std::make_unique<HTTPDownloaderWinHttp>());
-  if (!instance->Initialize(std::move(user_agent), error))
-    instance.reset();
-
-  return instance;
-}
-
-bool HTTPDownloaderWinHttp::Initialize(std::string user_agent, Error* error)
-{
-  static constexpr DWORD dwAccessType = WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY;
-
-  m_hSession = WinHttpOpen(StringUtil::UTF8StringToWideString(user_agent).c_str(), dwAccessType, nullptr, nullptr,
-                           WINHTTP_FLAG_ASYNC);
-  if (m_hSession == NULL)
-  {
-    Error::SetWin32(error, "WinHttpOpen() failed: ", GetLastError());
-    return false;
-  }
-
-  const DWORD notification_flags = WINHTTP_CALLBACK_FLAG_ALL_COMPLETIONS | WINHTTP_CALLBACK_FLAG_REQUEST_ERROR |
-                                   WINHTTP_CALLBACK_FLAG_HANDLES | WINHTTP_CALLBACK_FLAG_SECURE_FAILURE;
-  if (WinHttpSetStatusCallback(m_hSession, HTTPStatusCallback, notification_flags, NULL) ==
-      WINHTTP_INVALID_STATUS_CALLBACK)
-  {
-    Error::SetWin32(error, "WinHttpSetStatusCallback() failed: ", GetLastError());
-    return false;
-  }
-
-  return true;
-}
-
-void CALLBACK HTTPDownloaderWinHttp::HTTPStatusCallback(HINTERNET hRequest, DWORD_PTR dwContext, DWORD dwInternetStatus,
-                                                        LPVOID lpvStatusInformation, DWORD dwStatusInformationLength)
-{
-  Request* req = reinterpret_cast<Request*>(dwContext);
-  switch (dwInternetStatus)
-  {
-    case WINHTTP_CALLBACK_STATUS_HANDLE_CREATED:
-      return;
-
-    case WINHTTP_CALLBACK_STATUS_HANDLE_CLOSING:
-    {
-      if (!req)
-        return;
-
-      DebugAssert(hRequest == req->hRequest);
-
-      HTTPDownloaderWinHttp* parent = static_cast<HTTPDownloaderWinHttp*>(req->parent);
-      std::unique_lock lock(parent->m_pending_http_request_lock);
-      Assert(std::none_of(parent->m_pending_http_requests.begin(), parent->m_pending_http_requests.end(),
-                          [req](HTTPDownloader::Request* it) { return it == req; }));
-
-      // we can clean up the connection as well
-      DebugAssert(req->hConnection != NULL);
-      WinHttpCloseHandle(req->hConnection);
-      delete req;
-      return;
-    }
-
-    case WINHTTP_CALLBACK_STATUS_REQUEST_ERROR:
-    {
-      const WINHTTP_ASYNC_RESULT* res = reinterpret_cast<const WINHTTP_ASYNC_RESULT*>(lpvStatusInformation);
-      ERROR_LOG("WinHttp async function {} returned error {}", res->dwResult, res->dwError);
-      req->status_code = HTTP_STATUS_ERROR;
-      req->error.SetStringFmt("WinHttp async function {} returned error {}", res->dwResult, res->dwError);
-      req->state.store(Request::State::Complete, std::memory_order_release);
-      return;
-    }
-    case WINHTTP_CALLBACK_STATUS_SENDREQUEST_COMPLETE:
-    {
-      DEBUG_LOG("SendRequest complete");
-      if (!WinHttpReceiveResponse(hRequest, nullptr))
-      {
-        const DWORD err = GetLastError();
-        ERROR_LOG("WinHttpReceiveResponse() failed: {}", err);
-        req->status_code = HTTP_STATUS_ERROR;
-        req->error.SetWin32("WinHttpReceiveResponse() failed: ", err);
-        req->state.store(Request::State::Complete, std::memory_order_release);
-      }
-
-      return;
-    }
-    case WINHTTP_CALLBACK_STATUS_HEADERS_AVAILABLE:
-    {
-      DEBUG_LOG("Headers available");
-
-      DWORD buffer_size = sizeof(req->status_code);
-      if (!WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER,
-                               WINHTTP_HEADER_NAME_BY_INDEX, &req->status_code, &buffer_size, WINHTTP_NO_HEADER_INDEX))
-      {
-        const DWORD err = GetLastError();
-        ERROR_LOG("WinHttpQueryHeaders() for status code failed: {}", err);
-        req->status_code = HTTP_STATUS_ERROR;
-        req->error.SetWin32("WinHttpQueryHeaders() failed: ", err);
-        req->state.store(Request::State::Complete, std::memory_order_release);
-        return;
-      }
-
-      buffer_size = sizeof(req->content_length);
-      if (!WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_CONTENT_LENGTH | WINHTTP_QUERY_FLAG_NUMBER,
-                               WINHTTP_HEADER_NAME_BY_INDEX, &req->content_length, &buffer_size,
-                               WINHTTP_NO_HEADER_INDEX))
-      {
-        const DWORD err = GetLastError();
-        if (err != ERROR_WINHTTP_HEADER_NOT_FOUND)
-          WARNING_LOG("WinHttpQueryHeaders() for content length failed: {}", err);
-
-        req->content_length = 0;
-      }
-
-      DWORD content_type_length = 0;
-      if (!WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_CONTENT_TYPE, WINHTTP_HEADER_NAME_BY_INDEX,
-                               WINHTTP_NO_OUTPUT_BUFFER, &content_type_length, WINHTTP_NO_HEADER_INDEX) &&
-          GetLastError() == ERROR_INSUFFICIENT_BUFFER && content_type_length >= sizeof(content_type_length))
-      {
-        std::wstring content_type_wstring;
-        content_type_wstring.resize((content_type_length / sizeof(wchar_t)) - 1);
-        if (WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_CONTENT_TYPE, WINHTTP_HEADER_NAME_BY_INDEX,
-                                content_type_wstring.data(), &content_type_length, WINHTTP_NO_HEADER_INDEX))
-        {
-          req->content_type = StringUtil::WideStringToUTF8String(content_type_wstring);
-        }
-      }
-
-      DEV_LOG("Status code {}, content-length is {}", req->status_code, req->content_length);
-      req->data.reserve(req->content_length);
-      req->state.store(Request::State::Receiving, std::memory_order_release);
-
-      // start reading
-      if (!WinHttpQueryDataAvailable(hRequest, nullptr) && GetLastError() != ERROR_IO_PENDING)
-      {
-        const DWORD err = GetLastError();
-        ERROR_LOG("WinHttpQueryDataAvailable() failed: {}", err);
-        req->status_code = HTTP_STATUS_ERROR;
-        req->error.SetWin32("WinHttpQueryDataAvailable() failed: ", err);
-        req->state.store(Request::State::Complete, std::memory_order_release);
-      }
-
-      return;
-    }
-    case WINHTTP_CALLBACK_STATUS_DATA_AVAILABLE:
-    {
-      DWORD bytes_available;
-      std::memcpy(&bytes_available, lpvStatusInformation, sizeof(bytes_available));
-      if (bytes_available == 0)
-      {
-        // end of request
-        DEV_LOG("End of request '{}', {} bytes received", req->url, req->data.size());
-        req->state.store(Request::State::Complete, std::memory_order_release);
-        return;
-      }
-
-      // start the transfer
-      DEBUG_LOG("{} bytes available", bytes_available);
-      req->io_position = static_cast<u32>(req->data.size());
-      req->data.resize(req->io_position + bytes_available);
-      if (!WinHttpReadData(hRequest, req->data.data() + req->io_position, bytes_available, nullptr) &&
-          GetLastError() != ERROR_IO_PENDING)
-      {
-        const DWORD err = GetLastError();
-        ERROR_LOG("WinHttpReadData() failed: {}", err);
-        req->status_code = HTTP_STATUS_ERROR;
-        req->error.SetWin32("WinHttpReadData() failed: ", err);
-        req->state.store(Request::State::Complete, std::memory_order_release);
-      }
-
-      return;
-    }
-    case WINHTTP_CALLBACK_STATUS_READ_COMPLETE:
-    {
-      DEBUG_LOG("Read of {} complete", dwStatusInformationLength);
-
-      const u32 new_size = req->io_position + dwStatusInformationLength;
-      Assert(new_size <= req->data.size());
-      req->data.resize(new_size);
-      req->last_update_time = Timer::GetCurrentValue();
-
-      if (!WinHttpQueryDataAvailable(hRequest, nullptr) && GetLastError() != ERROR_IO_PENDING)
-      {
-        const DWORD err = GetLastError();
-        ERROR_LOG("WinHttpQueryDataAvailable() failed: {}", err);
-        req->status_code = HTTP_STATUS_ERROR;
-        req->error.SetWin32("WinHttpQueryDataAvailable() failed: ", err);
-        req->state.store(Request::State::Complete, std::memory_order_release);
-      }
-
-      return;
-    }
-    default:
-      // unhandled, ignore
-      return;
-  }
-}
-
-HTTPDownloader::Request* HTTPDownloaderWinHttp::InternalCreateRequest(Request::Type type, std::string url,
-                                                                      std::string post_data, const void* owner,
-                                                                      Request::Callback callback,
-                                                                      ProgressCallback* progress, u16 timeout_seconds,
-                                                                      HeaderList additional_headers)
-{
-  Request* req = new Request(this, owner, type, std::move(url), std::move(post_data), std::move(callback), progress,
-                             timeout_seconds);
-
-  if (!additional_headers.empty())
-  {
-    // Pre-reserve assuming 1 wchar per UTF-8 byte, plus 2 per header for \r\n.
-    size_t total_length = 0;
-    for (const char* header : additional_headers)
-      total_length += std::strlen(header) + 2;
-    req->additional_headers.reserve(total_length);
-
-    for (const char* header : additional_headers)
-    {
-      StringUtil::AppendUTF8ToWideString(req->additional_headers, header);
-      req->additional_headers += L"\r\n";
-    }
-  }
-
-  return req;
-}
-
-bool HTTPDownloaderWinHttp::StartRequest(HTTPDownloader::Request* request)
-{
-  Request* req = static_cast<Request*>(request);
-
-  std::wstring host_name;
-  host_name.resize(req->url.size());
-  req->object_name.resize(req->url.size());
-
-  URL_COMPONENTSW uc = {};
-  uc.dwStructSize = sizeof(uc);
-  uc.lpszHostName = host_name.data();
-  uc.dwHostNameLength = static_cast<DWORD>(host_name.size());
-  uc.lpszUrlPath = req->object_name.data();
-  uc.dwUrlPathLength = static_cast<DWORD>(req->object_name.size());
-
-  const std::wstring url_wide(StringUtil::UTF8StringToWideString(req->url));
-  if (!WinHttpCrackUrl(url_wide.c_str(), static_cast<DWORD>(url_wide.size()), 0, &uc))
-  {
-    const DWORD err = GetLastError();
-    ERROR_LOG("WinHttpCrackUrl() failed: {}", err);
-    req->error.SetWin32("WinHttpCrackUrl() failed: ", err);
-    req->callback(HTTP_STATUS_ERROR, req->error, req->content_type, req->data);
-    delete req;
-    return false;
-  }
-
-  host_name.resize(uc.dwHostNameLength);
-  req->object_name.resize(uc.dwUrlPathLength);
-
-  req->hConnection = WinHttpConnect(m_hSession, host_name.c_str(), uc.nPort, 0);
-  if (!req->hConnection)
-  {
-    const DWORD err = GetLastError();
-    ERROR_LOG("Failed to start HTTP request for '{}': {}", req->url, err);
-    req->error.SetWin32("WinHttpConnect() failed: ", err);
-    req->callback(HTTP_STATUS_ERROR, req->error, req->content_type, req->data);
-    delete req;
-    return false;
-  }
-
-  const DWORD request_flags = uc.nScheme == INTERNET_SCHEME_HTTPS ? WINHTTP_FLAG_SECURE : 0;
-  req->hRequest =
-    WinHttpOpenRequest(req->hConnection, (req->type == HTTPDownloader::Request::Type::Post) ? L"POST" : L"GET",
-                       req->object_name.c_str(), NULL, NULL, NULL, request_flags);
-  if (!req->hRequest)
-  {
-    const DWORD err = GetLastError();
-    ERROR_LOG("WinHttpOpenRequest() failed: {}", err);
-    req->error.SetWin32("WinHttpOpenRequest() failed: ", err);
-    req->callback(HTTP_STATUS_ERROR, req->error, req->content_type, req->data);
-    WinHttpCloseHandle(req->hConnection);
-    delete req;
-    return false;
-  }
-
-  if (!req->additional_headers.empty() &&
-      !WinHttpAddRequestHeaders(req->hRequest, req->additional_headers.data(),
-                                static_cast<DWORD>(req->additional_headers.size()),
-                                WINHTTP_ADDREQ_FLAG_ADD | WINHTTP_ADDREQ_FLAG_REPLACE))
-  {
-    const DWORD err = GetLastError();
-    ERROR_LOG("WinHttpAddRequestHeaders() failed: {}", err);
-    req->error.SetWin32("WinHttpAddRequestHeaders() failed: ", err);
-    req->callback(HTTP_STATUS_ERROR, req->error, req->content_type, req->data);
-    WinHttpCloseHandle(req->hRequest);
-    WinHttpCloseHandle(req->hConnection);
-    delete req;
-    return false;
-  }
-
-  BOOL result;
-  if (req->type == HTTPDownloader::Request::Type::Post)
-  {
-    const std::wstring_view additional_headers(L"Content-Type: application/x-www-form-urlencoded\r\n");
-    result = WinHttpSendRequest(req->hRequest, additional_headers.data(), static_cast<DWORD>(additional_headers.size()),
-                                req->post_data.data(), static_cast<DWORD>(req->post_data.size()),
-                                static_cast<DWORD>(req->post_data.size()), reinterpret_cast<DWORD_PTR>(req));
-  }
-  else
-  {
-    result = WinHttpSendRequest(req->hRequest, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0,
-                                reinterpret_cast<DWORD_PTR>(req));
-  }
-
-  if (!result && GetLastError() != ERROR_IO_PENDING)
-  {
-    const DWORD err = GetLastError();
-    ERROR_LOG("WinHttpSendRequest() failed: {}", err);
-    req->status_code = HTTP_STATUS_ERROR;
-    req->error.SetWin32("WinHttpSendRequest() failed: ", err);
-    req->state.store(Request::State::Complete);
-  }
-
-  DEV_LOG("Started HTTP request for '{}'", req->url);
-  req->state = Request::State::Started;
-  req->start_time = Timer::GetCurrentValue();
-  req->last_update_time = req->start_time;
-  return true;
-}
-
-void HTTPDownloaderWinHttp::CloseRequest(HTTPDownloader::Request* request)
-{
-  Request* req = static_cast<Request*>(request);
-
-  if (req->hRequest != NULL)
-  {
-    // req will be freed by the callback.
-    // the callback can fire immediately here if there's nothing running async, so don't touch req afterwards
-    WinHttpCloseHandle(req->hRequest);
-    return;
-  }
-
-  if (req->hConnection != NULL)
-    WinHttpCloseHandle(req->hConnection);
-
-  delete req;
-}

src/util/util.vcxproj

@@ -153,10 +153,6 @@
     <ClCompile Include="gpu_shader_cache.cpp" />
     <ClCompile Include="gpu_texture.cpp" />
     <ClCompile Include="http_downloader.cpp" />
-    <ClCompile Include="http_downloader_curl.cpp">
-      <ExcludedFromBuild>true</ExcludedFromBuild>
-    </ClCompile>
-    <ClCompile Include="http_downloader_winhttp.cpp" />
     <ClCompile Include="image.cpp" />
     <ClCompile Include="imgui_manager.cpp" />
     <ClCompile Include="ini_settings_interface.cpp" />

src/util/util.vcxproj.filters

@@ -144,8 +144,6 @@
     <ClCompile Include="d3d11_texture.cpp" />
     <ClCompile Include="postprocessing_shader_fx.cpp" />
     <ClCompile Include="pch.cpp" />
-    <ClCompile Include="http_downloader_curl.cpp" />
-    <ClCompile Include="http_downloader_winhttp.cpp" />
     <ClCompile Include="http_downloader.cpp" />
     <ClCompile Include="metal_device.mm" />
     <ClCompile Include="metal_stream_buffer.mm" />