diff --git a/src/duckstation-qt/svgiconengine.cpp b/src/duckstation-qt/svgiconengine.cpp
index b6888d39b..d277dcbac 100644
--- a/src/duckstation-qt/svgiconengine.cpp
+++ b/src/duckstation-qt/svgiconengine.cpp
@@ -130,6 +130,9 @@ SVGIconEngine::~SVGIconEngine()
 
 bool SVGIconEngine::ensureLoaded() const
 {
+  if (m_document)
+    return true;
+
   // Previous load failed?
   if (m_resource_path.isEmpty())
     return false;
@@ -171,7 +174,7 @@ QPixmap SVGIconEngine::getPixmap(const QSize& size, qreal dpr, QIcon::Mode mode,
   if (!QPixmapCache::find(cache_key, &pm))
   {
     // Don't reload multiple times if we hit the cache.
-    if (ensureLoaded() && RenderSVGToPixmap(pm, m_document, scaled_size, color))
+    if ((m_document || ensureLoaded()) && RenderSVGToPixmap(pm, m_document, scaled_size, color))
     {
       if (!m_is_colored && mode != QIcon::Normal)
       {
@@ -241,6 +244,14 @@ QIconEngine* SVGIconEnginePlugin::create(const QString& resource_path)
   return nullptr;
 }
 
+SVGImageHandler::SVGImageHandler() = default;
+
+SVGImageHandler::~SVGImageHandler()
+{
+  if (m_document)
+    plutosvg_document_destroy(m_document);
+}
+
 bool SVGImageHandler::canRead() const
 {
   if (m_document)
diff --git a/src/duckstation-qt/svgiconengine.h b/src/duckstation-qt/svgiconengine.h
index b5121fe32..ab7d7524e 100644
--- a/src/duckstation-qt/svgiconengine.h
+++ b/src/duckstation-qt/svgiconengine.h
@@ -76,8 +76,8 @@ public:
 class SVGImageHandler final : public QImageIOHandler
 {
 public:
-  SVGImageHandler() = default;
-  ~SVGImageHandler() override = default;
+  SVGImageHandler();
+  ~SVGImageHandler() override;
 
   /// Lazily load and parse the SVG data from device(). Returns false on failure.
   bool canRead() const override;