diff --git a/Common/UI/IconCache.cpp b/Common/UI/IconCache.cpp
index 451525c0b9..a2697897d0 100644
--- a/Common/UI/IconCache.cpp
+++ b/Common/UI/IconCache.cpp
@@ -26,8 +26,6 @@ struct DiskCacheEntry {
 };
 
 void IconCache::SaveToFile(FILE *file) {
-	Decimate();
-
 	std::unique_lock<std::mutex> lock(lock_);
 
 	DiskCacheHeader header;
@@ -115,9 +113,24 @@ void IconCache::ClearData() {
 	cache_.clear();
 }
 
-void IconCache::Decimate() {
+void IconCache::FrameUpdate() {
 	std::unique_lock<std::mutex> lock(lock_);
-
+	// Remove old textures after a while.
+	double now = time_now_d();
+	if (now > lastUpdate_ + 2.0) {
+		for (auto &iter : cache_) {
+			double useAge = now - iter.second.usedTimeStamp;
+			if (useAge > 5.0) {
+				// Release the texture after a few seconds of no use.
+				// Still, keep the png data loaded, it's small.
+				if (iter.second.texture) {
+					iter.second.texture->Release();
+					iter.second.texture = nullptr;
+				}
+			}
+		}
+		lastUpdate_ = now;
+	}
 }
 
 bool IconCache::GetDimensions(const std::string &key, int *width, int *height) {
@@ -177,6 +190,10 @@ bool IconCache::InsertIcon(const std::string &key, IconFormat format, std::strin
 		return false;
 	}
 
+	if (data.size() > 1024 * 512) {
+		WARN_LOG(G3D, "Unusually large icon inserted in icon cache: %s (%d bytes)", key.c_str(), (int)data.size());
+	}
+
 	pending_.erase(key);
 
 	double now = time_now_d();
@@ -240,6 +257,7 @@ Draw::Texture *IconCache::BindIconTexture(UIContext *context, const std::string
 
 	Draw::Texture *texture = context->GetDrawContext()->CreateTexture(iconDesc);
 	iter->second.texture = texture;
+	iter->second.usedTimeStamp = time_now_d();
 
 	free(buffer);
 
diff --git a/Common/UI/IconCache.h b/Common/UI/IconCache.h
index f1dbe397a2..82f8a2987c 100644
--- a/Common/UI/IconCache.h
+++ b/Common/UI/IconCache.h
@@ -30,6 +30,8 @@ struct IconCacheStats {
 
 class IconCache {
 public:
+	// NOTE: Don't store the returned texture. Only use it to look up dimensions or other properties,
+	// instead call BindIconTexture every time you want to use it.
 	Draw::Texture *BindIconTexture(UIContext *context, const std::string &key);
 
 	// It's okay to call these from any thread.
@@ -42,6 +44,8 @@ public:
 	void SaveToFile(FILE *file);
 	bool LoadFromFile(FILE *file);
 
+	void FrameUpdate();
+
 	void ClearTextures();
 	void ClearData();
 
@@ -57,12 +61,12 @@ private:
 		bool badData;
 	};
 
-	void Decimate();
-
 	std::map<std::string, Entry> cache_;
 	std::set<std::string> pending_;
 
 	std::mutex lock_;
+
+	double lastUpdate_ = 0.0;
 };
 
 extern IconCache g_iconCache;
diff --git a/Common/UI/Screen.cpp b/Common/UI/Screen.cpp
index 6e888c63cd..664f3f9275 100644
--- a/Common/UI/Screen.cpp
+++ b/Common/UI/Screen.cpp
@@ -52,6 +52,8 @@ void ScreenManager::update() {
 	if (stack_.size()) {
 		stack_.back().screen->update();
 	}
+
+	g_iconCache.FrameUpdate();
 }
 
 void ScreenManager::switchToNext() {