Make threading lock-free

The core and the headless tools no longer use mutexes, condition variables,
spinlocks, std::atomic, std::thread or volatile flags. Threads hand off through
libretro-common's retro_atomic and rthreads plus a few small pieces on top of
them in Common/Thread: Thread, MpscQueue, EventCounter, ParkingLot waits and
ReaderGate. Every wait blocks until it's woken; nothing polls on a timer.

State that only one thread touches just loses its lock. The PSP filesystem,
sceIo and the savedata dialog now belong to the emulation thread: async reads
and writes run when they're queued and complete later in emulated time, and
the IO and savedata threads are gone. The friend finder only talks to the
adhoc server and hands what it gets to the emulation thread. GPU init always
runs on the thread that starts the game.

The Vulkan render and compile queues, pipeline and render pass caches, delete
list and SPIR-V cache, the GL render queue, the software renderer's jit caches,
the file loader caches, the web debugger, UPnP, input, savestates, logging and
the GE debugger all hand off through queues and atomics. The IR jit's spinning
profiler thread goes.

A CPU thread with nothing to run now sleeps until work is queued instead of
spinning. Headless with --debugger waits for a client while stopped, and ends
the run once every client that connected has left. Web server shutdown keeps
servicing the CPU queue until the server has finished.

Core/FileLoaders/CachingFileLoader.cpp

@@ -17,10 +17,8 @@
 
 #include <algorithm>
 #include <cstring>
-#include <thread>
 
 #include "Common/Thread/ThreadUtil.h"
-#include "Common/TimeUtil.h"
 #include "Core/FileLoaders/CachingFileLoader.h"
 
 // Takes ownership of backend.
@@ -29,12 +27,14 @@ CachingFileLoader::CachingFileLoader(FileLoader *backend)
 }
 
 void CachingFileLoader::Prepare() {
-	std::call_once(preparedFlag_, [this](){
-		filesize_ = ProxiedFileLoader::FileSize();
-		if (filesize_ > 0) {
-			InitCache();
-		}
-	});
+	if (prepared_) {
+		return;
+	}
+	prepared_ = true;
+	filesize_ = ProxiedFileLoader::FileSize();
+	if (filesize_ > 0) {
+		InitCache();
+	}
 }
 
 CachingFileLoader::~CachingFileLoader() {
@@ -77,14 +77,19 @@ size_t CachingFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data, Flag
 		bytes = (size_t)(filesize_ - absolutePos);
 	}
 
+	if (!slots_) {
+		// Empty or unknown size: no cache and no worker.
+		return (flags & Flags::HINT_UNCACHED) != 0 ? backend_->ReadAt(absolutePos, bytes, data, flags) : 0;
+	}
+
 	size_t readSize = 0;
 	if ((flags & Flags::HINT_UNCACHED) != 0) {
-		readSize = backend_->ReadAt(absolutePos, bytes, data, flags);
+		readSize = RequestUncached(absolutePos, bytes, data, flags);
 	} else {
 		readSize = ReadFromCache(absolutePos, bytes, data);
 		// While in case the cache size is too small for the entire read.
 		while (readSize < bytes) {
-			SaveIntoCache(absolutePos + readSize, bytes - readSize, flags);
+			RequestIntoCache(absolutePos + readSize, bytes - readSize, flags);
 			size_t bytesFromCache = ReadFromCache(absolutePos + readSize, bytes - readSize, (u8 *)data + readSize);
 			readSize += bytesFromCache;
 			if (bytesFromCache == 0) {
@@ -102,21 +107,35 @@ size_t CachingFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data, Flag
 void CachingFileLoader::InitCache() {
 	oldestGeneration_ = 0;
 	generation_ = 0;
+	numBlocks_ = (filesize_ + BLOCK_SIZE - 1) >> BLOCK_SHIFT;
+	slots_.reset(new retro_atomic_ptr_t[(size_t)numBlocks_]);
+	for (s64 i = 0; i < numBlocks_; ++i) {
+		retro_atomic_ptr_init(&slots_[(size_t)i], nullptr);
+	}
+	generations_.reset(new u64[(size_t)numBlocks_]());
+	worker_ = Thread([this] {
+		WorkerLoop();
+	});
 }
 
 void CachingFileLoader::ShutdownCache() {
-	// We can't delete while the read-ahead thread is running; the join waits for it.
-	if (aheadThread_.joinable())
-		aheadThread_.join();
-
-	std::lock_guard<std::recursive_mutex> guard(blocksMutex_);
-	for (const auto &block : blocks_) {
-		delete [] block.second.ptr;
+	retro_atomic_store_release_int(&quit_, 1);
+	wake_.Notify();
+	// Waits out any backend read in progress.
+	if (worker_.joinable()) {
+		worker_.join();
 	}
-	blocks_.clear();
+
+	for (s64 i = 0; i < numBlocks_; ++i) {
+		delete [] BlockAt(i);
+	}
+	slots_.reset();
 }
 
 size_t CachingFileLoader::ReadFromCache(s64 pos, size_t bytes, void *data) {
+	if (bytes == 0) {
+		return 0;
+	}
 	s64 cacheStartPos = pos >> BLOCK_SHIFT;
 	s64 cacheEndPos = (pos + bytes - 1) >> BLOCK_SHIFT;
 	// TODO: Smarter.
@@ -124,16 +143,15 @@ size_t CachingFileLoader::ReadFromCache(s64 pos, size_t bytes, void *data) {
 	size_t offset = (size_t)(pos - (cacheStartPos << BLOCK_SHIFT));
 	u8 *p = (u8 *)data;
 
-	std::lock_guard<std::recursive_mutex> guard(blocksMutex_);
 	for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) {
-		auto block = blocks_.find(i);
-		if (block == blocks_.end()) {
+		const u8 *block = BlockAt(i);
+		if (!block) {
 			return readSize;
 		}
-		block->second.generation = generation_;
+		generations_[(size_t)i] = generation_;
 
 		size_t toRead = std::min(bytes - readSize, (size_t)BLOCK_SIZE - offset);
-		memcpy(p + readSize, block->second.ptr + offset, toRead);
+		memcpy(p + readSize, block + offset, toRead);
 		readSize += toRead;
 
 		// Don't need an offset after the first read.
@@ -142,17 +160,105 @@ size_t CachingFileLoader::ReadFromCache(s64 pos, size_t bytes, void *data) {
 	return readSize;
 }
 
+size_t CachingFileLoader::RunRequest() {
+	const int seq = (int)((unsigned)reqSeqLocal_ + 1);
+	reqSeqLocal_ = seq;
+	retro_atomic_store_release_int(&reqSeq_, seq);
+	wake_.Notify();
+	ParkingLotWait(&doneSeq_, [&] {
+		return retro_atomic_load_acquire_int(&doneSeq_) == seq;
+	});
+	return reqResult_;
+}
+
+size_t CachingFileLoader::RequestUncached(s64 pos, size_t bytes, void *data, Flags flags) {
+	reqPos_ = pos;
+	reqBytes_ = bytes;
+	reqData_ = data;
+	reqFlags_ = flags;
+	return RunRequest();
+}
+
+void CachingFileLoader::RequestIntoCache(s64 pos, size_t bytes, Flags flags) {
+	s64 cacheStartPos = pos >> BLOCK_SHIFT;
+	s64 cacheEndPos = std::min((pos + (s64)bytes - 1) >> BLOCK_SHIFT, numBlocks_ - 1);
+
+	size_t blocksToRead = 0;
+	for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) {
+		if (BlockAt(i)) {
+			break;
+		}
+		++blocksToRead;
+		if (blocksToRead >= MAX_BLOCKS_PER_READ) {
+			break;
+		}
+	}
+	if (blocksToRead == 0) {
+		return;
+	}
+
+	MakeCacheSpaceFor(blocksToRead);
+	reqPos_ = pos;
+	reqBytes_ = bytes;
+	reqData_ = nullptr;
+	reqFlags_ = flags;
+	RunRequest();
+	++generation_;
+}
+
+void CachingFileLoader::WorkerLoop() {
+	SetCurrentThreadName("FileLoaderReadAhead");
+
+	int done = 0;
+	for (;;) {
+		const int seen = wake_.Seen();
+		if (retro_atomic_load_acquire_int(&quit_)) {
+			break;
+		}
+
+		const int seq = retro_atomic_load_acquire_int(&reqSeq_);
+		if (seq != done) {
+			if (reqData_) {
+				reqResult_ = backend_->ReadAt(reqPos_, reqBytes_, reqData_, reqFlags_);
+			} else {
+				SaveIntoCache(reqPos_, reqBytes_, reqFlags_);
+			}
+			done = seq;
+			retro_atomic_store_release_int(&doneSeq_, seq);
+			ParkingLotNotify(&doneSeq_);
+			continue;
+		}
+
+		const int ahead = retro_atomic_exchange_int(&aheadBlock_, -1);
+		if (ahead >= 0) {
+			ReadAhead(ahead);
+			continue;
+		}
+
+		wake_.Wait(seen);
+	}
+}
+
+void CachingFileLoader::ReadAhead(s64 cacheStartPos) {
+	s64 cacheEndPos = std::min(cacheStartPos + BLOCK_READAHEAD - 1, numBlocks_ - 1);
+	for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) {
+		if (!BlockAt(i)) {
+			SaveIntoCache(i << BLOCK_SHIFT, BLOCK_SIZE * BLOCK_READAHEAD, Flags::NONE, true);
+			break;
+		}
+	}
+}
+
 void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes, Flags flags, bool readingAhead) {
 	s64 cacheStartPos = pos >> BLOCK_SHIFT;
 	s64 cacheEndPos = (pos + bytes - 1) >> BLOCK_SHIFT;
 	// Read-ahead can ask for blocks past the end of the file.
-	cacheEndPos = std::min(cacheEndPos, (filesize_ - 1) >> BLOCK_SHIFT);
+	cacheEndPos = std::min(cacheEndPos, numBlocks_ - 1);
 
-	std::lock_guard<std::recursive_mutex> guard(blocksMutex_);
+	// Only this thread fills slots, so the empty ones counted here stay empty until we fill them.
 	size_t blocksToRead = 0;
 	for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) {
-		auto block = blocks_.find(i);
-		if (block != blocks_.end()) {
+		if (BlockAt(i)) {
 			break;
 		}
 		++blocksToRead;
@@ -161,36 +267,30 @@ void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes, Flags flags, bool r
 		}
 	}
 
-	if (!MakeCacheSpaceFor(blocksToRead, readingAhead) || blocksToRead == 0) {
+	if (blocksToRead == 0) {
+		return;
+	}
+	// Read-ahead never evicts; the reader makes room before a demand read.
+	if (readingAhead && (size_t)retro_atomic_load_acquire_int(&cachedCount_) + blocksToRead > MAX_BLOCKS_CACHED) {
 		return;
 	}
 
 	if (blocksToRead == 1) {
-		blocksMutex_.unlock();
-
 		u8 *buf = new u8[BLOCK_SIZE];
 		size_t readBytes = backend_->ReadAt(cacheStartPos << BLOCK_SHIFT, BLOCK_SIZE, buf, flags);
 
-		blocksMutex_.lock();
-		// While blocksMutex_ was unlocked, another thread may have read.
-		// If so, free the one we just read.
-		if (blocks_.find(cacheStartPos) == blocks_.end()) {
-			// Only cache a block we actually fully read - a short/failed read (e.g. a
-			// dropped connection on a Remote ISO) must not be cached as if valid, or
-			// every later read of this block would silently return the uninitialized
-			// tail of `buf` as if it were real file data. The last block of the file
-			// is shorter, and complete if the read reached the end.
-			if (readBytes == BLOCK_SIZE || (readBytes > 0 && (cacheStartPos << BLOCK_SHIFT) + (s64)readBytes == filesize_)) {
-				blocks_[cacheStartPos] = BlockInfo{buf};
-			} else {
-				delete [] buf;
-			}
+		// Only cache a block we actually fully read - a short/failed read (e.g. a
+		// dropped connection on a Remote ISO) must not be cached as if valid, or
+		// every later read of this block would silently return the uninitialized
+		// tail of `buf` as if it were real file data. The last block of the file
+		// is shorter, and complete if the read reached the end.
+		if (readBytes == BLOCK_SIZE || (readBytes > 0 && (cacheStartPos << BLOCK_SHIFT) + (s64)readBytes == filesize_)) {
+			retro_atomic_store_release_ptr(&slots_[(size_t)cacheStartPos], buf);
+			retro_atomic_fetch_add_int(&cachedCount_, 1);
 		} else {
 			delete [] buf;
 		}
 	} else {
-		blocksMutex_.unlock();
-
 		u8 *wholeRead = new u8[blocksToRead << BLOCK_SHIFT];
 		size_t readBytes = backend_->ReadAt(cacheStartPos << BLOCK_SHIFT, blocksToRead << BLOCK_SHIFT, wholeRead, flags);
 		size_t wholeBlocksRead = readBytes >> BLOCK_SHIFT;
@@ -199,97 +299,67 @@ void CachingFileLoader::SaveIntoCache(s64 pos, size_t bytes, Flags flags, bool r
 			wholeBlocksRead++;
 		}
 
-		blocksMutex_.lock();
 		for (size_t i = 0; i < wholeBlocksRead; ++i) {
-			if (blocks_.find(cacheStartPos + i) != blocks_.end()) {
-				// Written while we were busy, just skip it.  Keep the existing block.
-				continue;
-			}
 			u8 *buf = new u8[BLOCK_SIZE];
 			memcpy(buf, wholeRead + (i << BLOCK_SHIFT), BLOCK_SIZE);
-			blocks_[cacheStartPos + i] = BlockInfo{buf};
+			retro_atomic_store_release_ptr(&slots_[(size_t)cacheStartPos + i], buf);
+			retro_atomic_fetch_add_int(&cachedCount_, 1);
 		}
 		delete[] wholeRead;
 	}
-
-	++generation_;
 }
 
-bool CachingFileLoader::MakeCacheSpaceFor(size_t blocks, bool readingAhead) {
+void CachingFileLoader::MakeCacheSpaceFor(size_t blocks) {
 	size_t goal = MAX_BLOCKS_CACHED - blocks;
 
-	if (readingAhead && blocks_.size() > goal) {
-		return false;
-	}
-
-	std::lock_guard<std::recursive_mutex> guard(blocksMutex_);
-	while (blocks_.size() > goal) {
+	while ((size_t)retro_atomic_load_acquire_int(&cachedCount_) > goal) {
 		u64 minGeneration = generation_;
 
-		// We increment the iterator inside because we delete things inside.
-		for (auto it = blocks_.begin(); it != blocks_.end(); ) {
+		for (s64 i = 0; i < numBlocks_; ++i) {
+			u8 *block = BlockAt(i);
+			if (!block) {
+				continue;
+			}
+			const u64 generation = generations_[(size_t)i];
 			// Check for the minimum seen generation.
 			// TODO: Do this smarter?
-			if (it->second.generation != 0 && it->second.generation < minGeneration) {
-				minGeneration = it->second.generation;
+			if (generation != 0 && generation < minGeneration) {
+				minGeneration = generation;
 			}
 
 			// 0 means it was never used yet or was the first read (e.g. block descriptor.)
-			if (it->second.generation == oldestGeneration_ || it->second.generation == 0) {
-				s64 pos = it->first;
-				delete [] it->second.ptr;
-				blocks_.erase(it);
-
-				// Our iterator is invalid now.  Keep going?
-				if (blocks_.size() > goal) {
-					// This finds the one at that position.
-					it = blocks_.lower_bound(pos);
-				} else {
+			if (generation == oldestGeneration_ || generation == 0) {
+				// The worker never reads a published block, so it can go right away.
+				retro_atomic_store_release_ptr(&slots_[(size_t)i], nullptr);
+				generations_[(size_t)i] = 0;
+				delete [] block;
+				if ((size_t)(retro_atomic_fetch_sub_int(&cachedCount_, 1) - 1) <= goal) {
 					break;
 				}
-			} else {
-				++it;
 			}
 		}
 
 		// If we didn't find any, update to the lowest we did find.
 		oldestGeneration_ = minGeneration;
 	}
-
-	return true;
 }
 
 void CachingFileLoader::StartReadAhead(s64 pos) {
-	std::lock_guard<std::recursive_mutex> guard(blocksMutex_);
-	if (aheadThreadRunning_) {
-		// Already going.
+	s64 cacheStartPos = pos >> BLOCK_SHIFT;
+	if (cacheStartPos >= numBlocks_) {
 		return;
 	}
-	if (blocks_.size() + BLOCK_READAHEAD > MAX_BLOCKS_CACHED) {
+	if ((size_t)retro_atomic_load_acquire_int(&cachedCount_) + BLOCK_READAHEAD > MAX_BLOCKS_CACHED) {
 		// Not enough space to readahead.
 		return;
 	}
-
-	aheadThreadRunning_ = true;
-	if (aheadThread_.joinable())
-		aheadThread_.join();
-	aheadThread_ = std::thread([this, pos] {
-		SetCurrentThreadName("FileLoaderReadAhead");
-
-
-		std::unique_lock<std::recursive_mutex> guard(blocksMutex_);
-		s64 cacheStartPos = pos >> BLOCK_SHIFT;
-		s64 cacheEndPos = cacheStartPos + BLOCK_READAHEAD - 1;
-
-		for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) {
-			auto block = blocks_.find(i);
-			if (block == blocks_.end()) {
-				guard.unlock();
-				SaveIntoCache(i << BLOCK_SHIFT, BLOCK_SIZE * BLOCK_READAHEAD, Flags::NONE, true);
-				break;
-			}
+	s64 cacheEndPos = std::min(cacheStartPos + BLOCK_READAHEAD - 1, numBlocks_ - 1);
+	for (s64 i = cacheStartPos; i <= cacheEndPos; ++i) {
+		if (!BlockAt(i)) {
+			// A newer position replaces one the worker hasn't picked up yet.
+			retro_atomic_store_release_int(&aheadBlock_, (int)cacheStartPos);
+			wake_.Notify();
+			return;
 		}
-
-		aheadThreadRunning_ = false;
-	});
+	}
 }