Give the Vulkan allocator one owning thread

Readback buffers are allocated when the readback is recorded rather than on the render thread,
so only the thread that records frames touches VMA. The allocator is created externally
synchronized, and VMA is built with no-op mutexes and plain counters instead of std::mutex and
std::atomic.

Headless runs Vulkan offscreen in builds without SDL too, and CI runs the headless tests on
lavapipe, skipping the pixel tests a hardware backend doesn't match the software references on.
sleep_ms and sleep_us go, with the thread manager test waiting on an event instead, and so does
the unused portable-file-dialogs header.

.github/workflows/build.yml

@@ -345,7 +345,7 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         sudo apt-get update -y -qq
-        sudo apt-get install libsdl3-dev libgl1-mesa-dev libglu1-mesa-dev libsdl3-ttf-dev libfontconfig1-dev libcurl4-openssl-dev
+        sudo apt-get install libsdl3-dev libgl1-mesa-dev libglu1-mesa-dev libsdl3-ttf-dev libfontconfig1-dev libcurl4-openssl-dev mesa-vulkan-drivers
 
     - name: Install macOS dependencies
       if: runner.os == 'macOS'
@@ -390,6 +390,12 @@ jobs:
         done
         exit $fail
 
+    # Runs the Vulkan backend's render and compile threads on lavapipe.
+    - name: Execute headless tests (Vulkan)
+      if: matrix.os == 'ubuntu-26.04'
+      working-directory: ${{ env.GITHUB_WORKSPACE }}
+      run: python3 test.py -g --graphics=vulkan --known-failures=vulkan
+
     # x86-64 only: the reference images don't match what the software renderer produces on arm64,
     # 23 of the 30 dumps differ there, one of them by an MSE of 100. That's a real difference and
     # worth chasing, but it predates the arm64 runner - it just never had a CI job to show up in.

CMakeLists.txt

@@ -668,7 +668,7 @@ endif(USE_FFMPEG)
 
 if(LIBRETRO)
 elseif(HEADLESS_CROSS OR WIN32 OR ANDROID OR IOS)
-	# No SDL3 here: the headless tools run with --graphics=software only.
+	# No SDL3 here: the headless tools run with --graphics=software or vulkan only.
 	add_definitions(-DHEADLESS_NO_SDL)
 else()
 	# The headless tools create their GL contexts with SDL.

Common/GPU/Vulkan/VulkanContext.cpp

@@ -926,6 +926,9 @@ VkResult VulkanContext::CreateDevice(int physical_device, const std::vector<cons
 	allocatorInfo.physicalDevice = physical_devices_[physical_device_];
 	allocatorInfo.device = device_;
 	allocatorInfo.instance = instance_;
+	// Only the thread that records frames allocates and frees (the render thread hands readbacks to
+	// it, deletes are queued), so the allocator needs no internal locking.
+	allocatorInfo.flags = VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT;
 	VkResult result = vmaCreateAllocator(&allocatorInfo, &allocator_);
 	_assert_(result == VK_SUCCESS);
 	_assert_(allocator_ != VK_NULL_HANDLE);

Common/GPU/Vulkan/VulkanContext.h

@@ -76,7 +76,7 @@ typedef std::function<void(VulkanContext *)> DeleteCallback;
 // This is a bit repetitive...
 //
 // Thread safety: the queueing functions can be called from any thread - the render thread queues
-// deletes too (VulkanDescSetPool::Recreate from FlushDescSets, ResizeReadbackBuffer). They push onto
+// deletes too (VulkanDescSetPool::Recreate from FlushDescSets). They push onto
 // a lock-free MPSC queue. Take() and PerformDeletes() belong to one thread, the main thread, which
 // moves the queued items into the per-kind vectors.
 class VulkanDeleteList {

Common/GPU/Vulkan/VulkanFrameData.h

@@ -55,6 +55,8 @@ struct CachedReadback {
 	VmaAllocation allocation;
 	VkDeviceSize bufferSize;
 	bool isCoherent;
+	// frameId of the frame that created it; holds no data until that frame comes round again.
+	uint64_t createdFrame;
 
 	void Destroy(VulkanContext *vulkan);
 };

Common/GPU/Vulkan/VulkanQueueRunner.cpp

@@ -1703,6 +1703,26 @@ void VulkanQueueRunner::ResizeReadbackBuffer(CachedReadback *readback, VkDeviceS
 	readback->isCoherent = (memoryType.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
 }
 
+CachedReadback *VulkanQueueRunner::PrepareReadback(FrameData &frameData, VKRFramebuffer *src, int width, int height, bool delayed) {
+	// TODO: Handle different readback formats!
+	const VkDeviceSize size = sizeof(uint32_t) * (VkDeviceSize)width * (VkDeviceSize)height;
+	CachedReadback *cached = &syncReadback_;
+	if (delayed) {
+		ReadbackKey key;
+		key.framebuf = src;
+		key.width = width;
+		key.height = height;
+		if (!frameData.readbacks_.Get(key, &cached)) {
+			cached = new CachedReadback();
+			cached->bufferSize = 0;
+			cached->createdFrame = frameData.frameId;
+			frameData.readbacks_.Insert(key, cached);
+		}
+	}
+	ResizeReadbackBuffer(cached, size);
+	return cached;
+}
+
 void VulkanQueueRunner::PerformReadback(const VKRStep &step, VkCommandBuffer cmd, FrameData &frameData) {
 	VkImage image;
 	VkImageLayout copyLayout;
@@ -1736,28 +1756,7 @@ void VulkanQueueRunner::PerformReadback(const VKRStep &step, VkCommandBuffer cmd
 
 	recordBarrier_.Flush(cmd);
 
-	// TODO: Handle different readback formats!
-	u32 readbackSizeInBytes = sizeof(uint32_t) * step.readback.srcRect.extent.width * step.readback.srcRect.extent.height;
-
-	CachedReadback *cached = nullptr;
-
-	if (step.readback.delayed) {
-		ReadbackKey key;
-		key.framebuf = step.readback.src;
-		key.width = step.readback.srcRect.extent.width;
-		key.height = step.readback.srcRect.extent.height;
-
-		// See if there's already a buffer we can reuse
-		if (!frameData.readbacks_.Get(key, &cached)) {
-			cached = new CachedReadback();
-			cached->bufferSize = 0;
-			frameData.readbacks_.Insert(key, cached);
-		}
-	} else {
-		cached = &syncReadback_;
-	}
-
-	ResizeReadbackBuffer(cached, readbackSizeInBytes);
+	CachedReadback *cached = step.readback.dst;
 
 	VkBufferImageCopy region{};
 	region.imageOffset = { step.readback.srcRect.offset.x, step.readback.srcRect.offset.y, 0 };
@@ -1792,8 +1791,6 @@ void VulkanQueueRunner::PerformReadbackImage(const VKRStep &step, VkCommandBuffe
 	recordBarrier_.TransitionColorImageAuto(step.readback_image.image, &layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 1, 1);
 	recordBarrier_.Flush(cmd);
 
-	ResizeReadbackBuffer(&syncReadback_, sizeof(uint32_t) * step.readback_image.srcRect.extent.width * step.readback_image.srcRect.extent.height);
-
 	VkBufferImageCopy region{};
 	region.imageOffset = { step.readback_image.srcRect.offset.x, step.readback_image.srcRect.offset.y, 0 };
 	region.imageExtent = { step.readback_image.srcRect.extent.width, step.readback_image.srcRect.extent.height, 1 };
@@ -1827,7 +1824,7 @@ bool VulkanQueueRunner::CopyReadbackBuffer(FrameData &frameData, VKRFramebuffer
 		key.width = width;
 		key.height = height;
 		CachedReadback *cached;
-		if (frameData.readbacks_.Get(key, &cached)) {
+		if (frameData.readbacks_.Get(key, &cached) && cached->createdFrame != frameData.frameId) {
 			readback = cached;
 		} else {
 			// Didn't have a cached image ready yet

Common/GPU/Vulkan/VulkanQueueRunner.h

@@ -187,6 +187,7 @@ struct VKRStep {
 			VkRect2D srcRect;
 			VkImageAspectFlags aspectMask;
 			bool delayed;
+			CachedReadback *dst;
 		} readback;
 		struct {
 			VkImage image;
@@ -245,6 +246,10 @@ public:
 		return (int)depth * 3 + (int)color;
 	}
 
+	// Readback buffers are allocated here on the recording thread, so the render thread never
+	// touches the allocator. src == nullptr or !delayed means the sync readback buffer.
+	CachedReadback *PrepareReadback(FrameData &frameData, VKRFramebuffer *src, int width, int height, bool delayed);
+
 	// src == 0 means to copy from the sync readback buffer.
 	bool CopyReadbackBuffer(FrameData &frameData, VKRFramebuffer *src, int width, int height, Draw::DataFormat srcFormat, Draw::DataFormat destFormat, int pixelStride, uint8_t *pixels);
 

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -1147,6 +1147,7 @@ bool VulkanRenderManager::CopyFramebufferToMemory(VKRFramebuffer *src, VkImageAs
 	step->readback.srcRect.offset = { x, y };
 	step->readback.srcRect.extent = { (uint32_t)w, (uint32_t)h };
 	step->readback.delayed = mode == Draw::ReadbackMode::OLD_DATA_OK;
+	step->readback.dst = queueRunner_.PrepareReadback(frameData_[vulkan_->GetCurFrame()], src, w, h, step->readback.delayed);
 	step->dependencies.insert(src);
 	step->tag = tag;
 	steps_.push_back(step);
@@ -1207,6 +1208,7 @@ void VulkanRenderManager::CopyImageToMemorySync(VkImage image, int mipLevel, int
 	step->readback_image.srcRect.extent = { (uint32_t)w, (uint32_t)h };
 	step->readback_image.mipLevel = mipLevel;
 	step->tag = tag;
+	queueRunner_.PrepareReadback(frameData_[vulkan_->GetCurFrame()], nullptr, w, h, false);
 	steps_.push_back(step);
 
 	FlushSync();

Common/TimeUtil.cpp

@@ -6,20 +6,11 @@
 #include "Common/TimeUtil.h"
 #include "Common/Log.h"
 
-#ifdef HAVE_LIBNX
-#include <switch.h>
-#endif // HAVE_LIBNX
-
-#ifdef __EMSCRIPTEN__
-#include <emscripten/emscripten.h>
-#endif // __EMSCRIPTEN__
-
 #ifdef _WIN32
 #include "CommonWindows.h"
 #include <sys/timeb.h>
 #else
 #include <sys/time.h>
-#include <unistd.h>
 #endif
 
 #include <ctime>
@@ -257,45 +248,6 @@ double Instant::ElapsedSeconds() const {
 
 #endif
 
-#ifndef __LIBRETRO__
-// The headless tools and tests only; see TimeUtil.h.
-void sleep_ms(int ms, const char *reason) {
-	if (ms <= 0) {
-		return;
-	}
-#if SLEEP_LOG_ENABLED
-	INFO_LOG(Log::System, "Sleep %d ms: %s", ms, reason);
-#endif
-#ifdef _WIN32
-	Sleep(ms);
-#elif defined(HAVE_LIBNX)
-	svcSleepThread(ms * 1000000);
-#elif defined(__EMSCRIPTEN__)
-	emscripten_sleep(ms);
-#else
-	usleep(ms * 1000);
-#endif
-}
-
-void sleep_us(int us, const char *reason) {
-	if (us <= 0) {
-		return;
-	}
-#if SLEEP_LOG_ENABLED
-	INFO_LOG(Log::System, "Sleep %d us: %s", us, reason);
-#endif
-#ifdef _WIN32
-	Sleep(us / 1000);
-#elif defined(HAVE_LIBNX)
-	svcSleepThread(us * 1000);
-#elif defined(__EMSCRIPTEN__)
-	emscripten_sleep(us / 1000);
-#else
-	usleep(us);
-#endif
-}
-#endif
-
 // Return the current time formatted as Minutes:Seconds:Milliseconds
 // in the form 00:00:000.
 void GetCurrentTimeFormatted(char formattedTime[13]) {

Common/TimeUtil.h

@@ -19,13 +19,6 @@ double from_time_raw_relative(uint64_t raw_time);
 double time_now_unix_utc();
 double time_to_unix_utc(double timeNowSeconds);
 
-#ifndef __LIBRETRO__
-// Sleeps, for the standalone frontends, headless tools and tests. The core never sleeps and
-// doesn't have this, so a sleep can't creep back into it.
-void sleep_ms(int ms, const char *reason);
-void sleep_us(int us, const char *reason);
-#endif
-
 void GetCurrentTimeFormatted(char formattedTime[13]);
 
 // Most accurate timer possible - no extra double conversions. Only for spans.

Core/HLE/sceNetAdhocMatching.cpp

@@ -1131,18 +1131,9 @@ static void MatchingEventPass(SceNetAdhocMatchingContext *context, int matchingI
 			// Log Matching Events
 			INFO_LOG(Log::sceNet, "EventLoop[%d]: Matching Event [%d=%s][%s] OptSize=%d", matchingId, msg->opcode, getMatchingEventStr(msg->opcode), mac2str(&msg->mac).c_str(), msg->optlen);
 
-			// Unlock to prevent race-condition with other threads due to recursive lock
-			//			// Call Event Handler
-			//context->handler(context->id, msg->opcode, &msg->mac, msg->optlen, opt);
 			// Notify Event Handlers
-			notifyMatchingHandler(context, msg, opt, bufAddr, bufLen, args); // If we're using shared Buffer & Args for All Events We should wait for the Mipscall to be fully executed before processing the next event. GTA VCS need this delay/sleep.
+			notifyMatchingHandler(context, msg, opt, bufAddr, bufLen, args);
 
-			// Give some time before executing the next mipscall to prevent event ACCEPT(6)->ESTABLISH(7) getting reversed After Action ESTABLISH(7)->ACCEPT(6)
-			// Must Not be delayed too long to prevent desync/disconnect. Not longer than the delays on callback's HLE?
-			//sleep_ms(10); //sceKernelDelayThread(10000);
-
-			// Lock again
-			//
 			// Pop event stack from front (this should be queue instead of stack?)
 			context->event_stack = msg->next;
 			free(msg);

ext/portable-file-dialogs/COPYING

@@ -1,14 +0,0 @@
-            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
-                    Version 2, December 2004
-
- Copyright (C) 2004 Sam Hocevar
-
- Everyone is permitted to copy and distribute verbatim or modified
- copies of this license document, and changing it is allowed as long
- as the name is changed.
-
-            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. You just DO WHAT THE FUCK YOU WANT TO.
-

ext/vma/vk_mem_alloc.cpp

@@ -11,6 +11,63 @@
 
 using namespace PPSSPP_VK;
 
+// The allocator has one owning thread (see VulkanContext::CreateDevice), so VMA gets no-op
+// mutexes and plain counters instead of std::mutex and std::atomic.
+#include <cassert>
+#include <cstdint>
+#define VMA_CONFIGURATION_USER_INCLUDES_H <algorithm>
+
+class VmaOwnerMutex {
+public:
+	void Lock() {}
+	void Unlock() {}
+	bool TryLock() { return true; }
+	void LockRead() {}
+	void UnlockRead() {}
+	bool TryLockRead() { return true; }
+	void LockWrite() {}
+	void UnlockWrite() {}
+	bool TryLockWrite() { return true; }
+};
+
+template <typename T>
+class VmaOwnerValue {
+public:
+	VmaOwnerValue() : v_() {}
+	VmaOwnerValue(T v) : v_(v) {}
+	VmaOwnerValue(const VmaOwnerValue &) = delete;
+	VmaOwnerValue &operator=(const VmaOwnerValue &) = delete;
+	T load() const { return v_; }
+	void store(T v) { v_ = v; }
+	T fetch_add(T d) { T o = v_; v_ += d; return o; }
+	T fetch_sub(T d) { T o = v_; v_ -= d; return o; }
+	bool compare_exchange_strong(T &expected, T desired) {
+		if (v_ == expected) {
+			v_ = desired;
+			return true;
+		}
+		expected = v_;
+		return false;
+	}
+	bool compare_exchange_weak(T &expected, T desired) { return compare_exchange_strong(expected, desired); }
+	T operator=(T v) { v_ = v; return v; }
+	operator T() const { return v_; }
+	T operator++() { return ++v_; }
+	T operator--() { return --v_; }
+	T operator++(int) { return v_++; }
+	T operator--(int) { return v_--; }
+	T operator+=(T d) { return v_ += d; }
+	T operator-=(T d) { return v_ -= d; }
+private:
+	T v_;
+};
+
+#define VMA_MUTEX VmaOwnerMutex
+#define VMA_RW_MUTEX VmaOwnerMutex
+#define VMA_ATOMIC_UINT32 VmaOwnerValue<uint32_t>
+#define VMA_ATOMIC_UINT64 VmaOwnerValue<uint64_t>
+#define VMA_ATOMIC_BOOL VmaOwnerValue<bool>
+
 #undef VK_NO_PROTOTYPES
 #include "vk_mem_alloc.h"
 #define VK_NO_PROTOTYPES

headless/Headless.cpp

@@ -977,16 +977,22 @@ int main(int argc, const char* argv[]) {
 		// We don't bother with a window.
 		graphicsContext = new NullGraphicsContext();
 	} else {
-#if PPSSPP_PLATFORM(ANDROID) || defined(HEADLESS_NO_SDL)
+#if PPSSPP_PLATFORM(ANDROID)
 		fprintf(stderr, "Headless graphics context creation is not supported on this platform.\n");
 		return 1;
 #else
 		if (gpuCore == GPUCORE_VULKAN) {
-			// Vulkan renders into images of its own, with no window or swapchain.
+			// Vulkan renders into images of its own, with no window or swapchain, so it needs no SDL.
 			VulkanGraphicsContext *vulkanContext = new VulkanGraphicsContext();
 			vulkanContext->SetOffscreen(480, 272);
 			graphicsContext = vulkanContext;
 			deviceSetting = &g_Config.sVulkanDevice;
+#if defined(HEADLESS_NO_SDL)
+		} else {
+			fprintf(stderr, "Headless graphics context creation is not supported in this build.\n");
+			return 1;
+		}
+#else
 #if PPSSPP_PLATFORM(MAC) && defined(SDL)
 		} else if (gpuCore == GPUCORE_GLES) {
 			// So does OpenGL, into a framebuffer object of its own.
@@ -1006,6 +1012,7 @@ int main(int argc, const char* argv[]) {
 				return 1;
 			}
 		}
+#endif
 #endif
 	}
 

headless/README.md

@@ -84,7 +84,7 @@ The `--graphics` option selects the rendering backend:
 | `software`     | Software rasterizer (most deterministic, recommended for tests) |
 | `gles`         | OpenGL ES (desktop OpenGL on non-Windows)                       |
 | `directx11`    | Direct3D 11 (Windows only)                                      |
-| `vulkan`       | Vulkan                                                          |
+| `vulkan`       | Vulkan, offscreen (also in builds without SDL)                  |
 
 The software backend produces deterministic output across runs and is the default. Hardware backends may produce slightly different pixels due to precision differences in shaders and rasterization.
 

test.py

@@ -554,6 +554,56 @@ known_failures = {
   "loongarch64": [
     "cpu/fpu/fpu",
   ],
+  # Vulkan on lavapipe. These compare pixels against software-rendered references that a
+  # hardware backend doesn't match exactly; the lane is there for the threading, not the pixels.
+  "vulkan": [
+    "gpu/clipping/guardband",
+    "gpu/clipping/homogeneous",
+    "gpu/clipping/xycull",
+    "gpu/commands/blend",
+    "gpu/commands/blend565",
+    "gpu/commands/fog",
+    "gpu/commands/light",
+    "gpu/complex/complex",
+    "gpu/depth/precision",
+    "gpu/depth/transformprecision",
+    "gpu/dither/dither",
+    "gpu/exact/coverage",
+    "gpu/exact/curves",
+    "gpu/exact/lighting",
+    "gpu/exact/lines",
+    "gpu/exact/nearclip",
+    "gpu/exact/pixel",
+    "gpu/exact/planes",
+    "gpu/exact/texture",
+    "gpu/exact/transform",
+    "gpu/filtering/linear",
+    "gpu/filtering/mipmaplinear",
+    "gpu/filtering/nearest",
+    "gpu/filtering/precisionlinear2d",
+    "gpu/filtering/precisionlinear3d",
+    "gpu/filtering/precisionnearest2d",
+    "gpu/filtering/precisionnearest3d",
+    "gpu/ge/get",
+    "gpu/primitives/bezier",
+    "gpu/primitives/continue",
+    "gpu/primitives/immediate",
+    "gpu/primitives/indices",
+    "gpu/primitives/spline",
+    "gpu/reflection/reflection",
+    "gpu/rendertarget/copy",
+    "gpu/rendertarget/rendertarget",
+    "gpu/texfunc/blend",
+    "gpu/texfunc/decal",
+    "gpu/texfunc/modulate",
+    "gpu/texmtx/normals",
+    "gpu/texmtx/prims",
+    "gpu/texmtx/source",
+    "gpu/textures/mipmap",
+    "gpu/textures/size",
+    "gpu/triangle/triangle",
+    "gpu/vertices/morph",
+  ],
 }
 
 tests_next = [

unittest/TestThreadManager.cpp

@@ -4,6 +4,7 @@
 #include "Common/Log.h"
 #include "Common/TimeUtil.h"
 #include "Common/Thread/Barrier.h"
+#include "Common/Thread/Event.h"
 #include "Common/Thread/ThreadManager.h"
 #include "Common/Thread/Channel.h"
 #include "Common/Thread/Promise.h"
@@ -18,8 +19,11 @@ struct ResultObject {
 	bool ok;
 };
 
+// Holds the result back until the parallel loop test is done, so BlockUntilReady really waits.
+static Event *g_resultGate;
+
 ResultObject *ResultProducer() {
-	sleep_ms(250, "test-result");
+	g_resultGate->Wait();
 	printf("result produced: thread %d\n", GetCurrentThreadIdForDebug());
 	return new ResultObject{ true };
 }
@@ -36,7 +40,6 @@ bool TestMailbox() {
 }
 
 void rangeFunc(int lower, int upper) {
-	sleep_ms(30, "test-range");
 	printf(" - range %d-%d (thread %d)\n", lower, upper, GetCurrentThreadIdForDebug());
 }
 
@@ -130,12 +133,12 @@ bool TestThreadManager() {
 
 	g_threadMan = &manager;
 
+	Event resultGate;
+	g_resultGate = &resultGate;
 	Promise<ResultObject *> *object(Promise<ResultObject *>::Spawn(&manager, &ResultProducer, TaskType::IO_BLOCKING));
 
-	if (!TestParallelLoop(&manager)) {
-		return false;
-	}
-	sleep_ms(100, "test-threadman");
+	const bool loopOk = TestParallelLoop(&manager);
+	resultGate.Notify();
 
 	ResultObject *result = object->BlockUntilReady();
 	if (result) {
@@ -143,6 +146,9 @@ bool TestThreadManager() {
 	}
 
 	delete object;
+	if (!loopOk) {
+		return false;
+	}
 
 	if (!TestMailbox()) {
 		return false;