SoftGPU: Bin queued primitives into tiles that any thread can draw

Each thread used to own a band of the screen, fixed when the work was
first split, and got its own copies of the primitives touching it. Bands
were rarely balanced (edge bands had half the work of middle ones, and
on big.LITTLE CPUs half the threads are slower), and primitives queued
up to the 2048 limit before the threads started.

Now primitives go to 64x16 tiles, as indices into the shared queue, in
batches of 16. A thread takes any tile with work and draws its items in
order; only one thread holds a tile at a time, so per-pixel order is
kept. A flush or a full queue has the main thread draw tiles too instead
of waiting, and when there's only a little work it does that alone
rather than wake the threads. Items are reclaimed once every tile has
drawn them. This also drops the per-thread queues, up to 23 MB.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

GPU/Software/BinManager.cpp

@@ -18,6 +18,7 @@
 #include <atomic>
 #include <mutex>
 #include <condition_variable>
+#include <thread>
 
 #include "Common/Profiler/Profiler.h"
 #include "Common/Thread/ThreadManager.h"
@@ -31,6 +32,10 @@
 
 // Sometimes useful for debugging.
 static constexpr bool FORCE_SINGLE_THREAD = false;
+// With threads, how many queued items are put in tiles at once.
+static constexpr int DISTRIBUTE_BATCH = 16;
+// How many tile pieces of work are worth waking the threads for.
+static constexpr int WAKE_ENTRIES = 48;
 
 using namespace Rasterizer;
 
@@ -69,38 +74,42 @@ public:
 	std::condition_variable cond_;
 };
 
-static inline void DrawBinItem(const BinItem &item, const RasterizerState &state) {
+static inline void DrawBinItem(const BinItem &item, const BinCoords &range, const RasterizerState &state) {
 	switch (item.type) {
 	case BinItemType::TRIANGLE:
-		DrawTriangle(item.v0, item.v1, item.v2, item.range, state);
+		DrawTriangle(item.v0, item.v1, item.v2, range, state);
 		break;
 
 	case BinItemType::CLEAR_RECT:
-		ClearRectangle(item.v0, item.v1, item.range, state);
+		ClearRectangle(item.v0, item.v1, range, state);
 		break;
 
 	case BinItemType::RECT:
-		DrawRectangle(item.v0, item.v1, item.range, state);
+		DrawRectangle(item.v0, item.v1, range, state);
 		break;
 
 	case BinItemType::SPRITE:
-		DrawSprite(item.v0, item.v1, item.range, state);
+		DrawSprite(item.v0, item.v1, range, state);
 		break;
 
 	case BinItemType::LINE:
-		DrawLine(item.v0, item.v1, item.range, state);
+		DrawLine(item.v0, item.v1, range, state);
 		break;
 
 	case BinItemType::POINT:
-		DrawPoint(item.v0, item.range, state);
+		DrawPoint(item.v0, range, state);
 		break;
 	}
 }
 
+static inline void DrawBinItem(const BinItem &item, const RasterizerState &state) {
+	DrawBinItem(item, item.range, state);
+}
+
 class DrawBinItemsTask : public Task {
 public:
-	DrawBinItemsTask(BinWaitable *notify, BinManager::BinItemQueue &items, std::atomic<bool> &status, const BinManager::BinStateQueue &states)
-		: notify_(notify), items_(items), status_(status), states_(states) {
+	DrawBinItemsTask(BinWaitable *notify, BinManager *binner, int index, std::atomic<bool> &status)
+		: notify_(notify), binner_(binner), index_(index), status_(status) {
 	}
 
 	TaskType Type() const override {
@@ -113,10 +122,10 @@ public:
 	}
 
 	void Run() override {
-		ProcessItems();
+		binner_->ProcessTiles(index_);
 		status_ = false;
-		// In case of any atomic issues, do another pass.
-		ProcessItems();
+		// Work queued after the last look, but before status_ said we were done, would otherwise wait.
+		binner_->ProcessTiles(index_);
 		notify_->Drain();
 	}
 
@@ -125,18 +134,10 @@ public:
 	}
 
 private:
-	void ProcessItems() {
-		while (!items_.Empty()) {
-			const BinItem &item = items_.PeekNext();
-			DrawBinItem(item, states_[item.stateIndex]);
-			items_.SkipNext();
-		}
-	}
-
 	BinWaitable *notify_;
-	BinManager::BinItemQueue &items_;
+	BinManager *binner_;
+	int index_;
 	std::atomic<bool> &status_;
-	const BinManager::BinStateQueue &states_;
 };
 
 constexpr int BinManager::MAX_POSSIBLE_TASKS;
@@ -153,10 +154,17 @@ BinManager::BinManager() {
 
 	int maxInitTasks = std::min(g_threadManager.GetNumLooperThreads(), MAX_POSSIBLE_TASKS);
 	for (int i = 0; i < maxInitTasks; ++i) {
-		taskQueues_[i].Setup();
 		for (DrawBinItemsTask *&task : taskLists_[i].tasks)
-			task = new DrawBinItemsTask(waitable_, taskQueues_[i], taskStatus_[i], states_);
+			task = new DrawBinItemsTask(waitable_, this, i, taskStatus_[i]);
 	}
+	tiles_ = (Tile *)AllocateAlignedMemory(sizeof(Tile) * TILES_X * TILES_Y, 64);
+	for (int i = 0; i < TILES_X * TILES_Y; ++i) {
+		tiles_[i].head = 0;
+		tiles_[i].tail = 0;
+		tiles_[i].busy = false;
+	}
+	for (auto &r : itemRefs_)
+		r = 0;
 	states_.Setup();
 	cluts_.Setup();
 	queue_.Setup();
@@ -164,6 +172,7 @@ BinManager::BinManager() {
 
 BinManager::~BinManager() {
 	delete waitable_;
+	FreeAlignedMemory(tiles_);
 
 	for (int i = 0; i < MAX_POSSIBLE_TASKS; ++i) {
 		for (DrawBinItemsTask *task : taskLists_[i].tasks)
@@ -498,7 +507,7 @@ void BinManager::AddTriangle(const VertexData &v0, const VertexData &v1, const V
 		return;
 
 	if (queue_.Full())
-		Drain();
+		MakeRoom();
 	queue_.Push(BinItem{ BinItemType::TRIANGLE, stateIndex_, range, v0, v1, v2 });
 	CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, v2);
 	Expand(range);
@@ -510,7 +519,7 @@ void BinManager::AddClearRect(const VertexData &v0, const VertexData &v1) {
 		return;
 
 	if (queue_.Full())
-		Drain();
+		MakeRoom();
 	queue_.Push(BinItem{ BinItemType::CLEAR_RECT, stateIndex_, range, v0, v1 });
 	CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, true);
 	Expand(range);
@@ -522,7 +531,7 @@ void BinManager::AddRect(const VertexData &v0, const VertexData &v1) {
 		return;
 
 	if (queue_.Full())
-		Drain();
+		MakeRoom();
 	queue_.Push(BinItem{ BinItemType::RECT, stateIndex_, range, v0, v1 });
 	CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, true);
 	Expand(range);
@@ -534,7 +543,7 @@ void BinManager::AddSprite(const VertexData &v0, const VertexData &v1) {
 		return;
 
 	if (queue_.Full())
-		Drain();
+		MakeRoom();
 	queue_.Push(BinItem{ BinItemType::SPRITE, stateIndex_, range, v0, v1 });
 	CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, true);
 	Expand(range);
@@ -546,7 +555,7 @@ void BinManager::AddLine(const VertexData &v0, const VertexData &v1) {
 		return;
 
 	if (queue_.Full())
-		Drain();
+		MakeRoom();
 	queue_.Push(BinItem{ BinItemType::LINE, stateIndex_, range, v0, v1 });
 	CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, false);
 	Expand(range);
@@ -558,7 +567,7 @@ void BinManager::AddPoint(const VertexData &v0) {
 		return;
 
 	if (queue_.Full())
-		Drain();
+		MakeRoom();
 	queue_.Push(BinItem{ BinItemType::POINT, stateIndex_, range, v0 });
 	CalculateRasterStateFlags(&states_[stateIndex_], v0);
 	Expand(range);
@@ -567,107 +576,155 @@ void BinManager::AddPoint(const VertexData &v0) {
 void BinManager::Drain(bool flushing) {
 	PROFILE_THIS_SCOPE("bin_drain");
 
-	// If the waitable has fully drained, we can update our binning decisions.
-	if (!tasksSplit_ || waitable_->Empty()) {
-		int w2 = (queueRange_.x2 - queueRange_.x1 + (SCREEN_SCALE_FACTOR * 2 - 1)) / (SCREEN_SCALE_FACTOR * 2);
-		int h2 = (queueRange_.y2 - queueRange_.y1 + (SCREEN_SCALE_FACTOR * 2 - 1)) / (SCREEN_SCALE_FACTOR * 2);
+	if (pendingOverlap_ && maxTasks_ == 1 && flushing && queue_.Size() == 1 && !FORCE_SINGLE_THREAD) {
+		// If the drawing is 1:1, we can potentially use threads.  It's worth checking.
+		const auto &item = queue_.PeekNext();
+		const auto &state = states_[item.stateIndex];
+		if (IsExactSelfRender(state, item))
+			maxTasks_ = std::min(g_threadManager.GetNumLooperThreads(), MAX_POSSIBLE_TASKS);
+	}
 
-		// Always bin the entire possible range, but focus on the drawn area.
-		ScreenCoords tl(0, 0, 0);
-		ScreenCoords br(1024 * SCREEN_SCALE_FACTOR, 1024 * SCREEN_SCALE_FACTOR, 0);
-
-		if (pendingOverlap_ && maxTasks_ == 1 && flushing && queue_.Size() == 1 && !FORCE_SINGLE_THREAD) {
-			// If the drawing is 1:1, we can potentially use threads.  It's worth checking.
-			const auto &item = queue_.PeekNext();
-			const auto &state = states_[item.stateIndex];
-			if (IsExactSelfRender(state, item))
-				maxTasks_ = std::min(g_threadManager.GetNumLooperThreads(), MAX_POSSIBLE_TASKS);
+	// Also when switching to one thread: what's left goes to the tiles, which hold the earlier items.
+	if (maxTasks_ > 1 || activeCount_ != 0) {
+		DistributeItems();
+		// Waking threads costs more than drawing a little: the flush draws that itself.
+		if (entriesSinceWake_ >= WAKE_ENTRIES) {
+			WakeTasks();
+			entriesSinceWake_ = 0;
 		}
-
-		taskRanges_.clear();
-		if (h2 >= 18 && w2 >= h2 * 4) {
-			int bin_w = std::max(4, (w2 + maxTasks_ - 1) / maxTasks_) * SCREEN_SCALE_FACTOR * 2;
-			taskRanges_.push_back(BinCoords{ tl.x, tl.y, queueRange_.x1 + bin_w - 1, br.y - 1 });
-			for (int x = queueRange_.x1 + bin_w; x <= queueRange_.x2; x += bin_w) {
-				int x2 = x + bin_w > queueRange_.x2 ? br.x : x + bin_w;
-				taskRanges_.push_back(BinCoords{ x, tl.y, x2 - 1, br.y - 1 });
-			}
-		} else if (h2 >= 18 && w2 >= 18) {
-			int bin_h = std::max(4, (h2 + maxTasks_ - 1) / maxTasks_) * SCREEN_SCALE_FACTOR * 2;
-			taskRanges_.push_back(BinCoords{ tl.x, tl.y, br.x - 1, queueRange_.y1 + bin_h - 1 });
-			for (int y = queueRange_.y1 + bin_h; y <= queueRange_.y2; y += bin_h) {
-				int y2 = y + bin_h > queueRange_.y2 ? br.y : y + bin_h;
-				taskRanges_.push_back(BinCoords{ tl.x, y, br.x - 1, y2 - 1 });
-			}
-		}
-
-		tasksSplit_ = true;
+		return;
 	}
 
 	// Let's try to optimize states, if we can.
 	OptimizePendingStates(pendingStateIndex_, stateIndex_);
 	pendingStateIndex_ = stateIndex_;
 
-	if (taskRanges_.size() <= 1) {
-		PROFILE_THIS_SCOPE("bin_drain_single");
-		// Anything (transfers, the CPU) may have written memory since the last drain.
-		selfTexValid_ = false;
-		while (!queue_.Empty()) {
-			const BinItem &item = queue_.PeekNext();
-			const RasterizerState &state = states_[item.stateIndex];
-			if (state.selfTexture) {
-				DrawBinItem(item, SelfTextureSnapshot(item, state));
-			} else {
-				selfTexValid_ = false;
-				DrawBinItem(item, state);
-			}
-			queue_.SkipNext();
+	PROFILE_THIS_SCOPE("bin_drain_single");
+	// Anything (transfers, the CPU) may have written memory since the last drain.
+	selfTexValid_ = false;
+	while (!queue_.Empty()) {
+		const BinItem &item = queue_.PeekNext();
+		const RasterizerState &state = states_[item.stateIndex];
+		if (state.selfTexture) {
+			DrawBinItem(item, SelfTextureSnapshot(item, state));
+		} else {
+			selfTexValid_ = false;
+			DrawBinItem(item, state);
 		}
-	} else {
-		int max = flushing ? QUEUED_PRIMS : QUEUED_PRIMS / 2;
-		while (!queue_.Empty()) {
-			const BinItem &item = queue_.PeekNext();
-			for (int i = 0; i < (int)taskRanges_.size(); ++i) {
-				const BinCoords range = taskRanges_[i].Intersect(item.range);
-				if (range.Invalid())
-					continue;
-
-				if (taskQueues_[i].NearFull()) {
-					// This shouldn't often happen, but if it does, wait for space.
-					if (taskQueues_[i].Full())
-						waitable_->Wait();
-					// If we're not flushing and not near full, let's just continue later.
-					// Near full means we'd drain on next prim, so better to finish it now.
-					else if (!flushing && !queue_.NearFull())
-						max = 0;
-				}
-
-				BinItem &taskItem = taskQueues_[i].PeekPush();
-				taskItem = item;
-				taskItem.range = range;
-				taskQueues_[i].PushPeeked();
-			}
-			queue_.SkipNext();
-			if (--max <= 0)
-				break;
-		}
-
-		int threads = 0;
-		for (int i = 0; i < (int)taskRanges_.size(); ++i) {
-			if (taskQueues_[i].Empty())
-				continue;
-			threads++;
-			if (taskStatus_[i])
-				continue;
-
-			waitable_->Fill();
-			taskStatus_[i] = true;
-			g_threadManager.EnqueueTaskOnThread(i, taskLists_[i].Next());
-			enqueues_++;
-		}
-
-		mostThreads_ = std::max(mostThreads_, threads);
+		queue_.SkipNext();
 	}
+	distributePos_ = queue_.tail_;
+}
+
+void BinManager::DistributeItems() {
+	OptimizePendingStates(pendingStateIndex_, stateIndex_);
+	pendingStateIndex_ = stateIndex_;
+	undistributed_ = 0;
+
+	const size_t end = queue_.tail_;
+	while (distributePos_ != end) {
+		const size_t index = distributePos_;
+		const BinItem &item = queue_[index];
+		const int tx1 = std::clamp(item.range.x1 / (SCREEN_SCALE_FACTOR * TILE_W), 0, TILES_X - 1);
+		const int tx2 = std::clamp(item.range.x2 / (SCREEN_SCALE_FACTOR * TILE_W), 0, TILES_X - 1);
+		const int ty1 = std::clamp(item.range.y1 / (SCREEN_SCALE_FACTOR * TILE_H), 0, TILES_Y - 1);
+		const int ty2 = std::clamp(item.range.y2 / (SCREEN_SCALE_FACTOR * TILE_H), 0, TILES_Y - 1);
+		// Set before any tile can draw it.
+		itemRefs_[index].store((tx2 - tx1 + 1) * (ty2 - ty1 + 1), std::memory_order_relaxed);
+		for (int ty = ty1; ty <= ty2; ++ty) {
+			for (int tx = tx1; tx <= tx2; ++tx) {
+				const int t = ty * TILES_X + tx;
+				Tile &tile = tiles_[t];
+				const uint32_t tail = tile.tail.load(std::memory_order_relaxed);
+				tile.items[tail % QUEUED_PRIMS] = (uint16_t)index;
+				entriesSinceWake_++;
+				tile.tail.store(tail + 1, std::memory_order_release);
+				if (!tileActive_[t]) {
+					tileActive_[t] = true;
+					const int count = activeCount_.load(std::memory_order_relaxed);
+					activeTiles_[count] = (uint16_t)t;
+					activeCount_.store(count + 1, std::memory_order_release);
+				}
+			}
+		}
+		distributePos_ = index + 1 == QUEUED_PRIMS ? 0 : index + 1;
+	}
+}
+
+void BinManager::ReclaimItems() {
+	while (!queue_.Empty() && queue_.head_ != distributePos_ && itemRefs_[queue_.head_].load(std::memory_order_acquire) == 0)
+		queue_.SkipNext();
+}
+
+void BinManager::WakeTasks() {
+	int threads = 0;
+	for (int i = 0; i < maxTasks_; ++i) {
+		threads++;
+		if (taskStatus_[i])
+			continue;
+
+		waitable_->Fill();
+		taskStatus_[i] = true;
+		g_threadManager.EnqueueTaskOnThread(i, taskLists_[i].Next());
+		enqueues_++;
+	}
+	mostThreads_ = std::max(mostThreads_, threads);
+}
+
+// The queue is full: with threads, wait for drawn items to free up, helping with the drawing.
+void BinManager::MakeRoom() {
+	if (maxTasks_ <= 1) {
+		Drain();
+		return;
+	}
+	Drain();
+	ReclaimItems();
+	while (queue_.Full()) {
+		if (!ProcessTiles(0))
+			std::this_thread::yield();
+		ReclaimItems();
+	}
+}
+
+bool BinManager::ProcessTiles(int start) {
+	bool any = false;
+	bool found;
+	do {
+		found = false;
+		const int count = activeCount_.load(std::memory_order_acquire);
+		for (int n = 0; n < count; ++n) {
+			const int t = activeTiles_[(start + n) % count];
+			Tile &tile = tiles_[t];
+			if (tile.head.load(std::memory_order_relaxed) == tile.tail.load(std::memory_order_acquire))
+				continue;
+			if (tile.busy.load(std::memory_order_relaxed) || tile.busy.exchange(true, std::memory_order_acquire))
+				continue;
+
+			const int tx = t % TILES_X, ty = t / TILES_X;
+			const BinCoords tileRange{
+				tx * TILE_W * SCREEN_SCALE_FACTOR, ty * TILE_H * SCREEN_SCALE_FACTOR,
+				(tx + 1) * TILE_W * SCREEN_SCALE_FACTOR - 1, (ty + 1) * TILE_H * SCREEN_SCALE_FACTOR - 1,
+			};
+			uint32_t head = tile.head.load(std::memory_order_relaxed);
+			uint32_t tail;
+			while (head != (tail = tile.tail.load(std::memory_order_acquire))) {
+				for (; head != tail; ++head) {
+					const uint16_t index = tile.items[head % QUEUED_PRIMS];
+					const BinItem &item = queue_[index];
+					// Clamping to the grid can put an item in an edge tile it doesn't reach.
+					const BinCoords range = tileRange.Intersect(item.range);
+					if (!range.Invalid())
+						DrawBinItem(item, range, states_[item.stateIndex]);
+					itemRefs_[index].fetch_sub(1, std::memory_order_release);
+				}
+				tile.head.store(head, std::memory_order_release);
+			}
+			tile.busy.store(false, std::memory_order_release);
+			found = true;
+			any = true;
+		}
+	} while (found);
+	return any;
 }
 
 void BinManager::Flush(const char *reason) {
@@ -689,9 +746,23 @@ void BinManager::Flush(const char *reason) {
 		st = time_now_d();
 	}
 	Drain(true);
+	if (maxTasks_ > 1 || activeCount_ != 0) {
+		// Help with the drawing, then wait for what the threads are still on.
+		while (ProcessTiles(0)) {
+		}
+	}
 	waitable_->Wait();
-	taskRanges_.clear();
-	tasksSplit_ = false;
+	const int activeCount = activeCount_;
+	for (int i = 0; i < activeCount; ++i) {
+		const int t = activeTiles_[i];
+		tiles_[t].head = 0;
+		tiles_[t].tail = 0;
+		tileActive_[t] = false;
+	}
+	activeCount_ = 0;
+	distributePos_ = 0;
+	undistributed_ = 0;
+	entriesSinceWake_ = 0;
 
 	queue_.Reset();
 	while (states_.Size() > 1)
@@ -887,10 +958,12 @@ void BinManager::Expand(const BinCoords &range) {
 	queueRange_.x2 = std::max(queueRange_.x2, range.x2);
 	queueRange_.y2 = std::max(queueRange_.y2, range.y2);
 
-	if (maxTasks_ == 1 || (queueRange_.y2 - queueRange_.y1 >= 224 * SCREEN_SCALE_FACTOR && enqueues_ < 36 * maxTasks_)) {
+	if (maxTasks_ == 1) {
 		if (pendingOverlap_)
 			Flush("expand");
 		else
 			Drain();
+	} else if (++undistributed_ >= DISTRIBUTE_BATCH) {
+		Drain();
 	}
 }

GPU/Software/BinManager.h

@@ -262,11 +262,33 @@ private:
 	SoftDirty dirty_ = SoftDirty::NONE;
 
 	int maxTasks_ = 1;
-	bool tasksSplit_ = false;
-	std::vector<BinCoords> taskRanges_;
-	BinItemQueue taskQueues_[MAX_POSSIBLE_TASKS];
 	BinTaskList taskLists_[MAX_POSSIBLE_TASKS];
 	std::atomic<bool> taskStatus_[MAX_POSSIBLE_TASKS];
+
+	// With threads, queued items are binned into screen tiles. Any thread can take a tile with work and
+	// draws its items in order; only one at a time, so each pixel still sees the primitives in order.
+	static constexpr int TILE_W = 64;
+	static constexpr int TILE_H = 16;
+	static constexpr int TILES_X = 1024 / TILE_W;
+	static constexpr int TILES_Y = 1024 / TILE_H;
+	struct Tile {
+		// Indices into queue_, as a ring: head_ is how many have been drawn, tail_ how many were pushed.
+		std::atomic<uint32_t> head;
+		std::atomic<uint32_t> tail;
+		std::atomic<bool> busy;
+		uint16_t items[QUEUED_PRIMS];
+	};
+	Tile *tiles_ = nullptr;
+	// For each queued item, how many tiles still have to draw it. It's reclaimed at zero.
+	std::atomic<int> itemRefs_[QUEUED_PRIMS];
+	// The tiles given work since the last flush, for the threads to look through.
+	uint16_t activeTiles_[TILES_X * TILES_Y];
+	std::atomic<int> activeCount_{ 0 };
+	bool tileActive_[TILES_X * TILES_Y]{};
+	// The queue_ index of the first item not yet put in tiles, and how many have been added since.
+	size_t distributePos_ = 0;
+	int undistributed_ = 0;
+	int entriesSinceWake_ = 0;
 	BinWaitable *waitable_ = nullptr;
 
 	BinDirtyRange pendingWrites_[2]{};
@@ -314,6 +336,11 @@ private:
 	BinCoords Range(const VertexData &v0, const VertexData &v1);
 	BinCoords Range(const VertexData &v0);
 	void Expand(const BinCoords &range);
+	void MakeRoom();
+	void DistributeItems();
+	void ReclaimItems();
+	void WakeTasks();
+	bool ProcessTiles(int start);
 
 	friend class DrawBinItemsTask;
 };