diff --git a/GPU/Software/BinManager.cpp b/GPU/Software/BinManager.cpp index d5c72f8c72..71b1177365 100644 --- a/GPU/Software/BinManager.cpp +++ b/GPU/Software/BinManager.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include "Common/Profiler/Profiler.h" #include "Common/Thread/ThreadManager.h" @@ -31,6 +32,10 @@ // Sometimes useful for debugging. static constexpr bool FORCE_SINGLE_THREAD = false; +// With threads, how many queued items are put in tiles at once. +static constexpr int DISTRIBUTE_BATCH = 16; +// How many tile pieces of work are worth waking the threads for. +static constexpr int WAKE_ENTRIES = 48; using namespace Rasterizer; @@ -69,38 +74,42 @@ public: std::condition_variable cond_; }; -static inline void DrawBinItem(const BinItem &item, const RasterizerState &state) { +static inline void DrawBinItem(const BinItem &item, const BinCoords &range, const RasterizerState &state) { switch (item.type) { case BinItemType::TRIANGLE: - DrawTriangle(item.v0, item.v1, item.v2, item.range, state); + DrawTriangle(item.v0, item.v1, item.v2, range, state); break; case BinItemType::CLEAR_RECT: - ClearRectangle(item.v0, item.v1, item.range, state); + ClearRectangle(item.v0, item.v1, range, state); break; case BinItemType::RECT: - DrawRectangle(item.v0, item.v1, item.range, state); + DrawRectangle(item.v0, item.v1, range, state); break; case BinItemType::SPRITE: - DrawSprite(item.v0, item.v1, item.range, state); + DrawSprite(item.v0, item.v1, range, state); break; case BinItemType::LINE: - DrawLine(item.v0, item.v1, item.range, state); + DrawLine(item.v0, item.v1, range, state); break; case BinItemType::POINT: - DrawPoint(item.v0, item.range, state); + DrawPoint(item.v0, range, state); break; } } +static inline void DrawBinItem(const BinItem &item, const RasterizerState &state) { + DrawBinItem(item, item.range, state); +} + class DrawBinItemsTask : public Task { public: - DrawBinItemsTask(BinWaitable *notify, BinManager::BinItemQueue &items, std::atomic &status, const BinManager::BinStateQueue &states) - : notify_(notify), items_(items), status_(status), states_(states) { + DrawBinItemsTask(BinWaitable *notify, BinManager *binner, int index, std::atomic &status) + : notify_(notify), binner_(binner), index_(index), status_(status) { } TaskType Type() const override { @@ -113,10 +122,10 @@ public: } void Run() override { - ProcessItems(); + binner_->ProcessTiles(index_); status_ = false; - // In case of any atomic issues, do another pass. - ProcessItems(); + // Work queued after the last look, but before status_ said we were done, would otherwise wait. + binner_->ProcessTiles(index_); notify_->Drain(); } @@ -125,18 +134,10 @@ public: } private: - void ProcessItems() { - while (!items_.Empty()) { - const BinItem &item = items_.PeekNext(); - DrawBinItem(item, states_[item.stateIndex]); - items_.SkipNext(); - } - } - BinWaitable *notify_; - BinManager::BinItemQueue &items_; + BinManager *binner_; + int index_; std::atomic &status_; - const BinManager::BinStateQueue &states_; }; constexpr int BinManager::MAX_POSSIBLE_TASKS; @@ -153,10 +154,17 @@ BinManager::BinManager() { int maxInitTasks = std::min(g_threadManager.GetNumLooperThreads(), MAX_POSSIBLE_TASKS); for (int i = 0; i < maxInitTasks; ++i) { - taskQueues_[i].Setup(); for (DrawBinItemsTask *&task : taskLists_[i].tasks) - task = new DrawBinItemsTask(waitable_, taskQueues_[i], taskStatus_[i], states_); + task = new DrawBinItemsTask(waitable_, this, i, taskStatus_[i]); } + tiles_ = (Tile *)AllocateAlignedMemory(sizeof(Tile) * TILES_X * TILES_Y, 64); + for (int i = 0; i < TILES_X * TILES_Y; ++i) { + tiles_[i].head = 0; + tiles_[i].tail = 0; + tiles_[i].busy = false; + } + for (auto &r : itemRefs_) + r = 0; states_.Setup(); cluts_.Setup(); queue_.Setup(); @@ -164,6 +172,7 @@ BinManager::BinManager() { BinManager::~BinManager() { delete waitable_; + FreeAlignedMemory(tiles_); for (int i = 0; i < MAX_POSSIBLE_TASKS; ++i) { for (DrawBinItemsTask *task : taskLists_[i].tasks) @@ -498,7 +507,7 @@ void BinManager::AddTriangle(const VertexData &v0, const VertexData &v1, const V return; if (queue_.Full()) - Drain(); + MakeRoom(); queue_.Push(BinItem{ BinItemType::TRIANGLE, stateIndex_, range, v0, v1, v2 }); CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, v2); Expand(range); @@ -510,7 +519,7 @@ void BinManager::AddClearRect(const VertexData &v0, const VertexData &v1) { return; if (queue_.Full()) - Drain(); + MakeRoom(); queue_.Push(BinItem{ BinItemType::CLEAR_RECT, stateIndex_, range, v0, v1 }); CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, true); Expand(range); @@ -522,7 +531,7 @@ void BinManager::AddRect(const VertexData &v0, const VertexData &v1) { return; if (queue_.Full()) - Drain(); + MakeRoom(); queue_.Push(BinItem{ BinItemType::RECT, stateIndex_, range, v0, v1 }); CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, true); Expand(range); @@ -534,7 +543,7 @@ void BinManager::AddSprite(const VertexData &v0, const VertexData &v1) { return; if (queue_.Full()) - Drain(); + MakeRoom(); queue_.Push(BinItem{ BinItemType::SPRITE, stateIndex_, range, v0, v1 }); CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, true); Expand(range); @@ -546,7 +555,7 @@ void BinManager::AddLine(const VertexData &v0, const VertexData &v1) { return; if (queue_.Full()) - Drain(); + MakeRoom(); queue_.Push(BinItem{ BinItemType::LINE, stateIndex_, range, v0, v1 }); CalculateRasterStateFlags(&states_[stateIndex_], v0, v1, false); Expand(range); @@ -558,7 +567,7 @@ void BinManager::AddPoint(const VertexData &v0) { return; if (queue_.Full()) - Drain(); + MakeRoom(); queue_.Push(BinItem{ BinItemType::POINT, stateIndex_, range, v0 }); CalculateRasterStateFlags(&states_[stateIndex_], v0); Expand(range); @@ -567,107 +576,155 @@ void BinManager::AddPoint(const VertexData &v0) { void BinManager::Drain(bool flushing) { PROFILE_THIS_SCOPE("bin_drain"); - // If the waitable has fully drained, we can update our binning decisions. - if (!tasksSplit_ || waitable_->Empty()) { - int w2 = (queueRange_.x2 - queueRange_.x1 + (SCREEN_SCALE_FACTOR * 2 - 1)) / (SCREEN_SCALE_FACTOR * 2); - int h2 = (queueRange_.y2 - queueRange_.y1 + (SCREEN_SCALE_FACTOR * 2 - 1)) / (SCREEN_SCALE_FACTOR * 2); + if (pendingOverlap_ && maxTasks_ == 1 && flushing && queue_.Size() == 1 && !FORCE_SINGLE_THREAD) { + // If the drawing is 1:1, we can potentially use threads. It's worth checking. + const auto &item = queue_.PeekNext(); + const auto &state = states_[item.stateIndex]; + if (IsExactSelfRender(state, item)) + maxTasks_ = std::min(g_threadManager.GetNumLooperThreads(), MAX_POSSIBLE_TASKS); + } - // Always bin the entire possible range, but focus on the drawn area. - ScreenCoords tl(0, 0, 0); - ScreenCoords br(1024 * SCREEN_SCALE_FACTOR, 1024 * SCREEN_SCALE_FACTOR, 0); - - if (pendingOverlap_ && maxTasks_ == 1 && flushing && queue_.Size() == 1 && !FORCE_SINGLE_THREAD) { - // If the drawing is 1:1, we can potentially use threads. It's worth checking. - const auto &item = queue_.PeekNext(); - const auto &state = states_[item.stateIndex]; - if (IsExactSelfRender(state, item)) - maxTasks_ = std::min(g_threadManager.GetNumLooperThreads(), MAX_POSSIBLE_TASKS); + // Also when switching to one thread: what's left goes to the tiles, which hold the earlier items. + if (maxTasks_ > 1 || activeCount_ != 0) { + DistributeItems(); + // Waking threads costs more than drawing a little: the flush draws that itself. + if (entriesSinceWake_ >= WAKE_ENTRIES) { + WakeTasks(); + entriesSinceWake_ = 0; } - - taskRanges_.clear(); - if (h2 >= 18 && w2 >= h2 * 4) { - int bin_w = std::max(4, (w2 + maxTasks_ - 1) / maxTasks_) * SCREEN_SCALE_FACTOR * 2; - taskRanges_.push_back(BinCoords{ tl.x, tl.y, queueRange_.x1 + bin_w - 1, br.y - 1 }); - for (int x = queueRange_.x1 + bin_w; x <= queueRange_.x2; x += bin_w) { - int x2 = x + bin_w > queueRange_.x2 ? br.x : x + bin_w; - taskRanges_.push_back(BinCoords{ x, tl.y, x2 - 1, br.y - 1 }); - } - } else if (h2 >= 18 && w2 >= 18) { - int bin_h = std::max(4, (h2 + maxTasks_ - 1) / maxTasks_) * SCREEN_SCALE_FACTOR * 2; - taskRanges_.push_back(BinCoords{ tl.x, tl.y, br.x - 1, queueRange_.y1 + bin_h - 1 }); - for (int y = queueRange_.y1 + bin_h; y <= queueRange_.y2; y += bin_h) { - int y2 = y + bin_h > queueRange_.y2 ? br.y : y + bin_h; - taskRanges_.push_back(BinCoords{ tl.x, y, br.x - 1, y2 - 1 }); - } - } - - tasksSplit_ = true; + return; } // Let's try to optimize states, if we can. OptimizePendingStates(pendingStateIndex_, stateIndex_); pendingStateIndex_ = stateIndex_; - if (taskRanges_.size() <= 1) { - PROFILE_THIS_SCOPE("bin_drain_single"); - // Anything (transfers, the CPU) may have written memory since the last drain. - selfTexValid_ = false; - while (!queue_.Empty()) { - const BinItem &item = queue_.PeekNext(); - const RasterizerState &state = states_[item.stateIndex]; - if (state.selfTexture) { - DrawBinItem(item, SelfTextureSnapshot(item, state)); - } else { - selfTexValid_ = false; - DrawBinItem(item, state); - } - queue_.SkipNext(); + PROFILE_THIS_SCOPE("bin_drain_single"); + // Anything (transfers, the CPU) may have written memory since the last drain. + selfTexValid_ = false; + while (!queue_.Empty()) { + const BinItem &item = queue_.PeekNext(); + const RasterizerState &state = states_[item.stateIndex]; + if (state.selfTexture) { + DrawBinItem(item, SelfTextureSnapshot(item, state)); + } else { + selfTexValid_ = false; + DrawBinItem(item, state); } - } else { - int max = flushing ? QUEUED_PRIMS : QUEUED_PRIMS / 2; - while (!queue_.Empty()) { - const BinItem &item = queue_.PeekNext(); - for (int i = 0; i < (int)taskRanges_.size(); ++i) { - const BinCoords range = taskRanges_[i].Intersect(item.range); - if (range.Invalid()) - continue; - - if (taskQueues_[i].NearFull()) { - // This shouldn't often happen, but if it does, wait for space. - if (taskQueues_[i].Full()) - waitable_->Wait(); - // If we're not flushing and not near full, let's just continue later. - // Near full means we'd drain on next prim, so better to finish it now. - else if (!flushing && !queue_.NearFull()) - max = 0; - } - - BinItem &taskItem = taskQueues_[i].PeekPush(); - taskItem = item; - taskItem.range = range; - taskQueues_[i].PushPeeked(); - } - queue_.SkipNext(); - if (--max <= 0) - break; - } - - int threads = 0; - for (int i = 0; i < (int)taskRanges_.size(); ++i) { - if (taskQueues_[i].Empty()) - continue; - threads++; - if (taskStatus_[i]) - continue; - - waitable_->Fill(); - taskStatus_[i] = true; - g_threadManager.EnqueueTaskOnThread(i, taskLists_[i].Next()); - enqueues_++; - } - - mostThreads_ = std::max(mostThreads_, threads); + queue_.SkipNext(); } + distributePos_ = queue_.tail_; +} + +void BinManager::DistributeItems() { + OptimizePendingStates(pendingStateIndex_, stateIndex_); + pendingStateIndex_ = stateIndex_; + undistributed_ = 0; + + const size_t end = queue_.tail_; + while (distributePos_ != end) { + const size_t index = distributePos_; + const BinItem &item = queue_[index]; + const int tx1 = std::clamp(item.range.x1 / (SCREEN_SCALE_FACTOR * TILE_W), 0, TILES_X - 1); + const int tx2 = std::clamp(item.range.x2 / (SCREEN_SCALE_FACTOR * TILE_W), 0, TILES_X - 1); + const int ty1 = std::clamp(item.range.y1 / (SCREEN_SCALE_FACTOR * TILE_H), 0, TILES_Y - 1); + const int ty2 = std::clamp(item.range.y2 / (SCREEN_SCALE_FACTOR * TILE_H), 0, TILES_Y - 1); + // Set before any tile can draw it. + itemRefs_[index].store((tx2 - tx1 + 1) * (ty2 - ty1 + 1), std::memory_order_relaxed); + for (int ty = ty1; ty <= ty2; ++ty) { + for (int tx = tx1; tx <= tx2; ++tx) { + const int t = ty * TILES_X + tx; + Tile &tile = tiles_[t]; + const uint32_t tail = tile.tail.load(std::memory_order_relaxed); + tile.items[tail % QUEUED_PRIMS] = (uint16_t)index; + entriesSinceWake_++; + tile.tail.store(tail + 1, std::memory_order_release); + if (!tileActive_[t]) { + tileActive_[t] = true; + const int count = activeCount_.load(std::memory_order_relaxed); + activeTiles_[count] = (uint16_t)t; + activeCount_.store(count + 1, std::memory_order_release); + } + } + } + distributePos_ = index + 1 == QUEUED_PRIMS ? 0 : index + 1; + } +} + +void BinManager::ReclaimItems() { + while (!queue_.Empty() && queue_.head_ != distributePos_ && itemRefs_[queue_.head_].load(std::memory_order_acquire) == 0) + queue_.SkipNext(); +} + +void BinManager::WakeTasks() { + int threads = 0; + for (int i = 0; i < maxTasks_; ++i) { + threads++; + if (taskStatus_[i]) + continue; + + waitable_->Fill(); + taskStatus_[i] = true; + g_threadManager.EnqueueTaskOnThread(i, taskLists_[i].Next()); + enqueues_++; + } + mostThreads_ = std::max(mostThreads_, threads); +} + +// The queue is full: with threads, wait for drawn items to free up, helping with the drawing. +void BinManager::MakeRoom() { + if (maxTasks_ <= 1) { + Drain(); + return; + } + Drain(); + ReclaimItems(); + while (queue_.Full()) { + if (!ProcessTiles(0)) + std::this_thread::yield(); + ReclaimItems(); + } +} + +bool BinManager::ProcessTiles(int start) { + bool any = false; + bool found; + do { + found = false; + const int count = activeCount_.load(std::memory_order_acquire); + for (int n = 0; n < count; ++n) { + const int t = activeTiles_[(start + n) % count]; + Tile &tile = tiles_[t]; + if (tile.head.load(std::memory_order_relaxed) == tile.tail.load(std::memory_order_acquire)) + continue; + if (tile.busy.load(std::memory_order_relaxed) || tile.busy.exchange(true, std::memory_order_acquire)) + continue; + + const int tx = t % TILES_X, ty = t / TILES_X; + const BinCoords tileRange{ + tx * TILE_W * SCREEN_SCALE_FACTOR, ty * TILE_H * SCREEN_SCALE_FACTOR, + (tx + 1) * TILE_W * SCREEN_SCALE_FACTOR - 1, (ty + 1) * TILE_H * SCREEN_SCALE_FACTOR - 1, + }; + uint32_t head = tile.head.load(std::memory_order_relaxed); + uint32_t tail; + while (head != (tail = tile.tail.load(std::memory_order_acquire))) { + for (; head != tail; ++head) { + const uint16_t index = tile.items[head % QUEUED_PRIMS]; + const BinItem &item = queue_[index]; + // Clamping to the grid can put an item in an edge tile it doesn't reach. + const BinCoords range = tileRange.Intersect(item.range); + if (!range.Invalid()) + DrawBinItem(item, range, states_[item.stateIndex]); + itemRefs_[index].fetch_sub(1, std::memory_order_release); + } + tile.head.store(head, std::memory_order_release); + } + tile.busy.store(false, std::memory_order_release); + found = true; + any = true; + } + } while (found); + return any; } void BinManager::Flush(const char *reason) { @@ -689,9 +746,23 @@ void BinManager::Flush(const char *reason) { st = time_now_d(); } Drain(true); + if (maxTasks_ > 1 || activeCount_ != 0) { + // Help with the drawing, then wait for what the threads are still on. + while (ProcessTiles(0)) { + } + } waitable_->Wait(); - taskRanges_.clear(); - tasksSplit_ = false; + const int activeCount = activeCount_; + for (int i = 0; i < activeCount; ++i) { + const int t = activeTiles_[i]; + tiles_[t].head = 0; + tiles_[t].tail = 0; + tileActive_[t] = false; + } + activeCount_ = 0; + distributePos_ = 0; + undistributed_ = 0; + entriesSinceWake_ = 0; queue_.Reset(); while (states_.Size() > 1) @@ -887,10 +958,12 @@ void BinManager::Expand(const BinCoords &range) { queueRange_.x2 = std::max(queueRange_.x2, range.x2); queueRange_.y2 = std::max(queueRange_.y2, range.y2); - if (maxTasks_ == 1 || (queueRange_.y2 - queueRange_.y1 >= 224 * SCREEN_SCALE_FACTOR && enqueues_ < 36 * maxTasks_)) { + if (maxTasks_ == 1) { if (pendingOverlap_) Flush("expand"); else Drain(); + } else if (++undistributed_ >= DISTRIBUTE_BATCH) { + Drain(); } } diff --git a/GPU/Software/BinManager.h b/GPU/Software/BinManager.h index 39d534623b..2027e67bc3 100644 --- a/GPU/Software/BinManager.h +++ b/GPU/Software/BinManager.h @@ -262,11 +262,33 @@ private: SoftDirty dirty_ = SoftDirty::NONE; int maxTasks_ = 1; - bool tasksSplit_ = false; - std::vector taskRanges_; - BinItemQueue taskQueues_[MAX_POSSIBLE_TASKS]; BinTaskList taskLists_[MAX_POSSIBLE_TASKS]; std::atomic taskStatus_[MAX_POSSIBLE_TASKS]; + + // With threads, queued items are binned into screen tiles. Any thread can take a tile with work and + // draws its items in order; only one at a time, so each pixel still sees the primitives in order. + static constexpr int TILE_W = 64; + static constexpr int TILE_H = 16; + static constexpr int TILES_X = 1024 / TILE_W; + static constexpr int TILES_Y = 1024 / TILE_H; + struct Tile { + // Indices into queue_, as a ring: head_ is how many have been drawn, tail_ how many were pushed. + std::atomic head; + std::atomic tail; + std::atomic busy; + uint16_t items[QUEUED_PRIMS]; + }; + Tile *tiles_ = nullptr; + // For each queued item, how many tiles still have to draw it. It's reclaimed at zero. + std::atomic itemRefs_[QUEUED_PRIMS]; + // The tiles given work since the last flush, for the threads to look through. + uint16_t activeTiles_[TILES_X * TILES_Y]; + std::atomic activeCount_{ 0 }; + bool tileActive_[TILES_X * TILES_Y]{}; + // The queue_ index of the first item not yet put in tiles, and how many have been added since. + size_t distributePos_ = 0; + int undistributed_ = 0; + int entriesSinceWake_ = 0; BinWaitable *waitable_ = nullptr; BinDirtyRange pendingWrites_[2]{}; @@ -314,6 +336,11 @@ private: BinCoords Range(const VertexData &v0, const VertexData &v1); BinCoords Range(const VertexData &v0); void Expand(const BinCoords &range); + void MakeRoom(); + void DistributeItems(); + void ReclaimItems(); + void WakeTasks(); + bool ProcessTiles(int start); friend class DrawBinItemsTask; };