From cd59a16d85234b1f3a818fd2ad4e799cec89516a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Thu, 15 Oct 2026 16:30:29 +0200 Subject: [PATCH] GPU: Rasterize software depth on a worker thread The depth draws were queued and rasterized on the emu thread at each render target change, in Wipeout about 40 small batches a frame, and around 40% of that thread's time. Now a render target change hands the batch to a worker thread and carries on. Waiting for it is left to whatever reads the depth buffers: DrawSync and ListSync, block transfers, framebuffer copies, showing a frame, savestates and shutdown, and the texture cache when the texture or CLUT it reads overlaps a depth buffer still being written. Batches grow as needed instead of being preallocated at full size. A full batch is now handed over and a new one started, where the draw used to be dropped. Wipeout headless, output unchanged: 16% less wall time on a Galaxy S8 (8.2 s to 6.9 s for 10 emulated seconds). On an M1, where headless waits on the GPU, the emu thread uses 38% less CPU time. Co-Authored-By: Claude Opus 5.5 (1M context) --- Core/System.cpp | 5 + GPU/Common/DepthRaster.cpp | 210 ++++++++++++++++++++++++++ GPU/Common/DepthRaster.h | 74 +++++++++ GPU/Common/DrawEngineCommon.cpp | 142 ++++++----------- GPU/Common/DrawEngineCommon.h | 16 +- GPU/Common/FramebufferManagerCommon.h | 3 + GPU/Common/TextureCacheCommon.cpp | 15 ++ GPU/Common/TextureCacheCommon.h | 2 + GPU/GPUCommon.cpp | 2 +- GPU/GPUCommonHW.cpp | 11 +- GPU/GPUCommonHW.h | 1 + 11 files changed, 377 insertions(+), 104 deletions(-) diff --git a/Core/System.cpp b/Core/System.cpp index 7c9567ade7..8924ee171c 100644 --- a/Core/System.cpp +++ b/Core/System.cpp @@ -652,6 +652,11 @@ void CPU_Shutdown(bool success) { // Core_LockAgainstShutdown(); it's recursive, so the nested acquire in Memory::Shutdown() is fine. CoreShutdownLock coreLock = Core_LockAgainstShutdown(); + // The GPU outlives memory (GPU_Shutdown comes later), so it must stop writing to it now. + if (gpu) { + gpu->FlushPendingDrawing(); + } + UninstallExceptionHandler(); GPURecord::Replay_Unload(); diff --git a/GPU/Common/DepthRaster.cpp b/GPU/Common/DepthRaster.cpp index 934d875286..b3f2a56d20 100644 --- a/GPU/Common/DepthRaster.cpp +++ b/GPU/Common/DepthRaster.cpp @@ -7,6 +7,10 @@ #include "GPU/Math3D.h" #include "Common/Math/math_util.h" #include "GPU/Common/VertexDecoderCommon.h" +#include "Common/MemoryUtil.h" +#include "Common/Thread/ThreadUtil.h" +#include "Common/TimeUtil.h" +#include "Core/MemMap.h" DepthScissor DepthScissor::Tile(int tile, int numTiles) const { if (numTiles == 1) { @@ -634,3 +638,209 @@ void DepthRasterScreenVerts(uint16_t *depth, int depthStride, const int *tx, con _dbg_assert_(false); } } + +// The depth buffers are all in VRAM, so busy ranges are kept as offsets into it, which also covers +// the mirrors. +static constexpr u32 VRAM_OFFSET_MASK = 0x001FFFFF; + +DepthRasterWorker::DepthRasterWorker(int maxVerts, int maxIndices) : maxVerts_(maxVerts), maxIndices_(maxIndices) { + // x, y and z, each with room for a group of four triangles past the end. + screenVerts_ = (int *)AllocateMemoryPages((maxVerts_ * sizeof(int) + 384) * 3, MEM_PROT_READ | MEM_PROT_WRITE); + current_ = NewBatch(); + thread_ = std::thread([this] { Run(); }); +} + +DepthRasterWorker::~DepthRasterWorker() { + Wait(); + { + std::lock_guard guard(mutex_); + quit_ = true; + } + work_.notify_one(); + thread_.join(); + for (DepthBatch *batch : all_) { + FreeBatch(batch); + } + FreeMemoryPages(screenVerts_, (maxVerts_ * sizeof(int) + 384) * 3); +} + +DepthBatch *DepthRasterWorker::NewBatch() { + DepthBatch *batch = new DepthBatch(); + batch->draws.reserve(16); + all_.push_back(batch); + return batch; +} + +void DepthRasterWorker::FreeBatch(DepthBatch *batch) { + FreeAlignedMemory(batch->transformed); + FreeAlignedMemory(batch->indices); + delete batch; +} + +void DepthRasterWorker::Reserve(int vertices, int indices) { + DepthBatch &batch = *current_; + const int needVerts = batch.vertexCount + vertices; + if (needVerts > batch.vertexCapacity) { + const int capacity = std::min(std::max(needVerts, batch.vertexCapacity * 2), maxVerts_); + float *grown = (float *)AllocateAlignedMemory(capacity * 4 * sizeof(float), 16); + if (batch.transformed) { + memcpy(grown, batch.transformed, batch.vertexCount * 4 * sizeof(float)); + FreeAlignedMemory(batch.transformed); + } + batch.transformed = grown; + batch.vertexCapacity = capacity; + } + const int needIndices = batch.indexCount + indices; + if (needIndices > batch.indexCapacity) { + const int capacity = std::min(std::max(needIndices, batch.indexCapacity * 2), maxIndices_); + uint16_t *grown = (uint16_t *)AllocateAlignedMemory(capacity * sizeof(uint16_t), 16); + if (batch.indices) { + memcpy(grown, batch.indices, batch.indexCount * sizeof(uint16_t)); + FreeAlignedMemory(batch.indices); + } + batch.indices = grown; + batch.indexCapacity = capacity; + } +} + +void DepthRasterWorker::Submit() { + if (current_->draws.empty()) { + return; + } + // Wipeout submits about 40 a frame. Past this many, wait for one to come back. + constexpr size_t MAX_BATCHES = 128; + std::unique_lock lock(mutex_); + for (const DepthDraw &draw : current_->draws) { + const u32 low = (draw.depthAddr & VRAM_OFFSET_MASK) + draw.scissor.y1 * draw.depthStride * 2; + const u32 high = (draw.depthAddr & VRAM_OFFSET_MASK) + (draw.scissor.y2 + 1) * draw.depthStride * 2; + if (!busy_) { + busyRanges_.clear(); + busyLow_ = low; + busyHigh_ = high; + busy_ = true; + } else { + busyLow_ = std::min((u32)busyLow_, low); + busyHigh_ = std::max((u32)busyHigh_, high); + } + bool merged = false; + for (auto &range : busyRanges_) { + if (low <= range.second && high >= range.first) { + range.first = std::min(range.first, low); + range.second = std::max(range.second, high); + merged = true; + break; + } + } + if (!merged) { + busyRanges_.emplace_back(low, high); + } + } + queue_.push_back(current_); + work_.notify_one(); + if (free_.empty() && all_.size() < MAX_BATCHES) { + lock.unlock(); + current_ = NewBatch(); + return; + } + done_.wait(lock, [this] { return !free_.empty(); }); + current_ = free_.back(); + free_.pop_back(); + current_->draws.clear(); + current_->vertexCount = 0; + current_->indexCount = 0; +} + +void DepthRasterWorker::Wait() { + Submit(); + if (!busy_) { + return; + } + std::unique_lock lock(mutex_); + done_.wait(lock, [this] { return queue_.empty() && running_ == 0; }); +} + +void DepthRasterWorker::WaitIfWriting(u32 addr, u32 size) { + if (!busy_) { + return; + } + const u32 low = addr & VRAM_OFFSET_MASK; + if (low >= busyHigh_ || low + size <= busyLow_) { + return; + } + std::unique_lock lock(mutex_); + bool overlaps = false; + for (const auto &range : busyRanges_) { + overlaps = overlaps || (low < range.second && low + size > range.first); + } + if (busy_ && overlaps) { + done_.wait(lock, [this] { return queue_.empty() && running_ == 0; }); + } +} + +void DepthRasterWorker::TakeTimes(double *cull, double *raster) { + std::lock_guard guard(mutex_); + *cull += cullTime_; + *raster += rasterTime_; + cullTime_ = 0.0; + rasterTime_ = 0.0; +} + +void DepthRasterWorker::Run() { + SetCurrentThreadName("DepthRaster"); + std::unique_lock lock(mutex_); + while (true) { + work_.wait(lock, [this] { return quit_ || !queue_.empty(); }); + if (quit_) { + break; + } + DepthBatch *batch = queue_.front(); + queue_.erase(queue_.begin()); + running_++; + lock.unlock(); + + double cull = 0.0, raster = 0.0; + Rasterize(*batch, &cull, &raster); + + lock.lock(); + running_--; + cullTime_ += cull; + rasterTime_ += raster; + free_.push_back(batch); + if (queue_.empty() && running_ == 0) { + busy_ = false; + } + done_.notify_all(); + } +} + +void DepthRasterWorker::Rasterize(const DepthBatch &batch, double *cull, double *raster) { + int *tx = screenVerts_; + int *ty = screenVerts_ + maxVerts_; + float *tz = (float *)(screenVerts_ + maxVerts_ * 2); + for (const DepthDraw &draw : batch.draws) { + const float *vertices = batch.transformed + 4 * draw.vertexOffset; + const uint16_t *indices = batch.indices + draw.indexOffset; + const DepthScissor tileScissor = draw.scissor.Tile(0, 1); + + double start = time_now_d(); + int outVertCount = 0; + switch (draw.prim) { + case GE_PRIM_RECTANGLES: + outVertCount = DepthRasterClipIndexedRectangles(tx, ty, tz, vertices, indices, draw, tileScissor); + break; + case GE_PRIM_TRIANGLES: + outVertCount = DepthRasterClipIndexedTriangles(tx, ty, tz, vertices, indices, draw, tileScissor, maxVerts_); + break; + default: + _dbg_assert_(false); + break; + } + double mid = time_now_d(); + *cull += mid - start; + if (outVertCount > 0 && Memory::IsValid4AlignedAddress(draw.depthAddr)) { + u16 *depthPtr = (uint16_t *)Memory::GetPointerWriteUnchecked(draw.depthAddr); + DepthRasterScreenVerts(depthPtr, draw.depthStride, tx, ty, tz, outVertCount, draw, tileScissor, batch.lowQ); + *raster += time_now_d() - mid; + } + } +} diff --git a/GPU/Common/DepthRaster.h b/GPU/Common/DepthRaster.h index c7fdbc7581..941da558db 100644 --- a/GPU/Common/DepthRaster.h +++ b/GPU/Common/DepthRaster.h @@ -1,5 +1,11 @@ #pragma once +#include +#include +#include +#include +#include + #include "Common/CommonTypes.h" #include "GPU/ge_constants.h" @@ -58,3 +64,71 @@ void DecodeAndTransformForDepthRaster(float *dest, const float *worldviewproj, c void TransformPredecodedForDepthRaster(float *dest, const float *worldviewproj, const void *decodedVertexData, const VertexDecoder *dec, int count); void ConvertPredecodedThroughForDepthRaster(float *dest, const void *decodedVertexData, const VertexDecoder *dec, int count); void DepthRasterScreenVerts(uint16_t *depth, int depthStride, const int *tx, const int *ty, const float *tz, int count, const DepthDraw &draw, const DepthScissor scissor, bool lowQ); + +// A batch of queued depth draws, with their transformed vertices and indices. +struct DepthBatch { + float *transformed = nullptr; + uint16_t *indices = nullptr; + std::vector draws; + int vertexCount = 0; + int indexCount = 0; + int vertexCapacity = 0; + int indexCapacity = 0; + bool lowQ = false; +}; + +// Rasterizes batches of depth draws on a thread of its own, in the order they're submitted, so +// that it overlaps with emulating the CPU and the rest of the GE. The PSP memory it writes is only +// the depth buffers of the draws, so anything reading those has to Wait() first (or WaitIfWriting). +class DepthRasterWorker { +public: + DepthRasterWorker(int maxVerts, int maxIndices); + ~DepthRasterWorker(); + + // The batch to add draws to. + DepthBatch &Current() { return *current_; } + // Makes room in the current batch for this many more vertices and indices. + void Reserve(int vertices, int indices); + // Starts rasterizing the current batch (if it has draws) and makes an empty one current. + void Submit(); + // Submits the current batch and waits until all are rasterized. + void Wait(); + // Waits only if a batch being rasterized may write to [addr, addr + size). + void WaitIfWriting(u32 addr, u32 size); + + // Time spent culling and rasterizing since the last call, for the stats. + void TakeTimes(double *cull, double *raster); + +private: + DepthBatch *NewBatch(); + static void FreeBatch(DepthBatch *batch); + void Run(); + void Rasterize(const DepthBatch &batch, double *cull, double *raster); + + const int maxVerts_; + const int maxIndices_; + int *screenVerts_ = nullptr; + + std::vector all_; + std::vector free_; + std::vector queue_; // Oldest first. + DepthBatch *current_ = nullptr; + int running_ = 0; // Batches taken off the queue and not yet done. + + // The depth buffer bytes the submitted, unfinished batches can write. + // The depth buffers being written (VRAM offsets, end exclusive), and their span, which only grows + // while busy_, so it can be checked without the lock. + std::vector> busyRanges_; + std::atomic busyLow_{}; + std::atomic busyHigh_{}; + std::atomic busy_{}; + + double cullTime_ = 0.0; + double rasterTime_ = 0.0; + + bool quit_ = false; + std::mutex mutex_; + std::condition_variable work_; + std::condition_variable done_; + std::thread thread_; +}; diff --git a/GPU/Common/DrawEngineCommon.cpp b/GPU/Common/DrawEngineCommon.cpp index 093c367363..1d23403867 100644 --- a/GPU/Common/DrawEngineCommon.cpp +++ b/GPU/Common/DrawEngineCommon.cpp @@ -1126,20 +1126,15 @@ bool DrawEngineCommon::DescribeCodePtr(const u8 *ptr, std::string &name) const { enum { DEPTH_TRANSFORMED_MAX_VERTS = VERTEX_BUFFER_MAX, - DEPTH_TRANSFORMED_BYTES = DEPTH_TRANSFORMED_MAX_VERTS * 4 * sizeof(float), - DEPTH_SCREENVERTS_COMPONENT_COUNT = VERTEX_BUFFER_MAX, - DEPTH_SCREENVERTS_COMPONENT_BYTES = DEPTH_SCREENVERTS_COMPONENT_COUNT * sizeof(int) + 384, - DEPTH_SCREENVERTS_TOTAL_BYTES = DEPTH_SCREENVERTS_COMPONENT_BYTES * 3, DEPTH_INDEXBUFFER_MAX_INDICES = DEPTH_TRANSFORMED_MAX_VERTS * 3, - DEPTH_INDEXBUFFER_BYTES = DEPTH_INDEXBUFFER_MAX_INDICES * sizeof(uint16_t), }; // We process vertices for depth rendering in several stages: -// First, we transform and collect vertices into depthTransformed_ (4-vectors, xyzw). +// First, we transform and collect vertices into a DepthBatch (4-vectors, xyzw). // Then, we group and cull the vertices into four-triangle groups, which are placed in -// depthScreenVerts_, with x, y and z separated into different part of the array. +// the worker's screen vertices, with x, y and z separated into different part of the array. // (Alternatively, if drawing rectangles, they're just added linearly). -// After that, we send these groups out for SIMD setup and rasterization. +// After that, the worker sends these groups out for SIMD setup and rasterization. void DrawEngineCommon::InitDepthRaster() { switch ((DepthRasterMode)g_Config.iDepthRasterMode) { case DepthRasterMode::DEFAULT: @@ -1154,23 +1149,13 @@ void DrawEngineCommon::InitDepthRaster() { } if (useDepthRaster_) { - depthDraws_.reserve(256); - depthTransformed_ = (float *)AllocateMemoryPages(DEPTH_TRANSFORMED_BYTES, MEM_PROT_READ | MEM_PROT_WRITE); - depthScreenVerts_ = (int *)AllocateMemoryPages(DEPTH_SCREENVERTS_TOTAL_BYTES, MEM_PROT_READ | MEM_PROT_WRITE); - depthIndices_ = (uint16_t *)AllocateMemoryPages(DEPTH_INDEXBUFFER_BYTES, MEM_PROT_READ | MEM_PROT_WRITE); + depthWorker_ = new DepthRasterWorker(DEPTH_TRANSFORMED_MAX_VERTS, DEPTH_INDEXBUFFER_MAX_INDICES); } } void DrawEngineCommon::ShutdownDepthRaster() { - if (depthTransformed_) { - FreeMemoryPages(depthTransformed_, DEPTH_TRANSFORMED_BYTES); - } - if (depthScreenVerts_) { - FreeMemoryPages(depthScreenVerts_, DEPTH_SCREENVERTS_TOTAL_BYTES); - } - if (depthIndices_) { - FreeMemoryPages(depthIndices_, DEPTH_INDEXBUFFER_BYTES); - } + delete depthWorker_; + depthWorker_ = nullptr; } Mat4F32 ComputeFinalProjMatrix() { @@ -1191,7 +1176,7 @@ Mat4F32 ComputeFinalProjMatrix() { return m; } -// numDecoded is how many vertices this draw will write to depthTransformed_, which is not the +// numDecoded is how many vertices this draw will write to the batch, which is not the // same thing as vertexCount (the number of indices) - an indexed draw can decode far more // vertices than it has indices, or far fewer. bool DrawEngineCommon::CalculateDepthDraw(DepthDraw *draw, GEPrimitiveType prim, int vertexCount, int numDecoded) { @@ -1240,18 +1225,21 @@ bool DrawEngineCommon::CalculateDepthDraw(DepthDraw *draw, GEPrimitiveType prim, _dbg_assert_(gstate.isDepthWriteEnabled()); } - if (depthVertexCount_ + numDecoded > DEPTH_TRANSFORMED_MAX_VERTS) { - // Can't add more. We need to flush. + if (numDecoded > DEPTH_TRANSFORMED_MAX_VERTS || vertexCount > DEPTH_INDEXBUFFER_MAX_INDICES) { return false; } - if (depthIndexCount_ + vertexCount > DEPTH_INDEXBUFFER_MAX_INDICES) { - return false; + const DepthBatch *batch = &depthWorker_->Current(); + if (batch->vertexCount + numDecoded > DEPTH_TRANSFORMED_MAX_VERTS || batch->indexCount + vertexCount > DEPTH_INDEXBUFFER_MAX_INDICES) { + // Full, start on another. + depthWorker_->Submit(); + batch = &depthWorker_->Current(); } draw->depthAddr = gstate.getDepthBufRawAddress() | 0x04000000; draw->depthStride = gstate.DepthBufStride(); - draw->vertexOffset = depthVertexCount_; - draw->indexOffset = depthIndexCount_; + depthWorker_->Reserve(numDecoded, vertexCount); + draw->vertexOffset = batch->vertexCount; + draw->indexOffset = batch->indexCount; draw->vertexCount = vertexCount; draw->cullEnabled = gstate.isCullEnabled(); draw->cullMode = gstate.getCullMode(); @@ -1289,6 +1277,7 @@ void DrawEngineCommon::DepthRasterSubmitRaw(GEPrimitiveType prim, const VertexDe if (!CalculateDepthDraw(&draw, prim, vertexCount, willDecode)) { return; } + DepthBatch &batch = depthWorker_->Current(); TimeCollector collectStat(&gpuStats.perFrame.msPrepareDepth, g_coreCollectDebugStats); @@ -1302,24 +1291,14 @@ void DrawEngineCommon::DepthRasterSubmitRaw(GEPrimitiveType prim, const VertexDe break; } // Decode the verts (and at the same time apply morphing/skinning). Simple. - DecodeAndTransformForDepthRaster(depthTransformed_ + (draw.vertexOffset + numDecoded) * 4, worldviewproj, dv.verts, dv.indexLowerBound, dv.indexUpperBound, dec, vertTypeID); + DecodeAndTransformForDepthRaster(batch.transformed + (draw.vertexOffset + numDecoded) * 4, worldviewproj, dv.verts, dv.indexLowerBound, dv.indexUpperBound, dec, vertTypeID); numDecoded += dv.indexUpperBound - dv.indexLowerBound + 1; } // Copy indices. - memcpy(depthIndices_ + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount); + memcpy(batch.indices + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount); - // Commit - depthIndexCount_ += vertexCount; - depthVertexCount_ += numDecoded; - - if (depthDraws_.empty()) { - rasterTimeStart_ = time_now_d(); - } - - depthDraws_.push_back(draw); - - // FlushQueuedDepth(); + CommitDepthDraw(draw, vertexCount, numDecoded); } void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *inVerts, int numDecoded, const VertexDecoder *dec, int vertexCount) { @@ -1331,6 +1310,7 @@ void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *i if (!CalculateDepthDraw(&draw, prim, vertexCount, numDecoded)) { return; } + DepthBatch &batch = depthWorker_->Current(); TimeCollector collectStat(&gpuStats.perFrame.msPrepareDepth, g_coreCollectDebugStats); @@ -1338,77 +1318,55 @@ void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *i _dbg_assert_(prim != GE_PRIM_TRIANGLE_STRIP && prim != GE_PRIM_TRIANGLE_FAN); if (dec->throughmode) { - ConvertPredecodedThroughForDepthRaster(depthTransformed_ + 4 * draw.vertexOffset, decoded_, dec, numDecoded); + ConvertPredecodedThroughForDepthRaster(batch.transformed + 4 * draw.vertexOffset, decoded_, dec, numDecoded); } else { if (dec->VertexType() & (GE_VTYPE_WEIGHT_MASK | GE_VTYPE_MORPHCOUNT_MASK)) { return; } float worldviewproj[16]; ComputeFinalProjMatrix().Store(worldviewproj); - TransformPredecodedForDepthRaster(depthTransformed_ + 4 * draw.vertexOffset, worldviewproj, decoded_, dec, numDecoded); + TransformPredecodedForDepthRaster(batch.transformed + 4 * draw.vertexOffset, worldviewproj, decoded_, dec, numDecoded); } // Copy indices. - memcpy(depthIndices_ + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount); + memcpy(batch.indices + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount); - // Commit - depthIndexCount_ += vertexCount; - depthVertexCount_ += numDecoded; + CommitDepthDraw(draw, vertexCount, numDecoded); +} - depthDraws_.push_back(draw); +void DrawEngineCommon::CommitDepthDraw(const DepthDraw &draw, int vertexCount, int numDecoded) { + DepthBatch &batch = depthWorker_->Current(); + if (batch.draws.empty()) { + batch.lowQ = g_Config.iDepthRasterMode == (int)DepthRasterMode::LOW_QUALITY; + if (rasterTimeStart_ == 0.0) { + rasterTimeStart_ = time_now_d(); + } + } + batch.indexCount += vertexCount; + batch.vertexCount += numDecoded; + batch.draws.push_back(draw); +} - if (depthDraws_.empty()) { - rasterTimeStart_ = time_now_d(); +void DrawEngineCommon::SubmitQueuedDepth() { + if (depthWorker_) { + depthWorker_->Submit(); } - // FlushQueuedDepth(); } void DrawEngineCommon::FlushQueuedDepth() { + if (!depthWorker_) { + return; + } + depthWorker_->Wait(); if (rasterTimeStart_ != 0.0) { gpuStats.perFrame.msRasterTimeAvailable += time_now_d() - rasterTimeStart_; rasterTimeStart_ = 0.0; } + depthWorker_->TakeTimes(&gpuStats.perFrame.msCullDepth, &gpuStats.perFrame.msRasterizeDepth); +} - const bool collectStats = g_coreCollectDebugStats; - const bool lowQ = g_Config.iDepthRasterMode == (int)DepthRasterMode::LOW_QUALITY; - for (const auto &draw : depthDraws_) { - int *tx = depthScreenVerts_; - int *ty = depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT; - float *tz = (float *)(depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT * 2); - - int outVertCount = 0; - - const float *vertices = depthTransformed_ + 4 * draw.vertexOffset; - const uint16_t *indices = depthIndices_ + draw.indexOffset; - - DepthScissor tileScissor = draw.scissor.Tile(0, 1); - - { - TimeCollector collectStat(&gpuStats.perFrame.msCullDepth, collectStats); - switch (draw.prim) { - case GE_PRIM_RECTANGLES: - outVertCount = DepthRasterClipIndexedRectangles(tx, ty, tz, vertices, indices, draw, tileScissor); - break; - case GE_PRIM_TRIANGLES: - outVertCount = DepthRasterClipIndexedTriangles(tx, ty, tz, vertices, indices, draw, tileScissor, DEPTH_SCREENVERTS_COMPONENT_COUNT); - break; - default: - _dbg_assert_(false); - break; - } - } - if (outVertCount > 0) { - TimeCollector collectStat(&gpuStats.perFrame.msRasterizeDepth, collectStats); - if (!Memory::IsValid4AlignedAddress(draw.depthAddr)) { - continue; - } - u16 *depthPtr = (uint16_t *)Memory::GetPointerWriteUnchecked(draw.depthAddr); - DepthRasterScreenVerts(depthPtr, draw.depthStride, tx, ty, tz, outVertCount, draw, tileScissor, lowQ); - } +void DrawEngineCommon::WaitForDepthWrites(u32 addr, u32 size) { + if (depthWorker_) { + depthWorker_->WaitIfWriting(addr, size); } - - // Reset queue - depthIndexCount_ = 0; - depthVertexCount_ = 0; - depthDraws_.clear(); } diff --git a/GPU/Common/DrawEngineCommon.h b/GPU/Common/DrawEngineCommon.h index e8de52fa66..9d7947e6de 100644 --- a/GPU/Common/DrawEngineCommon.h +++ b/GPU/Common/DrawEngineCommon.h @@ -33,6 +33,7 @@ class VertexDecoder; struct DepthDraw; +class DepthRasterWorker; enum { VERTEX_BUFFER_MAX = 65536, @@ -159,7 +160,12 @@ public: return decoded_ + 12 * 65536; } + // Rasterizes the queued depth draws, and waits for them. void FlushQueuedDepth(); + // Starts rasterizing the queued depth draws, without waiting. + void SubmitQueuedDepth(); + // Waits if depth rasterization in progress may write to these bytes. + void WaitForDepthWrites(u32 addr, u32 size); protected: bool CheckClipFlags(bool useHwTransform) const; @@ -181,6 +187,7 @@ protected: void DepthRasterSubmitRaw(GEPrimitiveType prim, const VertexDecoder *dec, uint32_t vertTypeID, int vertexCount); void DepthRasterPredecoded(GEPrimitiveType prim, const void *inVerts, int numDecoded, const VertexDecoder *dec, int vertexCount); bool CalculateDepthDraw(DepthDraw *draw, GEPrimitiveType prim, int vertexCount, int numDecoded); + void CommitDepthDraw(const DepthDraw &draw, int vertexCount, int numDecoded); static inline int IndexSize(u32 vtype) { const u32 indexType = (vtype & GE_VTYPE_IDX_MASK); @@ -368,19 +375,12 @@ protected: // Software depth raster bool useDepthRaster_ = false; - float *depthTransformed_ = nullptr; - int *depthScreenVerts_ = nullptr; - uint16_t *depthIndices_ = nullptr; + DepthRasterWorker *depthWorker_ = nullptr; // Depth tracking ClipInfoFlags clipInfoFlags_{}; ClipInfoFlags lastClipInfoFlags_{}; // Flags at the last flush. For dirtying. - // Queue - int depthVertexCount_ = 0; - int depthIndexCount_ = 0; - std::vector depthDraws_; - double rasterTimeStart_ = 0.0; bool lastUseHwTransform_ = true; diff --git a/GPU/Common/FramebufferManagerCommon.h b/GPU/Common/FramebufferManagerCommon.h index 90df499b3d..fbc9a4656a 100644 --- a/GPU/Common/FramebufferManagerCommon.h +++ b/GPU/Common/FramebufferManagerCommon.h @@ -313,6 +313,9 @@ public: void SetDrawEngine(DrawEngineCommon *td) { drawEngine_ = td; } + DrawEngineCommon *GetDrawEngine() const { + return drawEngine_; + } void Init(int msaaLevel); virtual void BeginFrame(const DisplayLayoutConfig &config); diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp index ec77feee43..8a4763ff95 100644 --- a/GPU/Common/TextureCacheCommon.cpp +++ b/GPU/Common/TextureCacheCommon.cpp @@ -36,6 +36,7 @@ #include "Core/Debugger/MemBlockInfo.h" #include "Core/System.h" #include "Core/HW/Display.h" +#include "GPU/Common/DrawEngineCommon.h" #include "GPU/Common/FramebufferManagerCommon.h" #include "GPU/Common/TextureCacheCommon.h" #include "GPU/Common/TextureDecoder.h" @@ -399,6 +400,13 @@ SamplerCacheKey GetFramebufferSamplingParams(const GEState &gstate, u16 bufferWi return key; } +// Before reading texture data: it may be in a depth buffer the software depth rasterizer is writing. +void TextureCacheCommon::WaitForDepthWrites(u32 addr, u32 bytes) { + if (Memory::IsVRAMAddress(addr)) { + framebufferManager_->GetDrawEngine()->WaitForDepthWrites(addr, bytes); + } +} + static u32 ComputeTextureHash(TextureReplacer &replacer, u32 addr, int bufw, int w, int h, bool swizzled, const TexCacheEntry *entry) { const GETextureFormat format = entry->format; // Swizzled CLUT4 glyph atlases can fill below the visible UV range while remaining @@ -723,6 +731,9 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { // A video texture is new every frame by definition, so hashing it only confirms what // the VIDEO flag already said. const bool skipHash = isVideo; + if (!skipHash) { + WaitForDepthWrites(entry->addr, (textureBitsPerPixel[entry->format] * entry->bufw * h) / 8); + } const u32 newFullHash = skipHash ? 0 : ComputeTextureHash(replacer_, entry->addr, entry->bufw, w, h, swizzled, entry); if (skipHash || newFullHash != entry->fullhash) { // The texture changed. Throw it in the secondary cache. Then we'll create a new entry later. @@ -932,6 +943,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { const bool skipHash = isVideo; if (!(entry->status & TexStatus::IS_PPGE_ATLAS) && !skipHash) { + WaitForDepthWrites(entry->addr, (textureBitsPerPixel[entry->format] * entry->bufw * h) / 8); entry->fullhash = ComputeTextureHash(replacer_, entry->addr, entry->bufw, w, h, swizzled, entry); } @@ -2015,6 +2027,9 @@ TextureAlpha TextureCacheCommon::DecodeTextureLevel(u8 *out, int outPitch, GETex const u8 *texptr = isPPGE ? (PPGeAtlasGetData() + ppgeOffset) : Memory::GetPointerOrException(texaddr); const uint32_t byteSize = (textureBitsPerPixel[format] * bufw * h) / 8; + if (!isPPGE) { + WaitForDepthWrites(texaddr, byteSize); + } // Validate the texture data fits in mapped RAM, like the DXT path does. // texaddr/bufw/w/h are all guest-controlled via the GE display list. diff --git a/GPU/Common/TextureCacheCommon.h b/GPU/Common/TextureCacheCommon.h index 30549035c1..4f706cab18 100644 --- a/GPU/Common/TextureCacheCommon.h +++ b/GPU/Common/TextureCacheCommon.h @@ -397,6 +397,8 @@ public: protected: + void WaitForDepthWrites(u32 addr, u32 bytes); + bool PrepareBuildTexture(BuildTexturePlan &plan, TexCacheEntry *entry); // This updates nextTexture_ / nextFramebufferTexture_, which is then used by ApplyTexture. diff --git a/GPU/GPUCommon.cpp b/GPU/GPUCommon.cpp index 6c17295cc4..2072085c53 100644 --- a/GPU/GPUCommon.cpp +++ b/GPU/GPUCommon.cpp @@ -1212,7 +1212,7 @@ void GPUCommon::Execute_Ret(u32 op, u32 diff) { void GPUCommon::Execute_End(u32 op, u32 diff) { if (flushOnParams_) { - drawEngineCommon_->FlushQueuedDepth(); + drawEngineCommon_->SubmitQueuedDepth(); Flush(); } diff --git a/GPU/GPUCommonHW.cpp b/GPU/GPUCommonHW.cpp index 269435a77a..f4abb810fb 100644 --- a/GPU/GPUCommonHW.cpp +++ b/GPU/GPUCommonHW.cpp @@ -927,7 +927,7 @@ void GPUCommonHW::Execute_Prim(u32 op, u32 diff) { } if (changed) { - drawEngineCommon_->FlushQueuedDepth(); + drawEngineCommon_->SubmitQueuedDepth(); } if (gstate_c.dirty & DIRTY_VERTEXSHADER_STATE) { @@ -1288,7 +1288,7 @@ void GPUCommonHW::Execute_Bezier(u32 op, u32 diff) { bool changed; VirtualFramebuffer *vfb = framebufferManager_->SetRenderFrameBuffer(gstate_c.IsDirty(DIRTY_FRAMEBUF), gstate_c.skipDrawReason, &changed); if (changed) { - drawEngineCommon_->FlushQueuedDepth(); + drawEngineCommon_->SubmitQueuedDepth(); } if (gstate_c.skipDrawReason & (SKIPDRAW_SKIPFRAME | SKIPDRAW_NON_DISPLAYED_FB)) { // TODO: Should this eat some cycles? Probably yes. Not sure if important. @@ -1359,7 +1359,7 @@ void GPUCommonHW::Execute_Spline(u32 op, u32 diff) { bool changed; VirtualFramebuffer *vfb = framebufferManager_->SetRenderFrameBuffer(gstate_c.IsDirty(DIRTY_FRAMEBUF), gstate_c.skipDrawReason, &changed); if (changed) { - drawEngineCommon_->FlushQueuedDepth(); + drawEngineCommon_->SubmitQueuedDepth(); } if (gstate_c.skipDrawReason & (SKIPDRAW_SKIPFRAME | SKIPDRAW_NON_DISPLAYED_FB)) { // TODO: Should this eat some cycles? Probably yes. Not sure if important. @@ -1468,6 +1468,7 @@ void GPUCommonHW::Execute_TexLevel(u32 op, u32 diff) { void GPUCommonHW::Execute_LoadClut(u32 op, u32 diff) { gstate_c.Dirty(DIRTY_TEXTURE_PARAMS); + drawEngineCommon_->WaitForDepthWrites(gstate.getClutAddress(), gstate.getClutLoadBytes()); textureCache_->LoadClut(gstate.getClutAddress(), gstate.getClutLoadBytes(), &recorder_); } @@ -1741,6 +1742,10 @@ void GPUCommonHW::Execute_TexFlush(u32 op, u32 diff) { framebufferManager_->DiscardFramebufferCopy(); } +void GPUCommonHW::FlushPendingDrawing() { + drawEngineCommon_->FlushQueuedDepth(); +} + u32 GPUCommonHW::DrawSync(int mode) { drawEngineCommon_->FlushQueuedDepth(); return GPUCommon::DrawSync(mode); diff --git a/GPU/GPUCommonHW.h b/GPU/GPUCommonHW.h index 69640402f5..03b967dc15 100644 --- a/GPU/GPUCommonHW.h +++ b/GPU/GPUCommonHW.h @@ -45,6 +45,7 @@ public: void InvalidateCache(u32 addr, int size, GPUInvalidationType type) override; u32 DrawSync(int mode) override; + void FlushPendingDrawing() override; int ListSync(int listid, int mode) override; bool FramebufferDirty() override;