From 89fabe21fb005e1583930450ded23e85fe939459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Thu, 8 Oct 2026 10:45:57 +0200 Subject: [PATCH] Rebuild the stencil from alpha after reinterpreting a framebuffer On the PSP the stencil is the alpha bits, so after the colors of another format are reinterpreted into a 5551, 4444 or 8888 buffer, its stencil has to follow the new alpha. Ultimate Ghosts 'n Goblins sets 5551 alpha with a masked 565 blue write and then stencil tests on it; Vulkan drew its glow through a stale stencil. The memory stencil upload's pipeline and passes are shared. Co-Authored-By: Claude Opus 5.5 --- GPU/Common/FramebufferManagerCommon.cpp | 4 + GPU/Common/FramebufferManagerCommon.h | 3 + GPU/Common/StencilCommon.cpp | 233 +++++++++++++++--------- 3 files changed, 150 insertions(+), 90 deletions(-) diff --git a/GPU/Common/FramebufferManagerCommon.cpp b/GPU/Common/FramebufferManagerCommon.cpp index 47f228d496..29878a51b1 100644 --- a/GPU/Common/FramebufferManagerCommon.cpp +++ b/GPU/Common/FramebufferManagerCommon.cpp @@ -913,6 +913,10 @@ void FramebufferManagerCommon::CopyToColorFromOverlappingFramebuffers(VirtualFra // We should also copy the depth buffer in this case! BlitFramebufferDepth(src, dst, true); } + + if (pipeline && src->fb_format != dst->fb_format) { + WriteStencilFromFramebufferAlpha(dst); + } } } diff --git a/GPU/Common/FramebufferManagerCommon.h b/GPU/Common/FramebufferManagerCommon.h index bfe5db0d47..5afa1a0ee4 100644 --- a/GPU/Common/FramebufferManagerCommon.h +++ b/GPU/Common/FramebufferManagerCommon.h @@ -346,6 +346,7 @@ public: void UpdateFromMemory(u32 addr, int size); void ApplyClearToMemory(int x1, int y1, int x2, int y2, u32 clearColor); bool PerformWriteStencilFromMemory(u32 addr, int size, WriteStencil flags); + void WriteStencilFromFramebufferAlpha(VirtualFramebuffer *dst); // We changed our depth mode, gotta start over. // Ideally, we should convert depth buffers here, not just clear them. @@ -543,6 +544,8 @@ protected: void NotifyRenderFramebufferSwitched(VirtualFramebuffer *prevVfb, VirtualFramebuffer *vfb, bool isClearingDepth); void BlitFramebufferDepth(VirtualFramebuffer *src, VirtualFramebuffer *dst, bool allowSizeMismatch = false); + void EnsureStencilWritePipeline(bool useExportShader); + void DrawStencilWritePasses(GEBufferFormat format, int values, u8 usedBits, bool useExportShader); void ResizeFramebufFBO(VirtualFramebuffer *vfb, int w, int h, bool force = false, bool skipCopy = false); diff --git a/GPU/Common/StencilCommon.cpp b/GPU/Common/StencilCommon.cpp index 47ea8fe0c3..c63cdfc690 100644 --- a/GPU/Common/StencilCommon.cpp +++ b/GPU/Common/StencilCommon.cpp @@ -162,6 +162,147 @@ void GenerateStencilVs(char *buffer, const ShaderLanguageDesc &lang) { writer.EndVSMain(varyings); } +void FramebufferManagerCommon::EnsureStencilWritePipeline(bool useExportShader) { + if (stencilWritePipeline_) { + return; + } + + using namespace Draw; + const ShaderLanguageDesc &shaderLanguageDesc = draw_->GetShaderLanguageDesc(); + + char *fsCode = new char[8192]; + char *vsCode = new char[8192]; + GenerateStencilFs(fsCode, shaderLanguageDesc, draw_->GetBugs(), useExportShader); + GenerateStencilVs(vsCode, shaderLanguageDesc); + + _assert_msg_(strlen(fsCode) < 8192, "StenFS length error: %d", (int)strlen(fsCode)); + _assert_msg_(strlen(vsCode) < 8192, "StenVS length error: %d", (int)strlen(vsCode)); + + ShaderModule *stencilUploadFs = draw_->CreateShaderModule(ShaderStage::Fragment, shaderLanguageDesc.shaderLanguage, (const uint8_t *)fsCode, strlen(fsCode), "stencil_fs"); + ShaderModule *stencilUploadVs = draw_->CreateShaderModule(ShaderStage::Vertex, shaderLanguageDesc.shaderLanguage, (const uint8_t *)vsCode, strlen(vsCode), "stencil_vs"); + + _assert_(stencilUploadFs && stencilUploadVs); + + InputLayoutDesc desc = { + 8, + { + { SEM_POSITION, DataFormat::R32G32_FLOAT, 0 }, + }, + }; + InputLayout *inputLayout = draw_->CreateInputLayout(desc); + + BlendState *blendOff = draw_->CreateBlendState({ false, 0x8 }); + DepthStencilStateDesc dsDesc{}; + dsDesc.stencilEnabled = true; + dsDesc.stencil.compareOp = Comparison::ALWAYS; + dsDesc.stencil.depthFailOp = StencilOp::REPLACE; + dsDesc.stencil.failOp = StencilOp::REPLACE; + dsDesc.stencil.passOp = StencilOp::REPLACE; + DepthStencilState *stencilWrite = draw_->CreateDepthStencilState(dsDesc); + RasterState *rasterNoCull = draw_->CreateRasterState({}); + + PipelineDesc stencilWriteDesc{ + Primitive::TRIANGLE_LIST, + { stencilUploadVs, stencilUploadFs }, + inputLayout, stencilWrite, blendOff, rasterNoCull, &stencilUBDesc, + }; + stencilWritePipeline_ = draw_->CreateGraphicsPipeline(stencilWriteDesc, "stencil_upload"); + _assert_(stencilWritePipeline_); + + delete[] fsCode; + delete[] vsCode; + + rasterNoCull->Release(); + blendOff->Release(); + stencilWrite->Release(); + inputLayout->Release(); + + stencilUploadFs->Release(); + stencilUploadVs->Release(); + + SamplerStateDesc descNearest{}; + stencilWriteSampler_ = draw_->CreateSamplerState(descNearest); +} + +// Draws the stencil write pipeline over the whole target, with the source's alpha bound as the texture. +void FramebufferManagerCommon::DrawStencilWritePasses(GEBufferFormat format, int values, u8 usedBits, bool useExportShader) { + // Fullscreen triangle coordinates. + static const float positions[6] = { + 0.0, 0.0, + 1.0, 0.0, + 0.0, 1.0, + }; + + if (useExportShader) { + // We only need to do one pass if using an export shader. + StencilUB ub{}; + draw_->SetStencilParams(0xFF, 0xFF, 0xFF); + draw_->UpdateDynamicUniformBuffer(&ub, sizeof(ub)); + draw_->DrawUP(positions, 3); + return; + } + + for (int i = 1; i < values; i += i) { + if (!(usedBits & i)) { + // It's already zero, let's skip it. + continue; + } + StencilUB ub{}; + if (format == GE_FORMAT_4444) { + draw_->SetStencilParams(0xFF, (i << 4) | i, 0xFF); + ub.stencilValue = i * (16.0f / 255.0f); + } else if (format == GE_FORMAT_5551) { + draw_->SetStencilParams(0xFF, 0xFF, 0xFF); + ub.stencilValue = i * (128.0f / 255.0f); + } else { + draw_->SetStencilParams(0xFF, i, 0xFF); + ub.stencilValue = i * (1.0f / 255.0f); + } + draw_->UpdateDynamicUniformBuffer(&ub, sizeof(ub)); + draw_->DrawUP(positions, 3); + } +} + +// On the PSP the stencil is the alpha bits, so after reinterpreting the colors of another format into a buffer, its +// stencil has to follow the new alpha (Ultimate Ghosts 'n Goblins sets 5551 alpha through a masked 565 blue write, +// then stencil tests on it). +void FramebufferManagerCommon::WriteStencilFromFramebufferAlpha(VirtualFramebuffer *dst) { + using namespace Draw; + + int values = 0; + switch (dst->fb_format) { + case GE_FORMAT_5551: values = 2; break; + case GE_FORMAT_4444: values = 16; break; + case GE_FORMAT_8888: values = 256; break; + default: return; + } + if (!dst->fbo) { + return; + } + + const bool useExportShader = draw_->GetDeviceCaps().fragmentShaderStencilWriteSupported; + EnsureStencilWritePipeline(useExportShader); + + // The source can't be the target, so read the alpha from a copy. + const int w = dst->renderWidth; + const int h = dst->renderHeight; + Framebuffer *copy = GetTempFBO(TempFBO::COPY, w, h); + BlitUsingRaster(dst->fbo, 0.0f, 0.0f, (float)w, (float)h, copy, 0.0f, 0.0f, (float)w, (float)h, false, dst->renderScaleFactor, Get2DPipeline(DRAW2D_COPY_COLOR), "StencilFromAlpha_Copy"); + + draw_->BindFramebufferAsRenderTarget(dst->fbo, { RPAction::KEEP, RPAction::KEEP, RPAction::CLEAR }, "StencilFromAlpha"); + Viewport viewport = { 0.0f, 0.0f, (float)w, (float)h, 0.0f, 1.0f }; + draw_->SetViewport(viewport); + draw_->SetScissorRect(0, 0, w, h); + draw_->BindFramebufferAsTexture(copy, TEX_SLOT_PSP_TEXTURE, Aspect::COLOR_BIT, 0); + draw_->BindSamplerStates(TEX_SLOT_PSP_TEXTURE, 1, &stencilWriteSampler_); + draw_->BindPipeline(stencilWritePipeline_); + DrawStencilWritePasses(dst->fb_format, values, 0xFF, useExportShader); + + textureCache_->ForgetLastTexture(); + draw_->Invalidate(InvalidationFlags::CACHED_RENDER_STATE); + gstate_c.Dirty(DIRTY_ALL_RENDER_STATE); +} + bool FramebufferManagerCommon::PerformWriteStencilFromMemory(u32 addr, int size, WriteStencil flags) { using namespace Draw; @@ -212,69 +353,7 @@ bool FramebufferManagerCommon::PerformWriteStencilFromMemory(u32 addr, int size, textureCache_->ForgetLastTexture(); - if (!stencilWritePipeline_) { - const ShaderLanguageDesc &shaderLanguageDesc = draw_->GetShaderLanguageDesc(); - - char *fsCode = new char[8192]; - char *vsCode = new char[8192]; - GenerateStencilFs(fsCode, shaderLanguageDesc, draw_->GetBugs(), useExportShader); - GenerateStencilVs(vsCode, shaderLanguageDesc); - - _assert_msg_(strlen(fsCode) < 8192, "StenFS length error: %d", (int)strlen(fsCode)); - _assert_msg_(strlen(vsCode) < 8192, "StenVS length error: %d", (int)strlen(vsCode)); - - ShaderModule *stencilUploadFs = draw_->CreateShaderModule(ShaderStage::Fragment, shaderLanguageDesc.shaderLanguage, (const uint8_t *)fsCode, strlen(fsCode), "stencil_fs"); - ShaderModule *stencilUploadVs = draw_->CreateShaderModule(ShaderStage::Vertex, shaderLanguageDesc.shaderLanguage, (const uint8_t *)vsCode, strlen(vsCode), "stencil_vs"); - - _assert_(stencilUploadFs && stencilUploadVs); - - InputLayoutDesc desc = { - 8, - { - { SEM_POSITION, DataFormat::R32G32_FLOAT, 0 }, - }, - }; - InputLayout *inputLayout = draw_->CreateInputLayout(desc); - - BlendState *blendOff = draw_->CreateBlendState({ false, 0x8 }); - DepthStencilStateDesc dsDesc{}; - dsDesc.stencilEnabled = true; - dsDesc.stencil.compareOp = Comparison::ALWAYS; - dsDesc.stencil.depthFailOp = StencilOp::REPLACE; - dsDesc.stencil.failOp = StencilOp::REPLACE; - dsDesc.stencil.passOp = StencilOp::REPLACE; - DepthStencilState *stencilWrite = draw_->CreateDepthStencilState(dsDesc); - RasterState *rasterNoCull = draw_->CreateRasterState({}); - - PipelineDesc stencilWriteDesc{ - Primitive::TRIANGLE_LIST, - { stencilUploadVs, stencilUploadFs }, - inputLayout, stencilWrite, blendOff, rasterNoCull, &stencilUBDesc, - }; - stencilWritePipeline_ = draw_->CreateGraphicsPipeline(stencilWriteDesc, "stencil_upload"); - _assert_(stencilWritePipeline_); - - delete[] fsCode; - delete[] vsCode; - - rasterNoCull->Release(); - blendOff->Release(); - stencilWrite->Release(); - inputLayout->Release(); - - stencilUploadFs->Release(); - stencilUploadVs->Release(); - - SamplerStateDesc descNearest{}; - stencilWriteSampler_ = draw_->CreateSamplerState(descNearest); - } - - // Fullscreen triangle coordinates. - static const float positions[6] = { - 0.0, 0.0, - 1.0, 0.0, - 0.0, 1.0, - }; + EnsureStencilWritePipeline(useExportShader); bool useBlit = draw_->GetDeviceCaps().framebufferStencilBlitSupported; @@ -322,33 +401,7 @@ bool FramebufferManagerCommon::PerformWriteStencilFromMemory(u32 addr, int size, draw_->SetScissorRect(0, 0, w, h); draw_->BindPipeline(stencilWritePipeline_); - if (useExportShader) { - // We only need to do one pass if using an export shader. - StencilUB ub{}; - draw_->SetStencilParams(0xFF, 0xFF, 0xFF); - draw_->UpdateDynamicUniformBuffer(&ub, sizeof(ub)); - draw_->DrawUP(positions, 3); - } else { - for (int i = 1; i < values; i += i) { - if (!(usedBits & i)) { - // It's already zero, let's skip it. - continue; - } - StencilUB ub{}; - if (dstBuffer->fb_format == GE_FORMAT_4444) { - draw_->SetStencilParams(0xFF, (i << 4) | i, 0xFF); - ub.stencilValue = i * (16.0f / 255.0f); - } else if (dstBuffer->fb_format == GE_FORMAT_5551) { - draw_->SetStencilParams(0xFF, 0xFF, 0xFF); - ub.stencilValue = i * (128.0f / 255.0f); - } else { - draw_->SetStencilParams(0xFF, i, 0xFF); - ub.stencilValue = i * (1.0f / 255.0f); - } - draw_->UpdateDynamicUniformBuffer(&ub, sizeof(ub)); - draw_->DrawUP(positions, 3); - } - } + DrawStencilWritePasses(dstBuffer->fb_format, values, usedBits, useExportShader); if (useBlit) { // Note that scissors don't affect blits on other APIs than OpenGL, so might want to try to get rid of this.