SoftGPU: Fix the stencil test's comparison shortcuts

The stencil test compares ref against the buffer's value, the other way
around from the alpha test, but shared its shortcuts: GREATER with ref 0 and
LESS with ref 0xFF became NOTEQUAL instead of NEVER, so those draws wrote
pixels the PSP rejects. Its own shortcuts are mirrored (gpu/probe exp120:
every function and op in 8888, 5551 and 4444, exact).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Henrik RydgårdandClaude Opus 5.5 committed 2026-10-02 15:09:43 -06:00
1 parent 47261f8b86
commit b1744e809e
1 file changed
+14 -1
+14 -1
View File
@@ -41,6 +41,19 @@ static inline GEComparison OptimizeRefByteCompare(GEComparison func, u8 ref) {
return func;
}
// The stencil test compares the other way around, ref against the buffer's value.
static inline GEComparison OptimizeStencilRefCompare(GEComparison func, u8 ref) {
if (ref == 0 && func == GE_COMP_GREATER)
return GE_COMP_NEVER;
if (ref == 0xFF && func == GE_COMP_LESS)
return GE_COMP_NEVER;
if (ref == 0 && func == GE_COMP_LEQUAL)
return GE_COMP_ALWAYS;
if (ref == 0xFF && func == GE_COMP_GEQUAL)
return GE_COMP_ALWAYS;
return func;
}
static inline PixelBlendFactor OptimizeAlphaFactor(uint32_t color) {
if (color == 0x00000000)
return PixelBlendFactor::ZERO;
@@ -84,7 +97,7 @@ void ComputePixelFuncID(PixelFuncID *id) {
if (id->stencilTest) {
id->stencilTestRef = gstate.getStencilTestRef() & gstate.getStencilTestMask();
id->stencilTestFunc = OptimizeRefByteCompare(gstate.getStencilTestFunction(), id->stencilTestRef);
id->stencilTestFunc = OptimizeStencilRefCompare(gstate.getStencilTestFunction(), id->stencilTestRef);
id->hasStencilTestMask = gstate.getStencilTestMask() != 0xFF && gstate.FrameBufFormat() != GE_FORMAT_565;
// Stencil can't be written on 565, and any invalid op acts like KEEP, which is 0.