Merge pull request #8346 from unknownbrackets/gpu-stencil

Adjust stencil ops for 5551 and 565 buffers

GPU/Common/GPUStateUtils.cpp

@@ -1119,3 +1119,148 @@ void ConvertBlendState(GenericBlendState &blendState, bool allowShaderBlend) {
 		blendState.setEquation(eqLookupNoMinMax[blendFuncEq], alphaEq);
 	}
 }
+
+static void ConvertStencilFunc5551(GenericStencilFuncState &state) {
+	// Decrement always zeros, so let's rewrite those to be safe (even if it's not 1.)
+	if (state.sFail == GE_STENCILOP_DECR)
+		state.sFail = GE_STENCILOP_ZERO;
+	if (state.zFail == GE_STENCILOP_DECR)
+		state.zFail = GE_STENCILOP_ZERO;
+	if (state.zPass == GE_STENCILOP_DECR)
+		state.zPass = GE_STENCILOP_ZERO;
+
+	state.writeMask = state.writeMask >= 0x80 ? 0xff : 0x00;
+
+	// Flaws:
+	// - INVERT should convert 1, 5, 0xFF to 0.  Currently it won't.
+	// - INCR twice shouldn't change the value.
+	// - REPLACE should write 0 for 0x00 - 0x7F, and non-zero for 0x80 - 0xFF.
+	// - Write mask may need double checking, but likely only the top bit matters.
+
+	const bool usesRef = state.sFail == GE_STENCILOP_REPLACE || state.zFail == GE_STENCILOP_REPLACE || state.zPass == GE_STENCILOP_REPLACE;
+	const u8 maskedRef = state.testRef & state.testMask;
+	const u8 usedRef = (state.testRef & 0x80) != 0 ? 0xFF : 0x00;
+
+	auto rewriteFunc = [&](GEComparison func, u8 ref, u8 mask = 0xFF) {
+		// We can only safely rewrite if it doesn't use the ref, or if the ref is the same.
+		if (!usesRef || usedRef == ref) {
+			state.testFunc = func;
+			state.testRef = ref;
+			state.testMask = mask;
+		}
+	};
+	auto rewriteRef = [&]() {
+		if (usesRef) {
+			// Rewrite the ref (for REPLACE) to 0x00 or 0xFF (the "best" values) if safe.
+			// This will only be called if the test doesn't need the ref.
+			state.testRef = usedRef;
+		}
+	};
+
+	// For 5551, we treat any non-zero value in the buffer as 255.  Only zero is treated as zero.
+	// See: https://github.com/hrydgard/ppsspp/pull/4150#issuecomment-26211193
+	switch (state.testFunc) {
+	case GE_COMP_NEVER:
+	case GE_COMP_ALWAYS:
+		// Fine as is.
+		rewriteRef();
+		break;
+	case GE_COMP_EQUAL: // maskedRef == maskedBuffer
+		if (maskedRef == 0) {
+			// Remove any mask, we might have bits less than 255 but that should not match.
+			rewriteFunc(GE_COMP_EQUAL, 0);
+		} else if (maskedRef == (0xFF & state.testMask) && state.testMask != 0) {
+			// Equal to 255, for our buffer, means not equal to zero.
+			rewriteFunc(GE_COMP_NOTEQUAL, 0);
+		} else {
+			// This should never pass, regardless of buffer value.  Only 0 and 255 are directly equal.
+			state.testFunc = GE_COMP_NEVER;
+			rewriteRef();
+		}
+		break;
+	case GE_COMP_NOTEQUAL: // maskedRef != maskedBuffer
+		if (maskedRef == 0) {
+			// Remove the mask, since our buffer might not be exactly 255.
+			rewriteFunc(GE_COMP_NOTEQUAL, 0);
+		} else if (maskedRef == (0xFF & state.testMask) && state.testMask != 0) {
+			// The only value != 255 is 0, in our buffer.
+			rewriteFunc(GE_COMP_EQUAL, 0);
+		} else {
+			// Every other value evaluates as not equal, always.
+			state.testFunc = GE_COMP_ALWAYS;
+			rewriteRef();
+		}
+		break;
+	case GE_COMP_LESS: // maskedRef < maskedBuffer
+		if (maskedRef == (0xFF & state.testMask) && state.testMask != 0) {
+			// No possible value is less than 255.
+			state.testFunc = GE_COMP_NEVER;
+			rewriteRef();
+		} else {
+			// "0 < (0 or 255)" and "254 < (0 or 255)" can only work for non zero.
+			rewriteFunc(GE_COMP_NOTEQUAL, 0);
+		}
+		break;
+	case GE_COMP_LEQUAL: // maskedRef <= maskedBuffer
+		if (maskedRef == 0) {
+			// 0 is <= every possible value.
+			state.testFunc = GE_COMP_ALWAYS;
+			rewriteRef();
+		} else {
+			// "1 <= (0 or 255)" and "255 <= (0 or 255)" simply mean, anything but zero.
+			rewriteFunc(GE_COMP_NOTEQUAL, 0);
+		}
+		break;
+	case GE_COMP_GREATER: // maskedRef > maskedBuffer
+		if (maskedRef > 0) {
+			// "1 > (0 or 255)" and "255 > (0 or 255)" can only match 0.
+			rewriteFunc(GE_COMP_EQUAL, 0);
+		} else {
+			// 0 is never greater than any possible value.
+			state.testFunc = GE_COMP_NEVER;
+			rewriteRef();
+		}
+		break;
+	case GE_COMP_GEQUAL: // maskedRef >= maskedBuffer
+		if (maskedRef == (0xFF & state.testMask) && state.testMask != 0) {
+			// 255 is >= every possible value.
+			state.testFunc = GE_COMP_ALWAYS;
+			rewriteRef();
+		} else {
+			// "0 >= (0 or 255)" and "254 >= "(0 or 255)" are the same, equal to zero.
+			rewriteFunc(GE_COMP_EQUAL, 0);
+		}
+		break;
+	}
+}
+
+void ConvertStencilFuncState(GenericStencilFuncState &state) {
+	state.enabled = gstate.isStencilTestEnabled() && !g_Config.bDisableStencilTest;
+	if (!state.enabled)
+		return;
+
+	// The PSP's mask is reversed (bits not to write.)
+	state.writeMask = (~(gstate.pmska >> 0)) & 0xFF;
+
+	state.sFail = gstate.getStencilOpSFail();
+	state.zFail = gstate.getStencilOpZFail();
+	state.zPass = gstate.getStencilOpZPass();
+
+	state.testFunc = gstate.getStencilTestFunction();
+	state.testRef = gstate.getStencilTestRef();
+	state.testMask = gstate.getStencilTestMask();
+
+	switch (gstate.FrameBufFormat()) {
+	case GE_FORMAT_565:
+		state.writeMask = 0;
+		break;
+
+	case GE_FORMAT_5551:
+		ConvertStencilFunc5551(state);
+		break;
+
+	default:
+		// Hard to do anything useful for 4444, and 8888 is fine.
+		break;
+	}
+}

GPU/Common/GPUStateUtils.h

@@ -137,3 +137,16 @@ struct GenericBlendState {
 
 void ConvertBlendState(GenericBlendState &blendState, bool allowShaderBlend);
 void ApplyStencilReplaceAndLogicOp(ReplaceAlphaType replaceAlphaWithStencil, GenericBlendState &blendState);
+
+struct GenericStencilFuncState {
+	bool enabled;
+	GEComparison testFunc;
+	u8 testRef;
+	u8 testMask;
+	u8 writeMask;
+	GEStencilOp sFail;
+	GEStencilOp zFail;
+	GEStencilOp zPass;
+};
+
+void ConvertStencilFuncState(GenericStencilFuncState &stencilFuncState);

GPU/Directx9/StateMappingDX9.cpp

@@ -243,8 +243,8 @@ void TransformDrawEngineDX9::ApplyDrawState(int prim) {
 		bool bmask = ((gstate.pmskc >> 16) & 0xFF) < 128;
 		bool amask = (gstate.pmska & 0xFF) < 128;
 
-		u8 abits = (gstate.pmska >> 0) & 0xFF;
 #ifndef MOBILE_DEVICE
+		u8 abits = (gstate.pmska >> 0) & 0xFF;
 		u8 rbits = (gstate.pmskc >> 0) & 0xFF;
 		u8 gbits = (gstate.pmskc >> 8) & 0xFF;
 		u8 bbits = (gstate.pmskc >> 16) & 0xFF;
@@ -268,21 +268,16 @@ void TransformDrawEngineDX9::ApplyDrawState(int prim) {
 		}
 
 		dxstate.colorMask.set(rmask, gmask, bmask, amask);
-		
+
+		GenericStencilFuncState stencilState;
+		ConvertStencilFuncState(stencilState);
+
 		// Stencil Test
-		if (gstate.isStencilTestEnabled()) {
+		if (stencilState.enabled) {
 			dxstate.stencilTest.enable();
-			dxstate.stencilFunc.set(ztests[gstate.getStencilTestFunction()],
-				gstate.getStencilTestRef(),
-				gstate.getStencilTestMask());
-			dxstate.stencilOp.set(stencilOps[gstate.getStencilOpSFail()],  // stencil fail
-				stencilOps[gstate.getStencilOpZFail()],  // depth fail
-				stencilOps[gstate.getStencilOpZPass()]); // depth pass
-			if (gstate.FrameBufFormat() == GE_FORMAT_5551) {
-				dxstate.stencilMask.set(abits <= 0x7f ? 0xff : 0x00);
-			} else {
-				dxstate.stencilMask.set(~abits);
-			}
+			dxstate.stencilFunc.set(ztests[stencilState.testFunc], stencilState.testRef, stencilState.testMask);
+			dxstate.stencilOp.set(stencilOps[stencilState.sFail], stencilOps[stencilState.zFail], stencilOps[stencilState.zPass]);
+			dxstate.stencilMask.set(stencilState.writeMask);
 		} else {
 			dxstate.stencilTest.disable();
 		}

GPU/GLES/StateMapping.cpp

@@ -312,8 +312,8 @@ void TransformDrawEngine::ApplyDrawState(int prim) {
 		bool bmask = ((gstate.pmskc >> 16) & 0xFF) < 128;
 		bool amask = (gstate.pmska & 0xFF) < 128;
 
-		u8 abits = (gstate.pmska >> 0) & 0xFF;
 #ifndef MOBILE_DEVICE
+		u8 abits = (gstate.pmska >> 0) & 0xFF;
 		u8 rbits = (gstate.pmskc >> 0) & 0xFF;
 		u8 gbits = (gstate.pmskc >> 8) & 0xFF;
 		u8 bbits = (gstate.pmskc >> 16) & 0xFF;
@@ -338,21 +338,15 @@ void TransformDrawEngine::ApplyDrawState(int prim) {
 
 		glstate.colorMask.set(rmask, gmask, bmask, amask);
 
-		// Stencil Test
-		if (gstate.isStencilTestEnabled() && enableStencilTest) {
-			glstate.stencilTest.enable();
-			glstate.stencilFunc.set(ztests[gstate.getStencilTestFunction()],
-				gstate.getStencilTestRef(),
-				gstate.getStencilTestMask());
-			glstate.stencilOp.set(stencilOps[gstate.getStencilOpSFail()],  // stencil fail
-				stencilOps[gstate.getStencilOpZFail()],  // depth fail
-				stencilOps[gstate.getStencilOpZPass()]); // depth pass
+		GenericStencilFuncState stencilState;
+		ConvertStencilFuncState(stencilState);
 
-			if (gstate.FrameBufFormat() == GE_FORMAT_5551) {
-				glstate.stencilMask.set(abits <= 0x7f ? 0xff : 0x00);
-			} else {
-				glstate.stencilMask.set(~abits);
-			}
+		// Stencil Test
+		if (stencilState.enabled) {
+			glstate.stencilTest.enable();
+			glstate.stencilFunc.set(ztests[stencilState.testFunc], stencilState.testRef, stencilState.testMask);
+			glstate.stencilOp.set(stencilOps[stencilState.sFail], stencilOps[stencilState.zFail], stencilOps[stencilState.zPass]);
+			glstate.stencilMask.set(stencilState.writeMask);
 		} else {
 			glstate.stencilTest.disable();
 		}