Merge pull request #6852 from unknownbrackets/gpu-minor

Clear stencil/alpha when switching from 565, minor frag shader simplication

GPU/GLES/FragmentShaderGenerator.cpp

@@ -907,47 +907,36 @@ void GenerateFragmentShader(char *buffer) {
 		}
 	}
 
-	switch (stencilToAlpha) {
-	case REPLACE_ALPHA_DUALSOURCE:
-		WRITE(p, "  fragColor0 = vec4(v.rgb, 0.0);\n");
-		WRITE(p, "  fragColor1 = vec4(0.0, 0.0, 0.0, v.a);\n");	
-		break;
-
-	case REPLACE_ALPHA_YES:
-		WRITE(p, "  %s = vec4(v.rgb, 0.0);\n", fragColor0);
-		break;
-
-	case REPLACE_ALPHA_NO:
-		WRITE(p, "  %s = v;\n", fragColor0);
-		break;
-	}
-
+	std::string replacedAlpha = "0.0";
+	char replacedAlphaTemp[64] = "";
 	if (stencilToAlpha != REPLACE_ALPHA_NO) {
 		switch (ReplaceAlphaWithStencilType()) {
 		case STENCIL_VALUE_UNIFORM:
-			WRITE(p, "  %s.a = u_stencilReplaceValue;\n", fragColor0);
+			replacedAlpha = "u_stencilReplaceValue";
 			break;
 
 		case STENCIL_VALUE_ZERO:
-			WRITE(p, "  %s.a = 0.0;\n", fragColor0);
+			replacedAlpha = "0.0";
 			break;
 
 		case STENCIL_VALUE_ONE:
 		case STENCIL_VALUE_INVERT:
 			// In invert, we subtract by one, but we want to output one here.
-			WRITE(p, "  %s.a = 1.0;\n", fragColor0);
+			replacedAlpha = "1.0";
 			break;
 
 		case STENCIL_VALUE_INCR_4:
 		case STENCIL_VALUE_DECR_4:
 			// We're adding/subtracting, just by the smallest value in 4-bit.
-			WRITE(p, "  %s.a = %f;\n", fragColor0, 1.0 / 15.0);
+			snprintf(replacedAlphaTemp, sizeof(replacedAlphaTemp), "%f", 1.0 / 15.0);
+			replacedAlpha = replacedAlphaTemp;
 			break;
 
 		case STENCIL_VALUE_INCR_8:
 		case STENCIL_VALUE_DECR_8:
 			// We're adding/subtracting, just by the smallest value in 8-bit.
-			WRITE(p, "  %s.a = %f;\n", fragColor0, 1.0 / 255.0);
+			snprintf(replacedAlphaTemp, sizeof(replacedAlphaTemp), "%f", 1.0 / 255.0);
+			replacedAlpha = replacedAlphaTemp;
 			break;
 
 		case STENCIL_VALUE_KEEP:
@@ -956,6 +945,21 @@ void GenerateFragmentShader(char *buffer) {
 		}
 	}
 
+	switch (stencilToAlpha) {
+	case REPLACE_ALPHA_DUALSOURCE:
+		WRITE(p, "  fragColor0 = vec4(v.rgb, %s);\n", replacedAlpha.c_str());
+		WRITE(p, "  fragColor1 = vec4(0.0, 0.0, 0.0, v.a);\n");
+		break;
+
+	case REPLACE_ALPHA_YES:
+		WRITE(p, "  %s = vec4(v.rgb, %s);\n", fragColor0, replacedAlpha.c_str());
+		break;
+
+	case REPLACE_ALPHA_NO:
+		WRITE(p, "  %s = v;\n", fragColor0);
+		break;
+	}
+
 #ifdef DEBUG_SHADER
 	if (doTexture) {
 		WRITE(p, "  %s = texture2D(tex, v_texcoord.xy);\n", fragColor0);

GPU/GLES/Framebuffer.cpp

@@ -889,6 +889,7 @@ void FramebufferManager::DoSetRenderFrameBuffer() {
 
 	gstate_c.cutRTOffsetX = 0;
 	bool vfbFormatChanged = false;
+	GEBufferFormat vfbOldFormat = GE_FORMAT_INVALID;
 
 	// Find a matching framebuffer
 	VirtualFramebuffer *vfb = 0;
@@ -899,9 +900,10 @@ void FramebufferManager::DoSetRenderFrameBuffer() {
 			vfb = v;
 			// Update fb stride in case it changed
 			if (vfb->fb_stride != fb_stride || vfb->format != fmt) {
+				vfbOldFormat = vfb->format;
+				vfbFormatChanged = true;
 				vfb->fb_stride = fb_stride;
 				vfb->format = fmt;
-				vfbFormatChanged = true;
 			}
 			// In throughmode, a higher height could be used.  Let's avoid shrinking the buffer.
 			if (gstate.isModeThrough() && (int)vfb->width < fb_stride) {
@@ -1107,9 +1109,16 @@ void FramebufferManager::DoSetRenderFrameBuffer() {
 			BlitFramebufferDepth(currentRenderVfb_, vfb);
 		}
 		currentRenderVfb_ = vfb;
+		if (vfbFormatChanged && vfbOldFormat != vfb->format) {
+			// TODO: Might ultimately combine this with the resize step above.
+			ReformatFramebufferFrom(vfb, vfbOldFormat);
+		}
 	} else {
 		if (vfbFormatChanged) {
 			textureCache_->NotifyFramebuffer(vfb->fb_address, vfb, NOTIFY_FB_UPDATED);
+			if (vfbOldFormat != vfb->format) {
+				ReformatFramebufferFrom(vfb, vfbOldFormat);
+			}
 		}
 
 		vfb->last_frame_render = gpuStats.numFlips;
@@ -1141,6 +1150,36 @@ void FramebufferManager::SetLineWidth() {
 #endif
 }
 
+void FramebufferManager::ReformatFramebufferFrom(VirtualFramebuffer *vfb, GEBufferFormat old) {
+	if (!useBufferedRendering_) {
+		return;
+	}
+
+	fbo_bind_as_render_target(vfb->fbo);
+
+	// Technically, we should at this point re-interpret the bytes of the old format to the new.
+	// That might get tricky, and could cause unnecessary slowness in some games.
+	// For now, we just clear alpha/stencil from 565, which fixes shadow issues in Kingdom Hearts.
+	// (it uses 565 to write zeros to the buffer, than 4444 to actually render the shadow.)
+	//
+	// The best way to do this may ultimately be to create a new FBO (combine with any resize?)
+	// and blit with a shader to that, then replace the FBO on vfb.  Stencil would still be complex
+	// to exactly reproduce in 4444 and 8888 formats.
+
+	if (old == GE_FORMAT_565) {
+		glstate.scissorTest.disable();
+		glstate.depthWrite.set(GL_FALSE);
+		glstate.colorMask.set(false, false, false, true);
+		glstate.stencilFunc.set(GL_ALWAYS, 0, 0);
+		glstate.stencilMask.set(0xFF);
+		glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
+		glClearStencil(0);
+		glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
+	}
+
+	RebindFramebuffer();
+}
+
 void FramebufferManager::BlitFramebufferDepth(VirtualFramebuffer *sourceframebuffer, VirtualFramebuffer *targetframebuffer) {
 	if (!sourceframebuffer->fbo || !targetframebuffer->fbo || !useBufferedRendering_) {
 		return;

GPU/GLES/Framebuffer.h

@@ -162,6 +162,7 @@ public:
 	}
 	void UpdateFromMemory(u32 addr, int size, bool safe);
 	void SetLineWidth();
+	void ReformatFramebufferFrom(VirtualFramebuffer *vfb, GEBufferFormat old);
 
 	void BlitFramebufferDepth(VirtualFramebuffer *sourceframebuffer, VirtualFramebuffer *targetframebuffer);