Merge pull request #12878 from unknownbrackets/gpu-bof-cleanup

GPU: Split up software transform into phases

GPU/Common/SoftwareTransformCommon.cpp

@@ -156,16 +156,9 @@ static int ColorIndexOffset(int prim, GEShadeMode shadeMode, bool clearMode) {
 	return 0;
 }
 
-// NOTE: The viewport must be up to date!
-// Also, this assumes SetTexture() has already figured out the actual texture height.
-void SoftwareTransform(
-	int prim, int vertexCount, u32 vertType, u16 *&inds, int indexType,
-	const DecVtxFormat &decVtxFormat, int &maxIndex, TransformedVertex *&drawBuffer, int &numTrans, bool &drawIndexed, const SoftwareTransformParams *params, SoftwareTransformResult *result) {
-	u8 *decoded = params->decoded;
-	FramebufferManagerCommon *fbman = params->fbman;
-	TextureCacheCommon *texCache = params->texCache;
-	TransformedVertex *transformed = params->transformed;
-	TransformedVertex *transformedExpanded = params->transformedExpanded;
+void SoftwareTransform::Decode(int prim, u32 vertType, const DecVtxFormat &decVtxFormat, int maxIndex, SoftwareTransformResult *result) {
+	u8 *decoded = params_.decoded;
+	TransformedVertex *transformed = params_.transformed;
 	bool throughmode = (vertType & GE_VTYPE_THROUGH_MASK) != 0;
 	bool lmode = gstate.isUsingSecondaryColor() && gstate.isLightingEnabled();
 
@@ -196,7 +189,7 @@ void SoftwareTransform(
 	}
 
 	int provokeIndOffset = 0;
-	if (params->provokeFlatFirst) {
+	if (params_.provokeFlatFirst) {
 		provokeIndOffset = ColorIndexOffset(prim, gstate.getShadeMode(), gstate.isModeClear());
 	}
 
@@ -331,12 +324,8 @@ void SoftwareTransform(
 					}
 				}
 			} else {
-				if (reader.hasColor0()) {
-					for (int j = 0; j < 4; j++) {
-						c0[j] = unlitColor[j];
-					}
-				} else {
-					c0 = Vec4f::FromRGBA(gstate.getMaterialAmbientRGBA());
+				for (int j = 0; j < 4; j++) {
+					c0[j] = unlitColor[j];
 				}
 				if (lmode) {
 					// c1 is already 0.
@@ -452,7 +441,7 @@ void SoftwareTransform(
 	// TODO: This bleeds outside the play area in non-buffered mode. Big deal? Probably not.
 	// TODO: Allow creating a depth clear and a color draw.
 	bool reallyAClear = false;
-	if (maxIndex > 1 && prim == GE_PRIM_RECTANGLES && gstate.isModeClear() && params->allowClear) {
+	if (maxIndex > 1 && prim == GE_PRIM_RECTANGLES && gstate.isModeClear() && params_.allowClear) {
 		int scissorX2 = gstate.getScissorX2() + 1;
 		int scissorY2 = gstate.getScissorY2() + 1;
 		reallyAClear = IsReallyAClear(transformed, maxIndex, scissorX2, scissorY2);
@@ -461,7 +450,7 @@ void SoftwareTransform(
 		// If alpha is not allowed to be separate, it must match for both depth/stencil and color.  Vulkan requires this.
 		bool alphaMatchesColor = gstate.isClearModeColorMask() == gstate.isClearModeAlphaMask();
 		bool depthMatchesStencil = gstate.isClearModeAlphaMask() == gstate.isClearModeDepthMask();
-		bool matchingComponents = params->allowSeparateAlphaClear || (alphaMatchesColor && depthMatchesStencil);
+		bool matchingComponents = params_.allowSeparateAlphaClear || (alphaMatchesColor && depthMatchesStencil);
 		bool stencilNotMasked = !gstate.isClearModeAlphaMask() || gstate.getStencilWriteMask() == 0x00;
 		if (matchingComponents && stencilNotMasked) {
 			result->color = transformed[1].color0_32;
@@ -472,6 +461,16 @@ void SoftwareTransform(
 			return;
 		}
 	}
+}
+
+// Also, this assumes SetTexture() has already figured out the actual texture height.
+void SoftwareTransform::DetectOffsetTexture(int maxIndex) {
+	TransformedVertex *transformed = params_.transformed;
+
+	const int w = gstate.getTextureWidth(0);
+	const int h = gstate.getTextureHeight(0);
+	float widthFactor = (float)w / (float)gstate_c.curTextureWidth;
+	float heightFactor = (float)h / (float)gstate_c.curTextureHeight;
 
 	// Breath of Fire 3 does some interesting rendering here, probably from being a port.
 	// It draws at 384x240 to two buffers in VRAM, one right after the other.
@@ -489,7 +488,7 @@ void SoftwareTransform(
 		// This is the max V that would've been inside the original texture size.
 		const float maxValidV = heightFactor;
 
-		// Apaprently, Assassin's Creed: Bloodlines accesses just outside.
+		// Apparently, Assassin's Creed: Bloodlines accesses just outside.
 		const float invTexH = 1.0f / gstate_c.curTextureHeight; // size of one texel.
 
 		// Are either TL or BR inside the texture but outside the framebuffer?
@@ -497,20 +496,22 @@ void SoftwareTransform(
 		const bool brOutside = transformed[1].v > maxAvailableV + invTexH && transformed[1].v <= maxValidV;
 
 		// If TL isn't outside, is it at least near the end?
+		// We check this because some games do 0-512 from a 272 tall framebuf.
 		const bool tlAlmostOutside = transformed[0].v > maxAvailableV * 0.5f && transformed[0].v <= maxValidV;
 
 		if (tlOutside || (brOutside && tlAlmostOutside)) {
-			// This is how far the nearest coord is, so that's where we'll look for the next framebuf.
 			const u32 prevXOffset = gstate_c.curTextureXOffset;
 			const u32 prevYOffset = gstate_c.curTextureYOffset;
+
+			// This is how far the nearest coord is, so that's where we'll look for the next framebuf.
 			const u32 yOffset = (int)(gstate_c.curTextureHeight * std::min(transformed[0].v, transformed[1].v));
-			if (texCache->SetOffsetTexture(yOffset)) {
+			if (params_.texCache->SetOffsetTexture(yOffset)) {
 				const float oldWidthFactor = widthFactor;
 				const float oldHeightFactor = heightFactor;
 				widthFactor = (float)w / (float)gstate_c.curTextureWidth;
 				heightFactor = (float)h / (float)gstate_c.curTextureHeight;
 
-				// We need to subtract this offset from the Vs to address the new framebuf.
+				// We need to subtract this offset from the UVs to address the new framebuf.
 				const float adjustedYOffset = yOffset + prevYOffset - gstate_c.curTextureYOffset;
 				const float yDiff = (float)adjustedYOffset / (float)h;
 				const float adjustedXOffset = prevXOffset - gstate_c.curTextureXOffset;
@@ -524,16 +525,22 @@ void SoftwareTransform(
 				// We undid the offset, so reset.  This avoids a different shader.
 				gstate_c.curTextureXOffset = prevXOffset;
 				gstate_c.curTextureYOffset = prevYOffset;
-				result->textureChanged = true;
 			}
 		}
 	}
+}
+
+// NOTE: The viewport must be up to date!
+void SoftwareTransform::BuildDrawingParams(int prim, int vertexCount, u32 vertType, u16 *&inds, int &maxIndex, SoftwareTransformResult *result) {
+	TransformedVertex *transformed = params_.transformed;
+	TransformedVertex *transformedExpanded = params_.transformedExpanded;
+	bool throughmode = (vertType & GE_VTYPE_THROUGH_MASK) != 0;
 
 	// Step 2: expand rectangles.
-	drawBuffer = transformed;
-	numTrans = 0;
-	drawIndexed = false;
+	result->drawBuffer = transformed;
+	int numTrans = 0;
 
+	FramebufferManagerCommon *fbman = params_.fbman;
 	bool useBufferedRendering = fbman->UseBufferedRendering();
 
 	bool flippedY = g_Config.iGPUBackend == (int)GPUBackend::OPENGL && !useBufferedRendering;
@@ -541,7 +548,7 @@ void SoftwareTransform(
 	if (prim != GE_PRIM_RECTANGLES) {
 		// We can simply draw the unexpanded buffer.
 		numTrans = vertexCount;
-		drawIndexed = true;
+		result->drawIndexed = true;
 	} else {
 		// Pretty bad hackery where we re-do the transform (in RotateUV) to see if the vertices are flipped in screen space.
 		// Since we've already got API-specific assumptions (Y direction, etc) baked into the projMatrix (which we arguably shouldn't),
@@ -570,7 +577,7 @@ void SoftwareTransform(
 		//rectangles always need 2 vertices, disregard the last one if there's an odd number
 		vertexCount = vertexCount & ~1;
 		numTrans = 0;
-		drawBuffer = transformedExpanded;
+		result->drawBuffer = transformedExpanded;
 		TransformedVertex *trans = &transformedExpanded[0];
 		const u16 *indsIn = (const u16 *)inds;
 		u16 *newInds = inds + vertexCount;
@@ -623,7 +630,7 @@ void SoftwareTransform(
 			numTrans += 6;
 		}
 		inds = newInds;
-		drawIndexed = true;
+		result->drawIndexed = true;
 
 		// We don't know the color until here, so we have to do it now, instead of in StateMapping.
 		// Might want to reconsider the order of things later...
@@ -645,4 +652,5 @@ void SoftwareTransform(
 	}
 
 	result->action = SW_DRAW_PRIMITIVES;
+	result->drawNumTrans = numTrans;
 }

GPU/Common/SoftwareTransformCommon.h

@@ -25,6 +25,7 @@ class FramebufferManagerCommon;
 class TextureCacheCommon;
 
 enum SoftwareTransformAction {
+	SW_NOT_READY,
 	SW_DRAW_PRIMITIVES,
 	SW_CLEAR,
 };
@@ -35,8 +36,11 @@ struct SoftwareTransformResult {
 	float depth;
 
 	bool setStencil;
-	bool textureChanged;
 	u8 stencilValue;
+
+	TransformedVertex *drawBuffer;
+	int drawNumTrans;
+	bool drawIndexed;
 };
 
 struct SoftwareTransformParams {
@@ -50,5 +54,15 @@ struct SoftwareTransformParams {
 	bool provokeFlatFirst;
 };
 
-void SoftwareTransform(int prim, int vertexCount, u32 vertexType, u16 *&inds, int indexType, const DecVtxFormat &decVtxFormat, int &maxIndex, TransformedVertex *&drawBuffer,
-	int &numTrans, bool &drawIndexed, const SoftwareTransformParams *params, SoftwareTransformResult *result);
+class SoftwareTransform {
+public:
+	SoftwareTransform(SoftwareTransformParams &params) : params_(params) {
+	}
+
+	void Decode(int prim, u32 vertexType, const DecVtxFormat &decVtxFormat, int maxIndex, SoftwareTransformResult *result);
+	void DetectOffsetTexture(int maxIndex);
+	void BuildDrawingParams(int prim, int vertexCount, u32 vertType, u16 *&inds, int &maxIndex, SoftwareTransformResult *result);
+
+protected:
+	const SoftwareTransformParams &params_;
+};

GPU/D3D11/DrawEngineD3D11.cpp

@@ -570,10 +570,7 @@ rotateVBO:
 			prim = GE_PRIM_TRIANGLES;
 		VERBOSE_LOG(G3D, "Flush prim %i SW! %i verts in one go", prim, indexGen.VertexCount());
 
-		int numTrans = 0;
-		bool drawIndexed = false;
 		u16 *inds = decIndex;
-		TransformedVertex *drawBuffer = NULL;
 		SoftwareTransformResult result{};
 		SoftwareTransformParams params{};
 		params.decoded = decoded;
@@ -586,10 +583,12 @@ rotateVBO:
 		params.provokeFlatFirst = true;
 
 		int maxIndex = indexGen.MaxIndex();
-		SoftwareTransform(
-			prim, indexGen.VertexCount(),
-			dec_->VertexType(), inds, GE_VTYPE_IDX_16BIT, dec_->GetDecVtxFmt(),
-			maxIndex, drawBuffer, numTrans, drawIndexed, &params, &result);
+		SoftwareTransform swTransform(params);
+		swTransform.Decode(prim, dec_->VertexType(), dec_->GetDecVtxFmt(), maxIndex, &result);
+		if (result.action == SW_NOT_READY) {
+			swTransform.DetectOffsetTexture(maxIndex);
+			swTransform.BuildDrawingParams(prim, indexGen.VertexCount(), dec_->VertexType(), inds, maxIndex, &result);
+		}
 
 		if (result.action == SW_DRAW_PRIMITIVES) {
 			const int vertexSize = sizeof(transformed[0]);
@@ -619,20 +618,20 @@ rotateVBO:
 			UINT vOffset = 0;
 			int vSize = maxIndex * stride;
 			uint8_t *vptr = pushVerts_->BeginPush(context_, &vOffset, vSize);
-			memcpy(vptr, drawBuffer, vSize);
+			memcpy(vptr, result.drawBuffer, vSize);
 			pushVerts_->EndPush(context_);
 			ID3D11Buffer *buf = pushVerts_->Buf();
 			context_->IASetVertexBuffers(0, 1, &buf, &stride, &vOffset);
-			if (drawIndexed) {
+			if (result.drawIndexed) {
 				UINT iOffset;
-				int iSize = sizeof(uint16_t) * numTrans;
+				int iSize = sizeof(uint16_t) * result.drawNumTrans;
 				uint8_t *iptr = pushInds_->BeginPush(context_, &iOffset, iSize);
 				memcpy(iptr, inds, iSize);
 				pushInds_->EndPush(context_);
 				context_->IASetIndexBuffer(pushInds_->Buf(), DXGI_FORMAT_R16_UINT, iOffset);
-				context_->DrawIndexed(numTrans, 0, 0);
+				context_->DrawIndexed(result.drawNumTrans, 0, 0);
 			} else {
-				context_->Draw(numTrans, 0);
+				context_->Draw(result.drawNumTrans, 0);
 			}
 		} else if (result.action == SW_CLEAR) {
 			u32 clearColor = result.color;

GPU/Directx9/DrawEngineDX9.cpp

@@ -530,10 +530,7 @@ rotateVBO:
 			prim = GE_PRIM_TRIANGLES;
 		VERBOSE_LOG(G3D, "Flush prim %i SW! %i verts in one go", prim, indexGen.VertexCount());
 
-		int numTrans = 0;
-		bool drawIndexed = false;
 		u16 *inds = decIndex;
-		TransformedVertex *drawBuffer = NULL;
 		SoftwareTransformResult result{};
 		SoftwareTransformParams params{};
 		params.decoded = decoded;
@@ -546,10 +543,12 @@ rotateVBO:
 		params.provokeFlatFirst = true;
 
 		int maxIndex = indexGen.MaxIndex();
-		SoftwareTransform(
-			prim, indexGen.VertexCount(),
-			dec_->VertexType(), inds, GE_VTYPE_IDX_16BIT, dec_->GetDecVtxFmt(),
-			maxIndex, drawBuffer, numTrans, drawIndexed, &params, &result);
+		SoftwareTransform swTransform(params);
+		swTransform.Decode(prim, dec_->VertexType(), dec_->GetDecVtxFmt(), maxIndex, &result);
+		if (result.action == SW_NOT_READY) {
+			swTransform.DetectOffsetTexture(maxIndex);
+			swTransform.BuildDrawingParams(prim, indexGen.VertexCount(), dec_->VertexType(), inds, maxIndex, &result);
+		}
 
 		ApplyDrawStateLate();
 		vshader = shaderManager_->ApplyShader(false, false, lastVType_);
@@ -566,10 +565,10 @@ rotateVBO:
 			const int vertexSize = sizeof(transformed[0]);
 
 			device_->SetVertexDeclaration(transformedVertexDecl_);
-			if (drawIndexed) {
-				device_->DrawIndexedPrimitiveUP(d3d_prim[prim], 0, maxIndex, D3DPrimCount(d3d_prim[prim], numTrans), inds, D3DFMT_INDEX16, drawBuffer, sizeof(TransformedVertex));
+			if (result.drawIndexed) {
+				device_->DrawIndexedPrimitiveUP(d3d_prim[prim], 0, maxIndex, D3DPrimCount(d3d_prim[prim], result.drawNumTrans), inds, D3DFMT_INDEX16, result.drawBuffer, sizeof(TransformedVertex));
 			} else {
-				device_->DrawPrimitiveUP(d3d_prim[prim], D3DPrimCount(d3d_prim[prim], numTrans), drawBuffer, sizeof(TransformedVertex));
+				device_->DrawPrimitiveUP(d3d_prim[prim], D3DPrimCount(d3d_prim[prim], result.drawNumTrans), result.drawBuffer, sizeof(TransformedVertex));
 			}
 		} else if (result.action == SW_CLEAR) {
 			u32 clearColor = result.color;

GPU/GLES/DrawEngineGLES.cpp

@@ -538,9 +538,6 @@ rotateVBO:
 		if (prim == GE_PRIM_TRIANGLE_STRIP)
 			prim = GE_PRIM_TRIANGLES;
 
-		TransformedVertex *drawBuffer = NULL;
-		int numTrans;
-		bool drawIndexed = false;
 		u16 *inds = decIndex;
 		SoftwareTransformResult result{};
 		// TODO: Keep this static?  Faster than repopulating?
@@ -563,23 +560,19 @@ rotateVBO:
 			vertexCount = 0x10000 / 3;
 #endif
 
+		SoftwareTransform swTransform(params);
+		swTransform.Decode(prim, dec_->VertexType(), dec_->GetDecVtxFmt(), maxIndex, &result);
+		if (result.action == SW_NOT_READY)
+			swTransform.DetectOffsetTexture(maxIndex);
+
 		if (textureNeedsApply)
 			textureCache_->ApplyTexture();
 
 		// Need to ApplyDrawState after ApplyTexture because depal can launch a render pass and that wrecks the state.
 		ApplyDrawState(prim);
 
-		SoftwareTransform(
-			prim, vertexCount,
-			dec_->VertexType(), inds, GE_VTYPE_IDX_16BIT, dec_->GetDecVtxFmt(),
-			maxIndex, drawBuffer, numTrans, drawIndexed, &params, &result);
-
-		// We have an offset texture to apply.
-		if (result.textureChanged) {
-			textureCache_->ApplyTexture();
-			// Apply again in case of depal.
-			ApplyDrawState(prim);
-		}
+		if (result.action == SW_NOT_READY)
+			swTransform.BuildDrawingParams(prim, vertexCount, dec_->VertexType(), inds, maxIndex, &result);
 
 		ApplyDrawStateLate(result.setStencil, result.stencilValue);
 
@@ -590,16 +583,16 @@ rotateVBO:
 
 			bool doTextureProjection = gstate.getUVGenMode() == GE_TEXMAP_TEXTURE_MATRIX;
 
-			if (drawIndexed) {
-				vertexBufferOffset = (uint32_t)frameData.pushVertex->Push(drawBuffer, maxIndex * sizeof(TransformedVertex), &vertexBuffer);
-				indexBufferOffset = (uint32_t)frameData.pushIndex->Push(inds, sizeof(uint16_t) * numTrans, &indexBuffer);
+			if (result.drawIndexed) {
+				vertexBufferOffset = (uint32_t)frameData.pushVertex->Push(result.drawBuffer, maxIndex * sizeof(TransformedVertex), &vertexBuffer);
+				indexBufferOffset = (uint32_t)frameData.pushIndex->Push(inds, sizeof(uint16_t) * result.drawNumTrans, &indexBuffer);
 				render_->BindVertexBuffer(softwareInputLayout_, vertexBuffer, vertexBufferOffset);
 				render_->BindIndexBuffer(indexBuffer);
-				render_->DrawIndexed(glprim[prim], numTrans, GL_UNSIGNED_SHORT, (void *)(intptr_t)indexBufferOffset);
+				render_->DrawIndexed(glprim[prim], result.drawNumTrans, GL_UNSIGNED_SHORT, (void *)(intptr_t)indexBufferOffset);
 			} else {
-				vertexBufferOffset = (uint32_t)frameData.pushVertex->Push(drawBuffer, numTrans * sizeof(TransformedVertex), &vertexBuffer);
+				vertexBufferOffset = (uint32_t)frameData.pushVertex->Push(result.drawBuffer, result.drawNumTrans * sizeof(TransformedVertex), &vertexBuffer);
 				render_->BindVertexBuffer(softwareInputLayout_, vertexBuffer, vertexBufferOffset);
-				render_->Draw(glprim[prim], 0, numTrans);
+				render_->Draw(glprim[prim], 0, result.drawNumTrans);
 			}
 		} else if (result.action == SW_CLEAR) {
 			u32 clearColor = result.color;

GPU/Vulkan/DrawEngineVulkan.cpp

@@ -880,10 +880,7 @@ void DrawEngineVulkan::DoFlush() {
 			prim = GE_PRIM_TRIANGLES;
 		VERBOSE_LOG(G3D, "Flush prim %i SW! %i verts in one go", prim, indexGen.VertexCount());
 
-		int numTrans = 0;
-		bool drawIndexed = false;
 		u16 *inds = decIndex;
-		TransformedVertex *drawBuffer = nullptr;
 		SoftwareTransformResult result{};
 		SoftwareTransformParams params{};
 		params.decoded = decoded;
@@ -906,10 +903,12 @@ void DrawEngineVulkan::DoFlush() {
 		}
 
 		int maxIndex = indexGen.MaxIndex();
-		SoftwareTransform(
-			prim, indexGen.VertexCount(),
-			dec_->VertexType(), inds, GE_VTYPE_IDX_16BIT, dec_->GetDecVtxFmt(),
-			maxIndex, drawBuffer, numTrans, drawIndexed, &params, &result);
+		SoftwareTransform swTransform(params);
+		swTransform.Decode(prim, dec_->VertexType(), dec_->GetDecVtxFmt(), maxIndex, &result);
+		if (result.action == SW_NOT_READY) {
+			swTransform.DetectOffsetTexture(maxIndex);
+			swTransform.BuildDrawingParams(prim, indexGen.VertexCount(), dec_->VertexType(), inds, maxIndex, &result);
+		}
 
 		// Only here, where we know whether to clear or to draw primitives, should we actually set the current framebuffer! Because that gives use the opportunity
 		// to use a "pre-clear" render pass, for high efficiency on tilers.
@@ -965,17 +964,17 @@ void DrawEngineVulkan::DoFlush() {
 
 			PROFILE_THIS_SCOPE("renderman_q");
 
-			if (drawIndexed) {
+			if (result.drawIndexed) {
 				VkBuffer vbuf, ibuf;
-				vbOffset = (uint32_t)frame->pushVertex->Push(drawBuffer, maxIndex * sizeof(TransformedVertex), &vbuf);
-				ibOffset = (uint32_t)frame->pushIndex->Push(inds, sizeof(short) * numTrans, &ibuf);
+				vbOffset = (uint32_t)frame->pushVertex->Push(result.drawBuffer, maxIndex * sizeof(TransformedVertex), &vbuf);
+				ibOffset = (uint32_t)frame->pushIndex->Push(inds, sizeof(short) * result.drawNumTrans, &ibuf);
 				VkDeviceSize offsets[1] = { vbOffset };
-				renderManager->DrawIndexed(pipelineLayout_, ds, ARRAY_SIZE(dynamicUBOOffsets), dynamicUBOOffsets, vbuf, vbOffset, ibuf, ibOffset, numTrans, 1, VK_INDEX_TYPE_UINT16);
+				renderManager->DrawIndexed(pipelineLayout_, ds, ARRAY_SIZE(dynamicUBOOffsets), dynamicUBOOffsets, vbuf, vbOffset, ibuf, ibOffset, result.drawNumTrans, 1, VK_INDEX_TYPE_UINT16);
 			} else {
 				VkBuffer vbuf;
-				vbOffset = (uint32_t)frame->pushVertex->Push(drawBuffer, numTrans * sizeof(TransformedVertex), &vbuf);
+				vbOffset = (uint32_t)frame->pushVertex->Push(result.drawBuffer, result.drawNumTrans * sizeof(TransformedVertex), &vbuf);
 				VkDeviceSize offsets[1] = { vbOffset };
-				renderManager->Draw(pipelineLayout_, ds, ARRAY_SIZE(dynamicUBOOffsets), dynamicUBOOffsets, vbuf, vbOffset, numTrans);
+				renderManager->Draw(pipelineLayout_, ds, ARRAY_SIZE(dynamicUBOOffsets), dynamicUBOOffsets, vbuf, vbOffset, result.drawNumTrans);
 			}
 		} else if (result.action == SW_CLEAR) {
 			// Note: we won't get here if the clear is alpha but not color, or color but not alpha.