Merge pull request #12304 from hrydgard/vulkan-display-rotation-fixes

Vulkan: Move scissor/viewport rotation into the VulkanRenderManager.

ext/native/base/display.cpp

@@ -1,3 +1,4 @@
+#include <algorithm>
 #include "base/display.h"
 
 int dp_xres;
@@ -17,3 +18,36 @@ float display_hz = 60.0f;
 
 DisplayRotation g_display_rotation;
 Matrix4x4 g_display_rot_matrix;
+
+void RotateRectToDisplay(FRect &rect, float curRTWidth, float curRTHeight) {
+	switch (g_display_rotation) {
+	case DisplayRotation::ROTATE_180:
+		rect.x = curRTWidth - rect.w - rect.x;
+		rect.y = curRTHeight - rect.h - rect.y;
+		break;
+	case DisplayRotation::ROTATE_90: {
+		// Note that curRTWidth_ and curRTHeight_ are "swapped"!
+		float origX = rect.x;
+		float origY = rect.y;
+		float rtw = curRTHeight;
+		float rth = curRTWidth;
+		rect.x = rth - rect.h - origY;
+		rect.y = origX;
+		std::swap(rect.w, rect.h);
+		break;
+	}
+	case DisplayRotation::ROTATE_270: {
+		float origX = rect.x;
+		float origY = rect.y;
+		float rtw = curRTHeight;
+		float rth = curRTWidth;
+		rect.x = origY;
+		rect.y = rtw - rect.w - origX;
+		std::swap(rect.w, rect.h);
+		break;
+	}
+	case DisplayRotation::ROTATE_0:
+	default:
+		break;
+	}
+}

ext/native/base/display.h

@@ -30,3 +30,8 @@ enum class DisplayRotation {
 
 extern DisplayRotation g_display_rotation;
 extern Matrix4x4 g_display_rot_matrix;
+
+struct FRect {
+	float x, y, w, h;
+};
+void RotateRectToDisplay(FRect &rect, float curRTWidth, float curRTHeight);

ext/native/math/math_util.h

@@ -220,5 +220,3 @@ void EnableFZ();
 // very slow VFP units).
 // http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0274h/Babffifj.html
 void FPU_SetFastMode();
-
-

ext/native/thin3d/VulkanRenderManager.h

@@ -10,6 +10,7 @@
 #include <mutex>
 #include <thread>
 
+#include "base/display.h"
 #include "Common/Vulkan/VulkanContext.h"
 #include "math/dataconv.h"
 #include "math/math_util.h"
@@ -119,13 +120,18 @@ public:
 	void SetViewport(const VkViewport &vp) {
 		_dbg_assert_(G3D, curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
 		_dbg_assert_(G3D, (int)vp.width >= 0);
+		FRect rc{ vp.x, vp.y, vp.width, vp.height };
+		if (curRenderStep_->render.framebuffer == nullptr) { // Only the backbuffer is actually rotated wrong!
+			RotateRectToDisplay(rc, (float)vulkan_->GetBackbufferWidth(), (float)vulkan_->GetBackbufferHeight());
+		}
 		VkRenderData data{ VKRRenderCommand::VIEWPORT };
-		data.viewport.vp.x = vp.x;
-		data.viewport.vp.y = vp.y;
-		data.viewport.vp.width = vp.width;
-		data.viewport.vp.height = vp.height;
+		data.viewport.vp.x = rc.x;
+		data.viewport.vp.y = rc.y;
+		data.viewport.vp.width = rc.w;
+		data.viewport.vp.height = rc.h;
 		// We can't allow values outside this range unless we use VK_EXT_depth_range_unrestricted.
 		// Sometimes state mapping produces 65536/65535 which is slightly outside.
+		// TODO: This should be fixed at the source.
 		data.viewport.vp.maxDepth = clamp_value(vp.maxDepth, 0.0f, 1.0f);
 		data.viewport.vp.minDepth = clamp_value(vp.minDepth, 0.0f, 1.0f);
 		curRenderStep_->commands.push_back(data);
@@ -136,7 +142,15 @@ public:
 		_dbg_assert_(G3D, (int)rc.extent.width >= 0);
 		_dbg_assert_(G3D, (int)rc.extent.height >= 0);
 		VkRenderData data{ VKRRenderCommand::SCISSOR };
-		data.scissor.scissor = rc;
+		if (curRenderStep_->render.framebuffer == nullptr) {
+			FRect frc{ (float)rc.offset.x, (float)rc.offset.y, (float)rc.extent.width, (float)rc.extent.height };
+			if (curRenderStep_->render.framebuffer == nullptr) { // Only the backbuffer is actually rotated wrong!
+				RotateRectToDisplay(frc, (float)vulkan_->GetBackbufferWidth(), (float)vulkan_->GetBackbufferHeight());
+			}
+			data.scissor.scissor = VkRect2D{ { (int32_t)frc.x, (int32_t)frc.y }, { (uint32_t)frc.w, (uint32_t)frc.h} };
+		} else {
+			data.scissor.scissor = rc;
+		}
 		curRenderStep_->commands.push_back(data);
 	}
 

ext/native/thin3d/thin3d.cpp

@@ -353,38 +353,6 @@ DrawContext::~DrawContext() {
 	DestroyPresets();
 }
 
-void DrawContext::RotateRectToDisplay(FRect &rect, float curRTWidth, float curRTHeight) {
-	if (g_display_rotation == DisplayRotation::ROTATE_0)
-		return;
-	switch (g_display_rotation) {
-	case DisplayRotation::ROTATE_180:
-		rect.x = curRTWidth - rect.w - rect.x;
-		rect.y = curRTHeight - rect.h - rect.y;
-		break;
-	case DisplayRotation::ROTATE_90: {
-		// Note that curRTWidth_ and curRTHeight_ are "swapped"!
-		float origX = rect.x;
-		float origY = rect.y;
-		float rtw = curRTHeight;
-		float rth = curRTWidth;
-		rect.x = rth - rect.h - origY;
-		rect.y = origX;
-		std::swap(rect.w, rect.h);
-		break;
-	}
-	case DisplayRotation::ROTATE_270: {
-		float origX = rect.x;
-		float origY = rect.y;
-		float rtw = curRTHeight;
-		float rth = curRTWidth;
-		rect.x = origY;
-		rect.y = rtw - rect.w - origX;
-		std::swap(rect.w, rect.h);
-		break;
-	}
-	}
-}
-
 // TODO: SSE/NEON
 // Could also make C fake-simd for 64-bit, two 8888 pixels fit in a register :)
 void ConvertFromRGBA8888(uint8_t *dst, const uint8_t *src, uint32_t dstStride, uint32_t srcStride, uint32_t width, uint32_t height, DataFormat format) {

ext/native/thin3d/thin3d.h

@@ -661,11 +661,6 @@ public:
 	virtual void FlushState() {}
 
 protected:
-	struct FRect {
-		float x, y, w, h;
-	};
-	void RotateRectToDisplay(FRect &rect, float curRTWidth, float curRTHeight);
-
 	ShaderModule *vsPresets_[VS_MAX_PRESET];
 	ShaderModule *fsPresets_[FS_MAX_PRESET];
 

ext/native/thin3d/thin3d_vulkan.cpp

@@ -1059,13 +1059,7 @@ Pipeline *VKContext::CreateGraphicsPipeline(const PipelineDesc &desc) {
 }
 
 void VKContext::SetScissorRect(int left, int top, int width, int height) {
-	FRect rc{ (float)left, (float)top, (float)width, (float)height };
-	if (curFramebuffer_ == nullptr) { // Only the backbuffer is actually rotated wrong!
-		int curRTWidth, curRTHeight;
-		GetFramebufferDimensions((Framebuffer *)curFramebuffer_, &curRTWidth, &curRTHeight);
-		RotateRectToDisplay(rc, (float)curRTWidth, (float)curRTHeight);
-	}
-	VkRect2D scissor{ {(int32_t)rc.x, (int32_t)rc.y}, {(uint32_t)rc.w, (uint32_t)rc.h} };
+	VkRect2D scissor{ {(int32_t)left, (int32_t)top}, {(uint32_t)width, (uint32_t)height} };
 	renderManager_.SetScissor(scissor);
 }
 
@@ -1074,11 +1068,6 @@ void VKContext::SetViewports(int count, Viewport *viewports) {
 		// Ignore viewports more than the first.
 		VkViewport viewport;
 		FRect rc{ viewports[0].TopLeftX , viewports[0].TopLeftY, viewports[0].Width, viewports[0].Height };
-		if (curFramebuffer_ == nullptr) { // Only the backbuffer is actually rotated wrong!
-			int curRTWidth, curRTHeight;
-			GetFramebufferDimensions((Framebuffer *)curFramebuffer_, &curRTWidth, &curRTHeight);
-			RotateRectToDisplay(rc, (float)curRTWidth, (float)curRTHeight);
-		}
 		viewport.x = rc.x;
 		viewport.y = rc.y;
 		viewport.width = rc.w;