Merge pull request #12262 from hrydgard/vulkan-gpu-profiling

Vulkan: Implement basic integrated GPU profiling.

Common/Vulkan/VulkanContext.cpp

@@ -516,8 +516,8 @@ void VulkanContext::ChooseDevice(int physical_device) {
 	vkGetPhysicalDeviceQueueFamilyProperties(physical_devices_[physical_device_], &queue_count, nullptr);
 	assert(queue_count >= 1);
 
-	queue_props.resize(queue_count);
-	vkGetPhysicalDeviceQueueFamilyProperties(physical_devices_[physical_device_], &queue_count, queue_props.data());
+	queueFamilyProperties_.resize(queue_count);
+	vkGetPhysicalDeviceQueueFamilyProperties(physical_devices_[physical_device_], &queue_count, queueFamilyProperties_.data());
 	assert(queue_count >= 1);
 
 	// Detect preferred formats, in this order.
@@ -619,7 +619,7 @@ VkResult VulkanContext::CreateDevice() {
 	queue_info.pQueuePriorities = queue_priorities;
 	bool found = false;
 	for (int i = 0; i < (int)queue_count; i++) {
-		if (queue_props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) {
+		if (queueFamilyProperties_[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) {
 			queue_info.queueFamilyIndex = i;
 			found = true;
 			break;
@@ -816,7 +816,7 @@ bool VulkanContext::InitQueue() {
 	uint32_t graphicsQueueNodeIndex = UINT32_MAX;
 	uint32_t presentQueueNodeIndex = UINT32_MAX;
 	for (uint32_t i = 0; i < queue_count; i++) {
-		if ((queue_props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) {
+		if ((queueFamilyProperties_[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) {
 			if (graphicsQueueNodeIndex == UINT32_MAX) {
 				graphicsQueueNodeIndex = i;
 			}

Common/Vulkan/VulkanContext.h

@@ -201,6 +201,10 @@ public:
 		return physicalDeviceProperties_[i];
 	}
 
+	const VkQueueFamilyProperties &GetQueueFamilyProperties(int family) const {
+		return queueFamilyProperties_[family];
+	}
+
 	VkResult GetInstanceLayerExtensionList(const char *layerName, std::vector<VkExtensionProperties> &extensions);
 	VkResult GetInstanceLayerProperties();
 
@@ -303,8 +307,8 @@ private:
 	int physical_device_ = -1;
 
 	uint32_t graphics_queue_family_index_ = -1;
-	std::vector<PhysicalDeviceProps> physicalDeviceProperties_{};
-	std::vector<VkQueueFamilyProperties> queue_props;
+	std::vector<PhysicalDeviceProps> physicalDeviceProperties_;
+	std::vector<VkQueueFamilyProperties> queueFamilyProperties_;
 	VkPhysicalDeviceMemoryProperties memory_properties{};
 
 	// Custom collection of things that are good to know

Common/Vulkan/VulkanDebug.cpp

@@ -113,6 +113,13 @@ VkBool32 VKAPI_CALL VulkanDebugUtilsCallback(
 	std::ostringstream message;
 
 	const char *pMessage = pCallbackData->pMessage;
+
+	// Apparent bugs around timestamp validation in the validation layers
+	if (strstr(pMessage, "vkCmdBeginQuery(): VkQueryPool"))
+		return false;
+	if (strstr(pMessage, "vkGetQueryPoolResults() on VkQueryPool"))
+		return false;
+
 	int messageCode = pCallbackData->messageIdNumber;
 	const char *pLayerPrefix = "";
 	if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) {

Core/Config.cpp

@@ -959,9 +959,10 @@ static ConfigSetting debuggerSettings[] = {
 	ConfigSetting("FontWidth", &g_Config.iFontWidth, 8),
 	ConfigSetting("FontHeight", &g_Config.iFontHeight, 12),
 	ConfigSetting("DisplayStatusBar", &g_Config.bDisplayStatusBar, true),
-	ConfigSetting("ShowBottomTabTitles",&g_Config.bShowBottomTabTitles,true),
+	ConfigSetting("ShowBottomTabTitles",&g_Config.bShowBottomTabTitles, true),
 	ConfigSetting("ShowDeveloperMenu", &g_Config.bShowDeveloperMenu, false),
 	ConfigSetting("ShowAllocatorDebug", &g_Config.bShowAllocatorDebug, false, false),
+	ConfigSetting("ShowGpuProfile", &g_Config.bShowGpuProfile, false, false),
 	ConfigSetting("SkipDeadbeefFilling", &g_Config.bSkipDeadbeefFilling, false),
 	ConfigSetting("FuncHashMap", &g_Config.bFuncHashMap, false),
 

Core/Config.h

@@ -68,21 +68,20 @@ public:
 	bool bEnableLogging;
 	bool bDumpDecryptedEboot;
 	bool bFullscreenOnDoubleclick;
-#if defined(USING_WIN_UI)
+
+	// These four are Win UI only
 	bool bPauseOnLostFocus;
 	bool bTopMost;
 	bool bIgnoreWindowsKey;
 	bool bRestartRequired;
-#endif
-#if defined(USING_WIN_UI) || defined(USING_QT_UI) || PPSSPP_PLATFORM(UWP)
+
 	std::string sFont;
-#endif
 
 	bool bPauseWhenMinimized;
 
-#if !defined(MOBILE_DEVICE)
+	// Not used on mobile devices.
 	bool bPauseExitsEmulator;
-#endif
+
 	bool bPauseMenuExitsEmulator;
 
 	// Core
@@ -129,9 +128,8 @@ public:
 	// We have separate device parameters for each backend so it doesn't get erased if you switch backends.
 	// If not set, will use the "best" device.
 	std::string sVulkanDevice;
-#ifdef _WIN32
-	std::string sD3D11Device;
-#endif
+	std::string sD3D11Device;  // Windows only
+
 	bool bSoftwareRendering;
 	bool bHardwareTransform; // only used in the GLES backend
 	bool bSoftwareSkinning;  // may speed up some games
@@ -237,6 +235,7 @@ public:
 	bool bLogFrameDrops;
 	bool bShowDebugStats;
 	bool bShowAudioDebug;
+	bool bShowGpuProfile;
 	bool bAudioResampler;
 
 	//Analog stick tilting
@@ -366,9 +365,7 @@ public:
 	int iPSPModel;
 	int iFirmwareVersion;
 	// TODO: Make this work with your platform, too!
-#if defined(USING_WIN_UI)
 	bool bBypassOSKWithKeyboard;
-#endif
 
 	// Debugger
 	int iDisasmWindowX;

GPU/Vulkan/DebugVisVulkan.cpp

@@ -26,6 +26,8 @@
 #include "GPU/Vulkan/GPU_Vulkan.h"
 #include "GPU/Vulkan/VulkanUtil.h"
 
+#undef DrawText
+
 void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu) {
 	if (!gpu) {
 		return;
@@ -94,3 +96,26 @@ void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu) {
 	for (auto iter : texturesToDelete)
 		iter->Release();
 }
+
+void DrawProfilerVis(UIContext *ui, GPUInterface *gpu) {
+	if (!gpu) {
+		return;
+	}
+	using namespace Draw;
+	const int padding = 10;
+	const int columnWidth = 256;
+	const int starty = padding * 8;
+	int x = padding;
+	int y = starty;
+	int w = columnWidth;  // We will double this when actually drawing to make the pixels visible.
+
+	ui->Begin();
+
+	GPU_Vulkan *gpuVulkan = static_cast<GPU_Vulkan *>(gpu);
+
+	std::string text = gpuVulkan->GetGpuProfileString();
+
+	Draw::DrawContext *draw = ui->GetDrawContext();
+	ui->DrawTextShadow(text.c_str(), 10, 50, 0xFFFFFFFF, FLAG_DYNAMIC_ASCII);
+	ui->Flush();
+}

GPU/Vulkan/DebugVisVulkan.h

@@ -24,3 +24,4 @@ class UIContext;
 
 // gpu MUST be an instance of GPU_Vulkan. If not, will definitely crash.
 void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu);
+void DrawProfilerVis(UIContext *ui, GPUInterface *gpu);

GPU/Vulkan/GPU_Vulkan.cpp

@@ -638,3 +638,8 @@ std::string GPU_Vulkan::DebugGetShaderString(std::string id, DebugShaderType typ
 		return std::string();
 	}
 }
+
+std::string GPU_Vulkan::GetGpuProfileString() {
+	VulkanRenderManager *rm = (VulkanRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
+	return rm->GetGpuProfileString();
+}

GPU/Vulkan/GPU_Vulkan.h

@@ -66,6 +66,8 @@ public:
 		return textureCacheVulkan_;
 	}
 
+	std::string GetGpuProfileString();
+
 protected:
 	void FinishDeferred() override;
 

UI/DevScreens.cpp

@@ -78,7 +78,10 @@ void DevMenu::CreatePopupContents(UI::ViewGroup *parent) {
 	items->Add(new Choice(sy->T("Developer Tools")))->OnClick.Handle(this, &DevMenu::OnDeveloperTools);
 	items->Add(new Choice(dev->T("Jit Compare")))->OnClick.Handle(this, &DevMenu::OnJitCompare);
 	items->Add(new Choice(dev->T("Shader Viewer")))->OnClick.Handle(this, &DevMenu::OnShaderView);
-	items->Add(new CheckBox(&g_Config.bShowAllocatorDebug, dev->T("Allocator Viewer")));
+	if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN) {
+		items->Add(new CheckBox(&g_Config.bShowAllocatorDebug, dev->T("Allocator Viewer")));
+		items->Add(new CheckBox(&g_Config.bShowGpuProfile, dev->T("GPU Profile")));
+	}
 	items->Add(new Choice(dev->T("Toggle Freeze")))->OnClick.Handle(this, &DevMenu::OnFreezeFrame);
 	items->Add(new Choice(dev->T("Dump Frame GPU Commands")))->OnClick.Handle(this, &DevMenu::OnDumpFrame);
 	items->Add(new Choice(dev->T("Toggle Audio Debug")))->OnClick.Handle(this, &DevMenu::OnToggleAudioDebug);

UI/EmuScreen.cpp

@@ -1359,6 +1359,11 @@ void EmuScreen::renderUI() {
 	if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN && g_Config.bShowAllocatorDebug) {
 		DrawAllocatorVis(ctx, gpu);
 	}
+
+	if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN && g_Config.bShowGpuProfile) {
+		DrawProfilerVis(ctx, gpu);
+	}
+
 #endif
 
 #ifdef USE_PROFILER

ext/native/thin3d/VulkanRenderManager.cpp

@@ -1,6 +1,8 @@
 #include <algorithm>
 #include <cstdint>
 
+#include <sstream>
+
 #include "Common/Log.h"
 #include "base/logging.h"
 
@@ -131,6 +133,11 @@ VulkanRenderManager::VulkanRenderManager(VulkanContext *vulkan) : vulkan_(vulkan
 		res = vkAllocateCommandBuffers(vulkan_->GetDevice(), &cmd_alloc, &frameData_[i].mainCmd);
 		assert(res == VK_SUCCESS);
 		frameData_[i].fence = vulkan_->CreateFence(true);  // So it can be instantly waited on
+
+		VkQueryPoolCreateInfo query_ci{ VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO };
+		query_ci.queryCount = MAX_TIMESTAMP_QUERIES;
+		query_ci.queryType = VK_QUERY_TYPE_TIMESTAMP;
+		res = vkCreateQueryPool(vulkan_->GetDevice(), &query_ci, nullptr, &frameData_[i].timestampQueryPool_);
 	}
 
 	queueRunner_.CreateDeviceObjects();
@@ -219,6 +226,8 @@ void VulkanRenderManager::StopThread() {
 				std::unique_lock<std::mutex> lock(frameData.pull_mutex);
 				frameData.pull_condVar.notify_all();
 			}
+			// Zero the queries so we don't try to pull them later.
+			frameData.timestampDescriptions.clear();
 		}
 		thread_.join();
 		ILOG("Vulkan submission thread joined. Frame=%d", vulkan_->GetCurFrame());
@@ -289,6 +298,7 @@ VulkanRenderManager::~VulkanRenderManager() {
 		vkDestroyCommandPool(device, frameData_[i].cmdPoolInit, nullptr);
 		vkDestroyCommandPool(device, frameData_[i].cmdPoolMain, nullptr);
 		vkDestroyFence(device, frameData_[i].fence, nullptr);
+		vkDestroyQueryPool(device, frameData_[i].timestampQueryPool_, nullptr);
 	}
 	queueRunner_.DestroyDeviceObjects();
 }
@@ -339,7 +349,8 @@ void VulkanRenderManager::ThreadFunc() {
 	VLOG("PULL: Quitting");
 }
 
-void VulkanRenderManager::BeginFrame() {
+void VulkanRenderManager::BeginFrame(bool enableProfiling) {
+	gpuProfilingEnabled_ = enableProfiling;
 	VLOG("BeginFrame");
 	VkDevice device = vulkan_->GetDevice();
 
@@ -360,6 +371,41 @@ void VulkanRenderManager::BeginFrame() {
 	vkWaitForFences(device, 1, &frameData.fence, true, UINT64_MAX);
 	vkResetFences(device, 1, &frameData.fence);
 
+	uint64_t queryResults[MAX_TIMESTAMP_QUERIES];
+
+	if (gpuProfilingEnabled_) {
+		// Pull the profiling results from last time and produce a summary!
+		if (!frameData.timestampDescriptions.empty()) {
+			int numQueries = (int)frameData.timestampDescriptions.size();
+			VkResult res = vkGetQueryPoolResults(
+				vulkan_->GetDevice(),
+				frameData.timestampQueryPool_, 0, numQueries, sizeof(uint64_t) * numQueries, &queryResults[0], sizeof(uint64_t),
+				VK_QUERY_RESULT_64_BIT);
+			if (res == VK_SUCCESS) {
+				double timestampConversionFactor = (double)vulkan_->GetPhysicalDeviceProperties().properties.limits.timestampPeriod * (1.0 / 1000000.0);
+				int validBits = vulkan_->GetQueueFamilyProperties(vulkan_->GetGraphicsQueueFamilyIndex()).timestampValidBits;
+				uint64_t timestampDiffMask = validBits == 64 ? 0xFFFFFFFFFFFFFFFFULL : ((1ULL << validBits) - 1);
+				std::stringstream str;
+
+				char line[256];
+				snprintf(line, sizeof(line), "Total GPU time: %0.3f ms\n", ((double)((queryResults[numQueries - 1] - queryResults[0]) & timestampDiffMask) * timestampConversionFactor));
+				str << line;
+				for (int i = 0; i < numQueries - 1; i++) {
+					uint64_t diff = (queryResults[i + 1] - queryResults[i]) & timestampDiffMask;
+					double milliseconds = (double)diff * timestampConversionFactor;
+					snprintf(line, sizeof(line), "%s: %0.3f ms\n", frameData.timestampDescriptions[i + 1].c_str(), milliseconds);
+					str << line;
+				}
+				frameData.profileSummary = str.str();
+			} else {
+				frameData.profileSummary = "(error getting GPU profile - not ready?)";
+			}
+		}
+		else {
+			frameData.profileSummary = "(no GPU profile data collected)";
+		}
+	}
+
 	// Must be after the fence - this performs deletes.
 	VLOG("PUSH: BeginFrame %d", curFrame);
 	if (!run_) {
@@ -368,6 +414,18 @@ void VulkanRenderManager::BeginFrame() {
 	vulkan_->BeginFrame();
 
 	insideFrame_ = true;
+
+	frameData.timestampDescriptions.clear();
+	if (gpuProfilingEnabled_) {
+		// For various reasons, we need to always use an init cmd buffer in this case to perform the vkCmdResetQueryPool,
+		// unless we want to limit ourselves to only measure the main cmd buffer.
+		// Reserve the first two queries for initCmd.
+		frameData.timestampDescriptions.push_back("initCmd Begin");
+		frameData.timestampDescriptions.push_back("initCmd");
+		VkCommandBuffer initCmd = GetInitCmd();
+		vkCmdResetQueryPool(initCmd, frameData.timestampQueryPool_, 0, MAX_TIMESTAMP_QUERIES);
+		vkCmdWriteTimestamp(initCmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, frameData.timestampQueryPool_, 0);
+	}
 }
 
 VkCommandBuffer VulkanRenderManager::GetInitCmd() {
@@ -884,6 +942,7 @@ void VulkanRenderManager::BeginSubmitFrame(int frame) {
 		VkCommandBufferBeginInfo begin{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
 		begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
 		res = vkBeginCommandBuffer(frameData.mainCmd, &begin);
+
 		_assert_msg_(G3D, res == VK_SUCCESS, "vkBeginCommandBuffer failed! result=%s", VulkanResultToString(res));
 
 		queueRunner_.SetBackbuffer(framebuffers_[frameData.curSwapchainImage], swapchainImages_[frameData.curSwapchainImage].image);
@@ -895,10 +954,20 @@ void VulkanRenderManager::BeginSubmitFrame(int frame) {
 void VulkanRenderManager::Submit(int frame, bool triggerFence) {
 	FrameData &frameData = frameData_[frame];
 	if (frameData.hasInitCommands) {
+		if (gpuProfilingEnabled_ && triggerFence) {
+			// Pre-allocated query ID 1.
+			vkCmdWriteTimestamp(frameData.initCmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, frameData.timestampQueryPool_, 1);
+		}
 		VkResult res = vkEndCommandBuffer(frameData.initCmd);
 		_assert_msg_(G3D, res == VK_SUCCESS, "vkEndCommandBuffer failed (init)! result=%s", VulkanResultToString(res));
 	}
 
+	if (gpuProfilingEnabled_) {
+		int numQueries = (int)frameData.timestampDescriptions.size();
+		vkCmdWriteTimestamp(frameData.mainCmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, frameData.timestampQueryPool_, numQueries);
+		frameData.timestampDescriptions.push_back("mainCmd");
+	}
+
 	VkResult res = vkEndCommandBuffer(frameData.mainCmd);
 	_assert_msg_(G3D, res == VK_SUCCESS, "vkEndCommandBuffer failed (main)! result=%s", VulkanResultToString(res));
 

ext/native/thin3d/VulkanRenderManager.h

@@ -92,7 +92,7 @@ public:
 	void ThreadFunc();
 
 	// Makes sure that the GPU has caught up enough that we can start writing buffers of this frame again.
-	void BeginFrame();
+	void BeginFrame(bool enableProfiling);
 	// Can run on a different thread!
 	void Finish();
 	void Run(int frame);
@@ -242,6 +242,10 @@ public:
 		return &queueRunner_;
 	}
 
+	std::string GetGpuProfileString() const {
+		return frameData_[vulkan_->GetCurFrame()].profileSummary;
+	}
+
 private:
 	bool InitBackbufferFramebuffers(int width, int height);
 	bool InitDepthStencilBuffer(VkCommandBuffer cmd);  // Used for non-buffered rendering.
@@ -284,9 +288,22 @@ private:
 		// Swapchain.
 		bool hasBegun = false;
 		uint32_t curSwapchainImage = -1;
+
+		// Profiling.
+		VkQueryPool timestampQueryPool_ = VK_NULL_HANDLE;
+		std::vector<std::string> timestampDescriptions;
+		std::string profileSummary;
 	};
+
 	FrameData frameData_[VulkanContext::MAX_INFLIGHT_FRAMES];
 
+	enum {
+		MAX_TIMESTAMP_QUERIES = 256,
+	};
+
+	// Global state
+	bool gpuProfilingEnabled_ = false;
+
 	// Submission time state
 	int curWidth_ = -1;
 	int curHeight_ = -1;

ext/native/thin3d/thin3d_vulkan.cpp

@@ -879,7 +879,7 @@ VKContext::~VKContext() {
 }
 
 void VKContext::BeginFrame() {
-	renderManager_.BeginFrame();
+	renderManager_.BeginFrame(g_Config.bShowGpuProfile);
 
 	FrameData &frame = frame_[vulkan_->GetCurFrame()];
 	push_ = frame.pushBuffer;