diff --git a/Common/Vulkan/VulkanContext.cpp b/Common/Vulkan/VulkanContext.cpp index 297bc72d2f..dc87073214 100644 --- a/Common/Vulkan/VulkanContext.cpp +++ b/Common/Vulkan/VulkanContext.cpp @@ -516,8 +516,8 @@ void VulkanContext::ChooseDevice(int physical_device) { vkGetPhysicalDeviceQueueFamilyProperties(physical_devices_[physical_device_], &queue_count, nullptr); assert(queue_count >= 1); - queue_props.resize(queue_count); - vkGetPhysicalDeviceQueueFamilyProperties(physical_devices_[physical_device_], &queue_count, queue_props.data()); + queueFamilyProperties_.resize(queue_count); + vkGetPhysicalDeviceQueueFamilyProperties(physical_devices_[physical_device_], &queue_count, queueFamilyProperties_.data()); assert(queue_count >= 1); // Detect preferred formats, in this order. @@ -619,7 +619,7 @@ VkResult VulkanContext::CreateDevice() { queue_info.pQueuePriorities = queue_priorities; bool found = false; for (int i = 0; i < (int)queue_count; i++) { - if (queue_props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) { + if (queueFamilyProperties_[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) { queue_info.queueFamilyIndex = i; found = true; break; @@ -816,7 +816,7 @@ bool VulkanContext::InitQueue() { uint32_t graphicsQueueNodeIndex = UINT32_MAX; uint32_t presentQueueNodeIndex = UINT32_MAX; for (uint32_t i = 0; i < queue_count; i++) { - if ((queue_props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) { + if ((queueFamilyProperties_[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) { if (graphicsQueueNodeIndex == UINT32_MAX) { graphicsQueueNodeIndex = i; } diff --git a/Common/Vulkan/VulkanContext.h b/Common/Vulkan/VulkanContext.h index 01761aab6b..d3ecb96963 100644 --- a/Common/Vulkan/VulkanContext.h +++ b/Common/Vulkan/VulkanContext.h @@ -201,6 +201,10 @@ public: return physicalDeviceProperties_[i]; } + const VkQueueFamilyProperties &GetQueueFamilyProperties(int family) const { + return queueFamilyProperties_[family]; + } + VkResult GetInstanceLayerExtensionList(const char *layerName, std::vector &extensions); VkResult GetInstanceLayerProperties(); @@ -303,8 +307,8 @@ private: int physical_device_ = -1; uint32_t graphics_queue_family_index_ = -1; - std::vector physicalDeviceProperties_{}; - std::vector queue_props; + std::vector physicalDeviceProperties_; + std::vector queueFamilyProperties_; VkPhysicalDeviceMemoryProperties memory_properties{}; // Custom collection of things that are good to know diff --git a/Common/Vulkan/VulkanDebug.cpp b/Common/Vulkan/VulkanDebug.cpp index be7512b738..55825f8209 100644 --- a/Common/Vulkan/VulkanDebug.cpp +++ b/Common/Vulkan/VulkanDebug.cpp @@ -113,6 +113,13 @@ VkBool32 VKAPI_CALL VulkanDebugUtilsCallback( std::ostringstream message; const char *pMessage = pCallbackData->pMessage; + + // Apparent bugs around timestamp validation in the validation layers + if (strstr(pMessage, "vkCmdBeginQuery(): VkQueryPool")) + return false; + if (strstr(pMessage, "vkGetQueryPoolResults() on VkQueryPool")) + return false; + int messageCode = pCallbackData->messageIdNumber; const char *pLayerPrefix = ""; if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) { diff --git a/Core/Config.cpp b/Core/Config.cpp index ceb13f165f..ab9be6da50 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -959,9 +959,10 @@ static ConfigSetting debuggerSettings[] = { ConfigSetting("FontWidth", &g_Config.iFontWidth, 8), ConfigSetting("FontHeight", &g_Config.iFontHeight, 12), ConfigSetting("DisplayStatusBar", &g_Config.bDisplayStatusBar, true), - ConfigSetting("ShowBottomTabTitles",&g_Config.bShowBottomTabTitles,true), + ConfigSetting("ShowBottomTabTitles",&g_Config.bShowBottomTabTitles, true), ConfigSetting("ShowDeveloperMenu", &g_Config.bShowDeveloperMenu, false), ConfigSetting("ShowAllocatorDebug", &g_Config.bShowAllocatorDebug, false, false), + ConfigSetting("ShowGpuProfile", &g_Config.bShowGpuProfile, false, false), ConfigSetting("SkipDeadbeefFilling", &g_Config.bSkipDeadbeefFilling, false), ConfigSetting("FuncHashMap", &g_Config.bFuncHashMap, false), diff --git a/Core/Config.h b/Core/Config.h index 859bfbb3ad..2b04bf222d 100644 --- a/Core/Config.h +++ b/Core/Config.h @@ -68,21 +68,20 @@ public: bool bEnableLogging; bool bDumpDecryptedEboot; bool bFullscreenOnDoubleclick; -#if defined(USING_WIN_UI) + + // These four are Win UI only bool bPauseOnLostFocus; bool bTopMost; bool bIgnoreWindowsKey; bool bRestartRequired; -#endif -#if defined(USING_WIN_UI) || defined(USING_QT_UI) || PPSSPP_PLATFORM(UWP) + std::string sFont; -#endif bool bPauseWhenMinimized; -#if !defined(MOBILE_DEVICE) + // Not used on mobile devices. bool bPauseExitsEmulator; -#endif + bool bPauseMenuExitsEmulator; // Core @@ -129,9 +128,8 @@ public: // We have separate device parameters for each backend so it doesn't get erased if you switch backends. // If not set, will use the "best" device. std::string sVulkanDevice; -#ifdef _WIN32 - std::string sD3D11Device; -#endif + std::string sD3D11Device; // Windows only + bool bSoftwareRendering; bool bHardwareTransform; // only used in the GLES backend bool bSoftwareSkinning; // may speed up some games @@ -237,6 +235,7 @@ public: bool bLogFrameDrops; bool bShowDebugStats; bool bShowAudioDebug; + bool bShowGpuProfile; bool bAudioResampler; //Analog stick tilting @@ -366,9 +365,7 @@ public: int iPSPModel; int iFirmwareVersion; // TODO: Make this work with your platform, too! -#if defined(USING_WIN_UI) bool bBypassOSKWithKeyboard; -#endif // Debugger int iDisasmWindowX; diff --git a/GPU/Vulkan/DebugVisVulkan.cpp b/GPU/Vulkan/DebugVisVulkan.cpp index d93195bc16..1745cf57c1 100644 --- a/GPU/Vulkan/DebugVisVulkan.cpp +++ b/GPU/Vulkan/DebugVisVulkan.cpp @@ -26,6 +26,8 @@ #include "GPU/Vulkan/GPU_Vulkan.h" #include "GPU/Vulkan/VulkanUtil.h" +#undef DrawText + void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu) { if (!gpu) { return; @@ -94,3 +96,26 @@ void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu) { for (auto iter : texturesToDelete) iter->Release(); } + +void DrawProfilerVis(UIContext *ui, GPUInterface *gpu) { + if (!gpu) { + return; + } + using namespace Draw; + const int padding = 10; + const int columnWidth = 256; + const int starty = padding * 8; + int x = padding; + int y = starty; + int w = columnWidth; // We will double this when actually drawing to make the pixels visible. + + ui->Begin(); + + GPU_Vulkan *gpuVulkan = static_cast(gpu); + + std::string text = gpuVulkan->GetGpuProfileString(); + + Draw::DrawContext *draw = ui->GetDrawContext(); + ui->DrawTextShadow(text.c_str(), 10, 50, 0xFFFFFFFF, FLAG_DYNAMIC_ASCII); + ui->Flush(); +} diff --git a/GPU/Vulkan/DebugVisVulkan.h b/GPU/Vulkan/DebugVisVulkan.h index 9c0590bab4..e220003a56 100644 --- a/GPU/Vulkan/DebugVisVulkan.h +++ b/GPU/Vulkan/DebugVisVulkan.h @@ -24,3 +24,4 @@ class UIContext; // gpu MUST be an instance of GPU_Vulkan. If not, will definitely crash. void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu); +void DrawProfilerVis(UIContext *ui, GPUInterface *gpu); diff --git a/GPU/Vulkan/GPU_Vulkan.cpp b/GPU/Vulkan/GPU_Vulkan.cpp index 6507f0f0de..178faac8d6 100644 --- a/GPU/Vulkan/GPU_Vulkan.cpp +++ b/GPU/Vulkan/GPU_Vulkan.cpp @@ -638,3 +638,8 @@ std::string GPU_Vulkan::DebugGetShaderString(std::string id, DebugShaderType typ return std::string(); } } + +std::string GPU_Vulkan::GetGpuProfileString() { + VulkanRenderManager *rm = (VulkanRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER); + return rm->GetGpuProfileString(); +} diff --git a/GPU/Vulkan/GPU_Vulkan.h b/GPU/Vulkan/GPU_Vulkan.h index 42de6916c7..bb2efe5877 100644 --- a/GPU/Vulkan/GPU_Vulkan.h +++ b/GPU/Vulkan/GPU_Vulkan.h @@ -66,6 +66,8 @@ public: return textureCacheVulkan_; } + std::string GetGpuProfileString(); + protected: void FinishDeferred() override; diff --git a/UI/DevScreens.cpp b/UI/DevScreens.cpp index b4f430266f..d3e1d54f5d 100644 --- a/UI/DevScreens.cpp +++ b/UI/DevScreens.cpp @@ -78,7 +78,10 @@ void DevMenu::CreatePopupContents(UI::ViewGroup *parent) { items->Add(new Choice(sy->T("Developer Tools")))->OnClick.Handle(this, &DevMenu::OnDeveloperTools); items->Add(new Choice(dev->T("Jit Compare")))->OnClick.Handle(this, &DevMenu::OnJitCompare); items->Add(new Choice(dev->T("Shader Viewer")))->OnClick.Handle(this, &DevMenu::OnShaderView); - items->Add(new CheckBox(&g_Config.bShowAllocatorDebug, dev->T("Allocator Viewer"))); + if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN) { + items->Add(new CheckBox(&g_Config.bShowAllocatorDebug, dev->T("Allocator Viewer"))); + items->Add(new CheckBox(&g_Config.bShowGpuProfile, dev->T("GPU Profile"))); + } items->Add(new Choice(dev->T("Toggle Freeze")))->OnClick.Handle(this, &DevMenu::OnFreezeFrame); items->Add(new Choice(dev->T("Dump Frame GPU Commands")))->OnClick.Handle(this, &DevMenu::OnDumpFrame); items->Add(new Choice(dev->T("Toggle Audio Debug")))->OnClick.Handle(this, &DevMenu::OnToggleAudioDebug); diff --git a/UI/EmuScreen.cpp b/UI/EmuScreen.cpp index f0b86207a4..81a008e095 100644 --- a/UI/EmuScreen.cpp +++ b/UI/EmuScreen.cpp @@ -1359,6 +1359,11 @@ void EmuScreen::renderUI() { if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN && g_Config.bShowAllocatorDebug) { DrawAllocatorVis(ctx, gpu); } + + if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN && g_Config.bShowGpuProfile) { + DrawProfilerVis(ctx, gpu); + } + #endif #ifdef USE_PROFILER diff --git a/ext/native/thin3d/VulkanRenderManager.cpp b/ext/native/thin3d/VulkanRenderManager.cpp index 84bfcd3266..2fe59d3be0 100644 --- a/ext/native/thin3d/VulkanRenderManager.cpp +++ b/ext/native/thin3d/VulkanRenderManager.cpp @@ -1,6 +1,8 @@ #include #include +#include + #include "Common/Log.h" #include "base/logging.h" @@ -131,6 +133,11 @@ VulkanRenderManager::VulkanRenderManager(VulkanContext *vulkan) : vulkan_(vulkan res = vkAllocateCommandBuffers(vulkan_->GetDevice(), &cmd_alloc, &frameData_[i].mainCmd); assert(res == VK_SUCCESS); frameData_[i].fence = vulkan_->CreateFence(true); // So it can be instantly waited on + + VkQueryPoolCreateInfo query_ci{ VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO }; + query_ci.queryCount = MAX_TIMESTAMP_QUERIES; + query_ci.queryType = VK_QUERY_TYPE_TIMESTAMP; + res = vkCreateQueryPool(vulkan_->GetDevice(), &query_ci, nullptr, &frameData_[i].timestampQueryPool_); } queueRunner_.CreateDeviceObjects(); @@ -219,6 +226,8 @@ void VulkanRenderManager::StopThread() { std::unique_lock lock(frameData.pull_mutex); frameData.pull_condVar.notify_all(); } + // Zero the queries so we don't try to pull them later. + frameData.timestampDescriptions.clear(); } thread_.join(); ILOG("Vulkan submission thread joined. Frame=%d", vulkan_->GetCurFrame()); @@ -289,6 +298,7 @@ VulkanRenderManager::~VulkanRenderManager() { vkDestroyCommandPool(device, frameData_[i].cmdPoolInit, nullptr); vkDestroyCommandPool(device, frameData_[i].cmdPoolMain, nullptr); vkDestroyFence(device, frameData_[i].fence, nullptr); + vkDestroyQueryPool(device, frameData_[i].timestampQueryPool_, nullptr); } queueRunner_.DestroyDeviceObjects(); } @@ -339,7 +349,8 @@ void VulkanRenderManager::ThreadFunc() { VLOG("PULL: Quitting"); } -void VulkanRenderManager::BeginFrame() { +void VulkanRenderManager::BeginFrame(bool enableProfiling) { + gpuProfilingEnabled_ = enableProfiling; VLOG("BeginFrame"); VkDevice device = vulkan_->GetDevice(); @@ -360,6 +371,41 @@ void VulkanRenderManager::BeginFrame() { vkWaitForFences(device, 1, &frameData.fence, true, UINT64_MAX); vkResetFences(device, 1, &frameData.fence); + uint64_t queryResults[MAX_TIMESTAMP_QUERIES]; + + if (gpuProfilingEnabled_) { + // Pull the profiling results from last time and produce a summary! + if (!frameData.timestampDescriptions.empty()) { + int numQueries = (int)frameData.timestampDescriptions.size(); + VkResult res = vkGetQueryPoolResults( + vulkan_->GetDevice(), + frameData.timestampQueryPool_, 0, numQueries, sizeof(uint64_t) * numQueries, &queryResults[0], sizeof(uint64_t), + VK_QUERY_RESULT_64_BIT); + if (res == VK_SUCCESS) { + double timestampConversionFactor = (double)vulkan_->GetPhysicalDeviceProperties().properties.limits.timestampPeriod * (1.0 / 1000000.0); + int validBits = vulkan_->GetQueueFamilyProperties(vulkan_->GetGraphicsQueueFamilyIndex()).timestampValidBits; + uint64_t timestampDiffMask = validBits == 64 ? 0xFFFFFFFFFFFFFFFFULL : ((1ULL << validBits) - 1); + std::stringstream str; + + char line[256]; + snprintf(line, sizeof(line), "Total GPU time: %0.3f ms\n", ((double)((queryResults[numQueries - 1] - queryResults[0]) & timestampDiffMask) * timestampConversionFactor)); + str << line; + for (int i = 0; i < numQueries - 1; i++) { + uint64_t diff = (queryResults[i + 1] - queryResults[i]) & timestampDiffMask; + double milliseconds = (double)diff * timestampConversionFactor; + snprintf(line, sizeof(line), "%s: %0.3f ms\n", frameData.timestampDescriptions[i + 1].c_str(), milliseconds); + str << line; + } + frameData.profileSummary = str.str(); + } else { + frameData.profileSummary = "(error getting GPU profile - not ready?)"; + } + } + else { + frameData.profileSummary = "(no GPU profile data collected)"; + } + } + // Must be after the fence - this performs deletes. VLOG("PUSH: BeginFrame %d", curFrame); if (!run_) { @@ -368,6 +414,18 @@ void VulkanRenderManager::BeginFrame() { vulkan_->BeginFrame(); insideFrame_ = true; + + frameData.timestampDescriptions.clear(); + if (gpuProfilingEnabled_) { + // For various reasons, we need to always use an init cmd buffer in this case to perform the vkCmdResetQueryPool, + // unless we want to limit ourselves to only measure the main cmd buffer. + // Reserve the first two queries for initCmd. + frameData.timestampDescriptions.push_back("initCmd Begin"); + frameData.timestampDescriptions.push_back("initCmd"); + VkCommandBuffer initCmd = GetInitCmd(); + vkCmdResetQueryPool(initCmd, frameData.timestampQueryPool_, 0, MAX_TIMESTAMP_QUERIES); + vkCmdWriteTimestamp(initCmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, frameData.timestampQueryPool_, 0); + } } VkCommandBuffer VulkanRenderManager::GetInitCmd() { @@ -884,6 +942,7 @@ void VulkanRenderManager::BeginSubmitFrame(int frame) { VkCommandBufferBeginInfo begin{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; res = vkBeginCommandBuffer(frameData.mainCmd, &begin); + _assert_msg_(G3D, res == VK_SUCCESS, "vkBeginCommandBuffer failed! result=%s", VulkanResultToString(res)); queueRunner_.SetBackbuffer(framebuffers_[frameData.curSwapchainImage], swapchainImages_[frameData.curSwapchainImage].image); @@ -895,10 +954,20 @@ void VulkanRenderManager::BeginSubmitFrame(int frame) { void VulkanRenderManager::Submit(int frame, bool triggerFence) { FrameData &frameData = frameData_[frame]; if (frameData.hasInitCommands) { + if (gpuProfilingEnabled_ && triggerFence) { + // Pre-allocated query ID 1. + vkCmdWriteTimestamp(frameData.initCmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, frameData.timestampQueryPool_, 1); + } VkResult res = vkEndCommandBuffer(frameData.initCmd); _assert_msg_(G3D, res == VK_SUCCESS, "vkEndCommandBuffer failed (init)! result=%s", VulkanResultToString(res)); } + if (gpuProfilingEnabled_) { + int numQueries = (int)frameData.timestampDescriptions.size(); + vkCmdWriteTimestamp(frameData.mainCmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, frameData.timestampQueryPool_, numQueries); + frameData.timestampDescriptions.push_back("mainCmd"); + } + VkResult res = vkEndCommandBuffer(frameData.mainCmd); _assert_msg_(G3D, res == VK_SUCCESS, "vkEndCommandBuffer failed (main)! result=%s", VulkanResultToString(res)); diff --git a/ext/native/thin3d/VulkanRenderManager.h b/ext/native/thin3d/VulkanRenderManager.h index 84f7cbeed2..0b272f1257 100644 --- a/ext/native/thin3d/VulkanRenderManager.h +++ b/ext/native/thin3d/VulkanRenderManager.h @@ -92,7 +92,7 @@ public: void ThreadFunc(); // Makes sure that the GPU has caught up enough that we can start writing buffers of this frame again. - void BeginFrame(); + void BeginFrame(bool enableProfiling); // Can run on a different thread! void Finish(); void Run(int frame); @@ -242,6 +242,10 @@ public: return &queueRunner_; } + std::string GetGpuProfileString() const { + return frameData_[vulkan_->GetCurFrame()].profileSummary; + } + private: bool InitBackbufferFramebuffers(int width, int height); bool InitDepthStencilBuffer(VkCommandBuffer cmd); // Used for non-buffered rendering. @@ -284,9 +288,22 @@ private: // Swapchain. bool hasBegun = false; uint32_t curSwapchainImage = -1; + + // Profiling. + VkQueryPool timestampQueryPool_ = VK_NULL_HANDLE; + std::vector timestampDescriptions; + std::string profileSummary; }; + FrameData frameData_[VulkanContext::MAX_INFLIGHT_FRAMES]; + enum { + MAX_TIMESTAMP_QUERIES = 256, + }; + + // Global state + bool gpuProfilingEnabled_ = false; + // Submission time state int curWidth_ = -1; int curHeight_ = -1; diff --git a/ext/native/thin3d/thin3d_vulkan.cpp b/ext/native/thin3d/thin3d_vulkan.cpp index a62761feea..a0f384f149 100644 --- a/ext/native/thin3d/thin3d_vulkan.cpp +++ b/ext/native/thin3d/thin3d_vulkan.cpp @@ -879,7 +879,7 @@ VKContext::~VKContext() { } void VKContext::BeginFrame() { - renderManager_.BeginFrame(); + renderManager_.BeginFrame(g_Config.bShowGpuProfile); FrameData &frame = frame_[vulkan_->GetCurFrame()]; push_ = frame.pushBuffer;