Win32: Plumb through an attempt at handling Vulkan device lost.

Unfortunately, recreating the device fails on NV.
This commit is contained in:
Henrik Rydgård committed 2023-05-29 18:32:29 +02:00
1 parent f33776bc4a
commit 485131a29c
23 files changed
+184 -43

No files matched your search

+3 -2
View File
@@ -137,7 +137,7 @@ public:
void DrawUP(const void *vdata, int vertexCount) override;
void Clear(int mask, uint32_t colorval, float depthVal, int stencilVal) override;
void BeginFrame() override;
bool BeginFrame() override;
std::string GetInfoString(InfoField info) const override {
switch (info) {
@@ -1448,7 +1448,7 @@ void D3D11DrawContext::Clear(int mask, uint32_t colorval, float depthVal, int st
}
}
void D3D11DrawContext::BeginFrame() {
bool D3D11DrawContext::BeginFrame() {
context_->OMSetRenderTargets(1, &curRenderTargetView_, curDepthStencilView_);
if (curBlend_ != nullptr) {
@@ -1475,6 +1475,7 @@ void D3D11DrawContext::BeginFrame() {
context_->PSSetConstantBuffers(0, 1, &curPipeline_->dynamicUniforms);
}
}
return true;
}
void D3D11DrawContext::CopyFramebufferImage(Framebuffer *srcfb, int level, int x, int y, int z, Framebuffer *dstfb, int dstLevel, int dstX, int dstY, int dstZ, int width, int height, int depth, int channelBit, const char *tag) {
+3 -2
View File
@@ -367,7 +367,7 @@ public:
Buffer *CreateBuffer(size_t size, uint32_t usageFlags) override;
Framebuffer *CreateFramebuffer(const FramebufferDesc &desc) override;
void BeginFrame() override;
bool BeginFrame() override;
void EndFrame() override;
void UpdateBuffer(Buffer *buffer, const uint8_t *data, size_t offset, size_t size, UpdateBufferFlags flags) override;
@@ -782,10 +782,11 @@ OpenGLContext::~OpenGLContext() {
}
}
void OpenGLContext::BeginFrame() {
bool OpenGLContext::BeginFrame() {
renderManager_.BeginFrame(debugFlags_ & DebugFlags::PROFILE_TIMESTAMPS);
FrameData &frameData = frameData_[renderManager_.GetCurFrame()];
renderManager_.BeginPushBuffer(frameData.push);
return true;
}
void OpenGLContext::EndFrame() {
+2 -1
View File
@@ -702,7 +702,8 @@ VkResult VulkanContext::CreateDevice() {
VkResult res = vkCreateDevice(physical_devices_[physical_device_], &device_info, nullptr, &device_);
if (res != VK_SUCCESS) {
init_error_ = "Unable to create Vulkan device";
ERROR_LOG(G3D, "Unable to create Vulkan device");
ERROR_LOG(G3D, "Unable to create Vulkan device: '%s'", VulkanResultToString(res));
return res;
} else {
VulkanLoadDeviceFunctions(device_, extensionsLookup_);
}
+8
View File
@@ -76,6 +76,14 @@ VKAPI_ATTR VkBool32 VKAPI_CALL VulkanDebugUtilsCallback(
// Extended validation (ARM best practices)
// Non-fifo validation not recommended
return false;
// These get triggered during the device lost simulation. Ignore.
case -556648736:
case 1812873262:
case 337425955:
WARN_LOG(G3D, "Validation message %d typical of device lost simulation, ignoring.", messageCode);
return false;
default:
break;
}
+6 -1
View File
@@ -93,6 +93,7 @@ void FrameData::AcquireNextImage(VulkanContext *vulkan, FrameDataShared &shared)
VkResult FrameData::QueuePresent(VulkanContext *vulkan, FrameDataShared &shared) {
_dbg_assert_(hasAcquired);
_dbg_assert_(!deviceLost);
hasAcquired = false;
_dbg_assert_(!skipSwap);
@@ -132,6 +133,8 @@ VkCommandBuffer FrameData::GetInitCmd(VulkanContext *vulkan) {
}
void FrameData::SubmitPending(VulkanContext *vulkan, FrameSubmitType type, FrameDataShared &sharedData) {
_dbg_assert_(!deviceLost);
VkCommandBuffer cmdBufs[3];
int numCmdBufs = 0;
@@ -206,7 +209,9 @@ void FrameData::SubmitPending(VulkanContext *vulkan, FrameSubmitType type, Frame
}
if (res == VK_ERROR_DEVICE_LOST) {
_assert_msg_(false, "Lost the Vulkan device in vkQueueSubmit! If this happens again, switch Graphics Backend away from Vulkan");
ERROR_LOG(G3D, "Lost the Vulkan device in vkQueueSubmit! If this happens again, switch Graphics Backend away from Vulkan");
deviceLost = true;
return;
} else {
_assert_msg_(res == VK_SUCCESS, "vkQueueSubmit failed (main)! result=%s", VulkanResultToString(res));
}
+3
View File
@@ -88,6 +88,9 @@ struct FrameData {
bool syncDone = false;
// Set if the device was just lost.
bool deviceLost = false;
// Swapchain.
uint32_t curSwapchainImage = -1;
+4
View File
@@ -395,6 +395,9 @@ void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, FrameData &frame
vkCmdEndDebugUtilsLabelEXT(cmd);
}
frameData.SubmitPending(vulkan_, FrameSubmitType::Pending, frameDataShared);
if (frameData.deviceLost) {
goto bail;
}
// When stepping in the GE debugger, we can end up here multiple times in a "frame".
// So only acquire once.
@@ -447,6 +450,7 @@ void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, FrameData &frame
}
}
bail:
// Deleting all in one go should be easier on the instruction cache than deleting
// them as we go - and easier to debug because we can look backwards in the frame.
if (!keepSteps) {
+58 -5
View File
@@ -492,6 +492,9 @@ void VulkanRenderManager::DrainCompileQueue() {
void VulkanRenderManager::ThreadFunc() {
SetCurrentThreadName("RenderMan");
while (true) {
if (deviceLost_) {
break;
}
// Pop a task of the queue and execute it.
VKRRenderThreadTask *task = nullptr;
{
@@ -503,6 +506,12 @@ void VulkanRenderManager::ThreadFunc() {
renderThreadQueue_.pop();
}
if (deviceLost_) {
delete task;
// We'll clear out the rest after the break.
break;
}
// Oh, we got a task! We can now have pushMutex_ unlocked, allowing the host to
// push more work when it feels like it, and just start working.
if (task->runType == VKRRunType::EXIT) {
@@ -516,6 +525,16 @@ void VulkanRenderManager::ThreadFunc() {
delete task;
}
{
// Make sure nothing is left.
std::unique_lock<std::mutex> lock(pushMutex_);
while (!renderThreadQueue_.empty()) {
VKRRenderThreadTask *task = renderThreadQueue_.front();
renderThreadQueue_.pop();
delete task;
}
}
// Wait for the device to be done with everything, before tearing stuff down.
// TODO: Do we need this?
vkDeviceWaitIdle(vulkan_->GetDevice());
@@ -523,8 +542,11 @@ void VulkanRenderManager::ThreadFunc() {
VLOG("PULL: Quitting");
}
void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfiler) {
bool VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfiler) {
VLOG("BeginFrame");
if (deviceLost_) {
return false;
}
VkDevice device = vulkan_->GetDevice();
int curFrame = vulkan_->GetCurFrame();
@@ -545,8 +567,15 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile
// This must be the very first Vulkan call we do in a new frame.
// Makes sure the very last command buffer from the frame before the previous has been fully executed.
if (vkWaitForFences(device, 1, &frameData.fence, true, UINT64_MAX) == VK_ERROR_DEVICE_LOST) {
_assert_msg_(false, "Device lost in vkWaitForFences");
ERROR_LOG(G3D, "Device lost in vkWaitForFences");
frameData.deviceLost = true;
deviceLost_ = true;
// If the render thread is waiting for an event that won't come, kick it loose.
pushCondVar_.notify_one();
frameData.readyForFence = true;
return false;
}
vkResetFences(device, 1, &frameData.fence);
int validBits = vulkan_->GetQueueFamilyProperties(vulkan_->GetGraphicsQueueFamilyIndex()).timestampValidBits;
@@ -616,6 +645,7 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile
frameData.profile.timestampDescriptions.push_back("initCmd");
VkCommandBuffer initCmd = GetInitCmd();
}
return true;
}
VkCommandBuffer VulkanRenderManager::GetInitCmd() {
@@ -1280,6 +1310,10 @@ void VulkanRenderManager::Finish() {
steps_.clear();
vulkan_->EndFrame();
insideFrame_ = false;
if (deviceLost_) {
WARN_LOG(G3D, "VulkanRenderManager::Finish: Device lost");
}
}
void VulkanRenderManager::Wipe() {
@@ -1292,11 +1326,16 @@ void VulkanRenderManager::Wipe() {
// Called on the render thread.
//
// Can be called again after a VKRRunType::SYNC on the same frame.
void VulkanRenderManager::Run(VKRRenderThreadTask &task) {
bool VulkanRenderManager::Run(VKRRenderThreadTask &task) {
_dbg_assert_(!deviceLost_);
FrameData &frameData = frameData_[task.frame];
_dbg_assert_(!frameData.hasPresentCommands);
frameData.SubmitPending(vulkan_, FrameSubmitType::Pending, frameDataShared_);
if (frameData.deviceLost) {
return false;
}
if (!frameData.hasMainCommands) {
// Effectively resets both main and present command buffers, since they both live in this pool.
@@ -1312,8 +1351,9 @@ void VulkanRenderManager::Run(VKRRenderThreadTask &task) {
queueRunner_.PreprocessSteps(task.steps);
// Likely during shutdown, happens in headless.
if (task.steps.empty() && !frameData.hasAcquired)
if (task.steps.empty() && !frameData.hasAcquired) {
frameData.skipSwap = true;
}
//queueRunner_.LogSteps(stepsOnThread, false);
if (IsVREnabled()) {
int passes = GetVRPassesCount();
@@ -1326,10 +1366,18 @@ void VulkanRenderManager::Run(VKRRenderThreadTask &task) {
queueRunner_.RunSteps(task.steps, frameData, frameDataShared_);
}
if (frameData.deviceLost) {
deviceLost_ = true;
return false;
}
switch (task.runType) {
case VKRRunType::PRESENT:
frameData.SubmitPending(vulkan_, FrameSubmitType::Present, frameDataShared_);
if (frameData.deviceLost) {
deviceLost_ = true;
return false;
}
if (!frameData.skipSwap) {
VkResult res = frameData.QueuePresent(vulkan_, frameDataShared_);
if (res == VK_ERROR_OUT_OF_DATE_KHR) {
@@ -1369,7 +1417,12 @@ void VulkanRenderManager::Run(VKRRenderThreadTask &task) {
_dbg_assert_(false);
}
if (frameData.deviceLost) {
deviceLost_ = true;
}
VLOG("PULL: Finished running frame %d", task.frame);
return !deviceLost_;
}
// Called from main thread.
+8 -2
View File
@@ -185,7 +185,8 @@ public:
~VulkanRenderManager();
// Makes sure that the GPU has caught up enough that we can start writing buffers of this frame again.
void BeginFrame(bool enableProfiling, bool enableLogProfiler);
// A false return value means that the device is lost, and we should just try to end the frame ASAP without doing anything.
bool BeginFrame(bool enableProfiling, bool enableLogProfiler);
// Can run on a different thread!
void Finish();
// Zaps queued up commands. Use if you know there's a risk you've queued up stuff that has already been deleted. Can happen during in-game shutdown.
@@ -461,13 +462,16 @@ public:
void ResetStats();
void DrainCompileQueue();
bool DeviceIsLost() const { return deviceLost_; }
private:
void EndCurRenderStep();
void ThreadFunc();
void CompileThreadFunc();
void Run(VKRRenderThreadTask &task);
// Fails if the device was lost.
bool Run(VKRRenderThreadTask &task);
// Bad for performance but sometimes necessary for synchronous CPU readbacks (screenshots and whatnot).
void FlushSync();
@@ -481,6 +485,8 @@ private:
int outOfDateFrames_ = 0;
bool deviceLost_ = false;
// Submission time state
// Note: These are raw backbuffer-sized. Rotated.
+7 -4
View File
@@ -476,7 +476,7 @@ public:
void Clear(int mask, uint32_t colorval, float depthVal, int stencilVal) override;
void BeginFrame() override;
bool BeginFrame() override;
void EndFrame() override;
void WipeQueue() override;
@@ -1072,15 +1072,18 @@ VKContext::~VKContext() {
vulkan_->Delete().QueueDeletePipelineCache(pipelineCache_);
}
void VKContext::BeginFrame() {
// TODO: Bad dependency on g_Config here!
renderManager_.BeginFrame(debugFlags_ & DebugFlags::PROFILE_TIMESTAMPS, debugFlags_ & DebugFlags::PROFILE_SCOPES);
bool VKContext::BeginFrame() {
if (!renderManager_.BeginFrame(debugFlags_ & DebugFlags::PROFILE_TIMESTAMPS, debugFlags_ & DebugFlags::PROFILE_SCOPES)) {
// Something failed badly, let's bail.
return false;
}
FrameData &frame = frame_[vulkan_->GetCurFrame()];
push_->BeginFrame();
frame.descriptorPool.Reset();
return true;
}
void VKContext::EndFrame() {
+1 -1
View File
@@ -805,7 +805,7 @@ public:
virtual void DrawUP(const void *vdata, int vertexCount) = 0;
// Frame management (for the purposes of sync and resource management, necessary with modern APIs). Default implementations here.
virtual void BeginFrame() {}
virtual bool BeginFrame() { return true; }
virtual void EndFrame() = 0;
virtual void WipeQueue() {}
+4
View File
@@ -38,5 +38,9 @@ public:
// Should strive to get rid of these.
virtual void Poll() {}
virtual bool DeviceIsLost() const {
return false;
}
virtual Draw::DrawContext *GetDrawContext() = 0;
};
+14 -12
View File
@@ -165,21 +165,23 @@ void ScreenManager::render() {
// TODO: Make really sure that this "mismatched" pre/post only happens
// when screens are "compatible" (both are UIScreens, for example).
backback.screen->preRender();
backback.screen->render();
stack_.back().screen->render();
if (postRenderCb_)
postRenderCb_(getUIContext(), postRenderUserdata_);
backback.screen->postRender();
break;
if (backback.screen->preRender()) {
backback.screen->render();
stack_.back().screen->render();
if (postRenderCb_)
postRenderCb_(getUIContext(), postRenderUserdata_);
backback.screen->postRender();
break;
}
}
default:
_assert_(stack_.back().screen);
stack_.back().screen->preRender();
stack_.back().screen->render();
if (postRenderCb_)
postRenderCb_(getUIContext(), postRenderUserdata_);
stack_.back().screen->postRender();
if (stack_.back().screen->preRender()) {
stack_.back().screen->render();
if (postRenderCb_)
postRenderCb_(getUIContext(), postRenderUserdata_);
stack_.back().screen->postRender();
}
break;
}
} else {
+1 -1
View File
@@ -50,7 +50,7 @@ public:
virtual void onFinish(DialogResult reason) {}
virtual void update() {}
virtual void preRender() {}
virtual bool preRender() { return true; } // If this returns false, something is really bad and we should try to skip the rest of the frame, the error will be handled at the end.
virtual void render() {}
virtual void postRender() {}
virtual void resized() {}
+6 -3
View File
@@ -189,13 +189,15 @@ void UIScreen::deviceRestored() {
root_->DeviceRestored(screenManager()->getDrawContext());
}
void UIScreen::preRender() {
bool UIScreen::preRender() {
using namespace Draw;
Draw::DrawContext *draw = screenManager()->getDrawContext();
if (!draw) {
return;
return true;
}
if (!draw->BeginFrame()) {
return false;
}
draw->BeginFrame();
// Bind and clear the back buffer
draw->BindFramebufferAsRenderTarget(nullptr, { RPAction::CLEAR, RPAction::CLEAR, RPAction::CLEAR, 0xFF000000 }, "UI");
screenManager()->getUIContext()->BeginFrame();
@@ -209,6 +211,7 @@ void UIScreen::preRender() {
viewport.MinDepth = 0.0;
draw->SetViewport(viewport);
draw->SetTargetSize(g_display.pixel_xres, g_display.pixel_yres);
return true;
}
void UIScreen::postRender() {
+1 -1
View File
@@ -36,7 +36,7 @@ public:
~UIScreen();
void update() override;
void preRender() override;
bool preRender() override;
void render() override;
void postRender() override;
void deviceLost() override;
+14 -1
View File
@@ -230,6 +230,10 @@ void Core_RunLoop(GraphicsContext *ctx) {
Core_StateProcessed();
double startTime = time_now_d();
UpdateRunLoop();
if (graphicsContext->DeviceIsLost()) {
// Let the outer loop take care of this.
return;
}
// Simple throttling to not burn the GPU in the menu.
double diffTime = time_now_d() - startTime;
@@ -243,6 +247,10 @@ void Core_RunLoop(GraphicsContext *ctx) {
while ((coreState == CORE_RUNNING || coreState == CORE_STEPPING) && GetUIState() == UISTATE_INGAME) {
UpdateRunLoop();
if (graphicsContext->DeviceIsLost()) {
// Let the outer loop take care of this.
break;
}
if (!windowHidden && !Core_IsStepping()) {
ctx->SwapBuffers();
@@ -337,6 +345,9 @@ bool Core_Run(GraphicsContext *ctx) {
return false;
}
Core_RunLoop(ctx);
if (ctx->DeviceIsLost()) {
return true;
}
continue;
}
@@ -345,6 +356,9 @@ bool Core_Run(GraphicsContext *ctx) {
case CORE_STEPPING:
// enter a fast runloop
Core_RunLoop(ctx);
if (ctx->DeviceIsLost()) {
return true;
}
if (coreState == CORE_POWERDOWN) {
Core_StateProcessed();
return true;
@@ -357,7 +371,6 @@ bool Core_Run(GraphicsContext *ctx) {
case CORE_RUNTIME_ERROR:
// Exit loop!!
Core_StateProcessed();
return true;
case CORE_NEXTFRAME:
+5 -2
View File
@@ -1414,10 +1414,12 @@ static void DrawFrameTimes(UIContext *ctx, const Bounds &bounds) {
ctx->RebindTexture();
}
void EmuScreen::preRender() {
bool EmuScreen::preRender() {
using namespace Draw;
DrawContext *draw = screenManager()->getDrawContext();
draw->BeginFrame();
if (!draw->BeginFrame()) {
return false;
}
// Here we do NOT bind the backbuffer or clear the screen, unless non-buffered.
// The emuscreen is different than the others - we really want to allow the game to render to framebuffers
// before we ever bind the backbuffer for rendering. On mobile GPUs, switching back and forth between render
@@ -1443,6 +1445,7 @@ void EmuScreen::preRender() {
draw->SetViewport(viewport);
}
draw->SetTargetSize(g_display.pixel_xres, g_display.pixel_yres);
return true;
}
void EmuScreen::postRender() {
+1 -1
View File
@@ -44,7 +44,7 @@ public:
void update() override;
void render() override;
void preRender() override;
bool preRender() override;
void postRender() override;
void dialogFinished(const Screen *dialog, DialogResult result) override;
void sendMessage(const char *msg, const char *value) override;
+1
View File
@@ -1045,6 +1045,7 @@ void RenderOverlays(UIContext *dc, void *userdata) {
void NativeRender(GraphicsContext *graphicsContext) {
_dbg_assert_(graphicsContext != nullptr);
_dbg_assert_(g_screenManager != nullptr);
_dbg_assert_(!graphicsContext->DeviceIsLost())
g_GameManager.Update();
+28 -4
View File
@@ -259,6 +259,7 @@ void MainThreadFunc() {
// No safe way out without graphics.
ExitProcess(1);
return; // This return never executes, but helps the compiler.
}
GraphicsContext *graphicsContext = g_graphicsContext;
@@ -301,6 +302,28 @@ void MainThreadFunc() {
if (!Core_IsActive())
UpdateUIState(UISTATE_MENU);
Core_Run(g_graphicsContext);
if (g_graphicsContext->DeviceIsLost()) {
// Try to recreate the device here.
NativeShutdownGraphics();
graphicsContext->StopThread();
graphicsContext->ShutdownFromRenderThread();
delete graphicsContext;
graphicsContext = nullptr;
bool success = CreateGraphicsBackend(&error_string, &g_graphicsContext);
if (success) {
graphicsContext = g_graphicsContext;
// Main thread is the render thread.
success = g_graphicsContext->InitFromRenderThread(&error_string);
}
if (!success) {
ERROR_LOG(G3D, "Failed to recreate Vulkan device after device loss");
coreState = CORE_POWERDOWN;
break;
}
NativeInitGraphics(graphicsContext);
}
if (coreState == CORE_BOOT_ERROR) {
break;
}
@@ -329,11 +352,12 @@ shutdown:
if (!useEmuThread) {
NativeShutdownGraphics();
}
if (g_graphicsContext) {
g_graphicsContext->ThreadEnd();
g_graphicsContext->ShutdownFromRenderThread();
g_graphicsContext->ThreadEnd();
g_graphicsContext->ShutdownFromRenderThread();
g_graphicsContext->Shutdown();
g_graphicsContext->Shutdown();
}
UpdateConsolePosition();
NativeShutdown();
+5
View File
@@ -119,6 +119,7 @@ bool WindowsVulkanContext::Init(HINSTANCE hInst, HWND hWnd, std::string *error_m
vulkan_->ChooseDevice(deviceNum);
if (vulkan_->CreateDevice() != VK_SUCCESS) {
*error_message = vulkan_->InitError();
vulkan_->DestroyInstance();
delete vulkan_;
vulkan_ = nullptr;
return false;
@@ -184,3 +185,7 @@ void WindowsVulkanContext::Poll() {
void *WindowsVulkanContext::GetAPIContext() {
return vulkan_;
}
bool WindowsVulkanContext::DeviceIsLost() const {
return renderManager_->DeviceIsLost();
}
+1
View File
@@ -35,6 +35,7 @@ public:
void Poll() override;
void *GetAPIContext() override;
bool DeviceIsLost() const override;
Draw::DrawContext *GetDrawContext() override { return draw_; }
private: