diff --git a/Common/GPU/Vulkan/VulkanQueueRunner.cpp b/Common/GPU/Vulkan/VulkanQueueRunner.cpp index 0b4ca51983..ef48084812 100644 --- a/Common/GPU/Vulkan/VulkanQueueRunner.cpp +++ b/Common/GPU/Vulkan/VulkanQueueRunner.cpp @@ -5,6 +5,7 @@ #include "Common/GPU/Vulkan/VulkanQueueRunner.h" #include "Common/GPU/Vulkan/VulkanRenderManager.h" #include "Common/Log.h" +#include "Common/Profiler/Profiler.h" #include "Common/TimeUtil.h" #include "Common/VR/PPSSPPVR.h" @@ -1303,6 +1304,7 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c if (compileHere) { graphicsPipeline->Create(vulkan_, renderPass->Get(vulkan_, rpType, fbSampleCount), rpType, fbSampleCount, time_now_d(), -1); } + PROFILE_THIS_SCOPE("wait_pipeline"); VkPipeline pipeline = promise->BlockUntilReady(); if (pipeline != VK_NULL_HANDLE) { diff --git a/Common/GPU/Vulkan/VulkanRenderManager.cpp b/Common/GPU/Vulkan/VulkanRenderManager.cpp index 8632a5423d..b73fa8a6d5 100644 --- a/Common/GPU/Vulkan/VulkanRenderManager.cpp +++ b/Common/GPU/Vulkan/VulkanRenderManager.cpp @@ -30,6 +30,8 @@ using namespace PPSSPP_VK; // renderPass is an example of the "compatibility class" or RenderPassType type. bool VKRGraphicsPipeline::Create(VulkanContext *vulkan, VkRenderPass compatibleRenderPass, RenderPassType rpType, VkSampleCountFlagBits sampleCount, double scheduleTime, int countToCompile) { + PROFILE_THIS_SCOPE("pipeline_create"); + PROFILE_LABEL_COPY(tag_.c_str()); // Good torture test to test the shutdown-while-precompiling-shaders issue on PC where it's normally // hard to catch because shaders compile so fast. // sleep_ms(200); @@ -52,8 +54,12 @@ bool VKRGraphicsPipeline::Create(VulkanContext *vulkan, VkRenderPass compatibleR } // Fill in the last part of the desc since now it's time to block. - VkShaderModule vs = desc->vertexShader->BlockUntilReady(); - VkShaderModule fs = desc->fragmentShader->BlockUntilReady(); + VkShaderModule vs, fs; + { + PROFILE_THIS_SCOPE("wait_shaders"); + vs = desc->vertexShader->BlockUntilReady(); + fs = desc->fragmentShader->BlockUntilReady(); + } if (!vs || !fs) { ERROR_LOG(Log::G3D, "Failed creating graphics pipeline - missing shader modules"); @@ -968,6 +974,7 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile // Makes sure the submission from the previous time around has happened. Otherwise // we are not allowed to wait from another thread here.. if (useRenderThread_) { + PROFILE_THIS_SCOPE("wait_submit"); std::unique_lock lock(frameData.fenceMutex); while (!frameData.readyForFence) { frameData.fenceCondVar.wait(lock); @@ -984,8 +991,11 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile // This must be the very first Vulkan call we do in a new frame. // Makes sure the very last command buffer from the frame before the previous has been fully executed. - if (vkWaitForFences(device, 1, &frameData.fence, true, UINT64_MAX) == VK_ERROR_DEVICE_LOST) { - _assert_msg_(false, "Device lost in vkWaitForFences"); + { + PROFILE_THIS_SCOPE("wait_gpu"); + if (vkWaitForFences(device, 1, &frameData.fence, true, UINT64_MAX) == VK_ERROR_DEVICE_LOST) { + _assert_msg_(false, "Device lost in vkWaitForFences"); + } } if (!RecreatePresentationIfNeeded()) { @@ -2028,6 +2038,8 @@ void VulkanRenderManager::FlushSync() { } { + // Waits for the render thread to run everything queued so far, and the GPU to finish it. + PROFILE_THIS_SCOPE("wait_sync"); std::unique_lock lock(syncMutex_); // Wait for the flush to be hit, since we're syncing. while (!frameData.syncDone) { diff --git a/Common/Profiler/Profiler.cpp b/Common/Profiler/Profiler.cpp index 8e44d9a7cf..60f4899e90 100644 --- a/Common/Profiler/Profiler.cpp +++ b/Common/Profiler/Profiler.cpp @@ -11,6 +11,11 @@ #ifdef USE_PROFILER +#include +#include + +#include "Common/File/FileUtil.h" +#include "Common/File/Path.h" #include "Common/Log.h" #include "Common/StringUtils.h" #include "Common/Thread/ThreadUtil.h" @@ -21,6 +26,10 @@ #define MAX_DEPTH 32 // Deeper scopes are recorded at this level. #define LANE_EVENTS 16384 // Must be a power of 2. 256 KB a thread. #define HISTORY_SIZE 128 // Must be a power of 2. +#define LABEL_COPIES 64 +#define RECENT_FRAMES 61 +#define MAX_FRAME_SPANS 4096 +#define NO_EVENT UINT64_MAX namespace { @@ -34,13 +43,31 @@ struct Lane { // Owning thread only. int depth = 0; + // Event index of each open scope's Begin, for labelling it. NO_EVENT if it wasn't recorded. + uint64_t openBegin[MAX_DEPTH]{}; + char labelCopies[LABEL_COPIES][48]{}; + int labelCopyPos = 0; // End of frame only (one thread). Replays the events to attribute the time between them to the - // innermost open scope. + // innermost open scope, and to find the long scopes. uint64_t aggRead = 0; uint64_t aggLastTicks = 0; int aggDepth = 0; uint16_t aggStack[MAX_DEPTH]{}; + uint64_t aggBeginTicks[MAX_DEPTH]{}; + const char *aggBeginLabel[MAX_DEPTH]{}; + bool aggBeginValid[MAX_DEPTH]{}; +}; + +// A scope that ended in the current frame, long enough to log. +struct Span { + int lane; + int category; + int level; + uint64_t begin; + uint64_t end; + const char *label; + uint64_t self; // Minus the logged spans nested in it. }; struct FrameTotals { @@ -58,7 +85,6 @@ const char *g_categories[MAX_CATEGORIES]; std::atomic g_numCategories; std::mutex g_categoriesMutex; -std::atomic g_paused; bool g_initialized; // End of frame only, apart from reads of the finished frames. @@ -68,6 +94,14 @@ int g_historyPos; double g_secondsPerTick; uint64_t g_startTicks; +// Logging. Set before init or from the end-of-frame thread. +double g_hitchMs; +double g_spanMs; +FILE *g_logFile; +uint64_t g_lastFrameEnd; +uint64_t g_frameNumber; +std::vector g_frameSpans; + struct LaneOwner { Lane *lane = nullptr; bool failed = false; @@ -114,14 +148,17 @@ inline Lane *CurrentLane() { return owner.lane; } -inline void Record(Lane *lane, ProfilerEventType type, int category, int level) { +// Returns the event's index. +inline uint64_t Record(Lane *lane, ProfilerEventType type, int category, int level) { const uint64_t w = lane->written.load(std::memory_order_relaxed); ProfilerEvent &e = lane->events[w & (LANE_EVENTS - 1)]; e.ticks = time_now_fast_ticks(); + e.label = nullptr; e.category = (uint16_t)category; e.type = type; - e.level = (int8_t)std::min(level, MAX_DEPTH - 1); + e.level = (int8_t)level; lane->written.store(w + 1, std::memory_order_release); + return w; } // Copies event index i of the lane. False if the writer has overwritten (or may be overwriting) it. @@ -145,18 +182,23 @@ void Attribute(Lane *lane, int laneIndex, FrameTotals &frame, uint64_t until) { } } -void AggregateLane(Lane *lane, int laneIndex, FrameTotals &frame, uint64_t frameEnd) { +void LoseTrack(Lane *lane) { + lane->aggDepth = 0; + memset(lane->aggBeginValid, 0, sizeof(lane->aggBeginValid)); +} + +void AggregateLane(Lane *lane, int laneIndex, FrameTotals &frame, uint64_t frameEnd, uint64_t spanTicks) { const uint64_t written = lane->written.load(std::memory_order_acquire); uint64_t i = lane->aggRead; if (i < OldestReadable(written)) { // Lost some, so the open scopes are unknown. The next event's level resyncs them. i = OldestReadable(written); - lane->aggDepth = 0; + LoseTrack(lane); } for (; i < written; i++) { ProfilerEvent e; if (!ReadEvent(lane, i, &e)) { - lane->aggDepth = 0; + LoseTrack(lane); continue; } if (e.ticks > frameEnd) { @@ -167,12 +209,24 @@ void AggregateLane(Lane *lane, int laneIndex, FrameTotals &frame, uint64_t frame lane->aggLastTicks = e.ticks; } Attribute(lane, laneIndex, frame, e.ticks); + const int level = e.level; if (e.type == ProfilerEventType::Begin) { - lane->aggStack[e.level] = e.category; - lane->aggDepth = e.level + 1; + lane->aggStack[level] = e.category; + lane->aggDepth = level + 1; + lane->aggBeginTicks[level] = e.ticks; + // The label may come after this read, if a frame ended in between. Rare enough. + lane->aggBeginLabel[level] = e.label; + lane->aggBeginValid[level] = true; } else { - lane->aggDepth = e.level; + lane->aggDepth = level; frame.counts[laneIndex][e.category]++; + if (lane->aggBeginValid[level] && lane->aggStack[level] == e.category) { + const uint64_t length = e.ticks - lane->aggBeginTicks[level]; + if (spanTicks && length >= spanTicks && g_frameSpans.size() < MAX_FRAME_SPANS) { + g_frameSpans.push_back(Span{ laneIndex, e.category, level, lane->aggBeginTicks[level], e.ticks, lane->aggBeginLabel[level], length }); + } + } + lane->aggBeginValid[level] = false; } } lane->aggRead = i; @@ -184,8 +238,143 @@ int HistoryIndex(int i, int count) { return (g_historyPos - count + i + HISTORY_SIZE) & (HISTORY_SIZE - 1); } +// Writes s as a JSON string. +void WriteJsonString(FILE *f, const char *s) { + fputc('"', f); + for (; *s; s++) { + const unsigned char c = (unsigned char)*s; + if (c == '"' || c == '\\') { + fputc('\\', f); + fputc(c, f); + } else if (c < 0x20) { + fprintf(f, "\\u%04x", c); + } else { + fputc(c, f); + } + } + fputc('"', f); +} + +double TicksToMs(uint64_t ticks) { + return ticks * g_secondsPerTick * 1000.0; +} + +// Recent frame lengths, for what's normal for the game right now. +double g_recentFrameMs[RECENT_FRAMES]; +int g_recentFramePos; + +void LogFrame(const FrameTotals &frame, int numLanes, uint64_t frameStart, uint64_t frameEnd) { + const double frameMs = TicksToMs(frameEnd - frameStart); + // Slow both in absolute terms and for the game: one that runs at 30 fps takes 33 ms every frame. + double sorted[RECENT_FRAMES]; + memcpy(sorted, g_recentFrameMs, sizeof(sorted)); + std::nth_element(sorted, sorted + RECENT_FRAMES / 2, sorted + RECENT_FRAMES); + const double typicalMs = sorted[RECENT_FRAMES / 2]; + g_recentFrameMs[g_recentFramePos] = frameMs; + g_recentFramePos = (g_recentFramePos + 1) % RECENT_FRAMES; + const bool hitch = frameMs >= g_hitchMs && frameMs >= typicalMs * 1.5; + const int numCategories = g_numCategories.load(std::memory_order_acquire); + + char names[MAX_LANES][32]; + { + std::lock_guard guard(g_lanesMutex); + for (int i = 0; i < numLanes; i++) { + truncate_cpy(names[i], g_lanes[i]->name); + } + } + + // Self time of each span: what's left after the logged spans directly inside it. + std::sort(g_frameSpans.begin(), g_frameSpans.end(), [](const Span &a, const Span &b) { + return a.lane != b.lane ? a.lane < b.lane : (a.begin != b.begin ? a.begin < b.begin : a.level < b.level); + }); + for (size_t i = 0; i < g_frameSpans.size(); i++) { + Span &parent = g_frameSpans[i]; + for (size_t j = i + 1; j < g_frameSpans.size(); j++) { + const Span &child = g_frameSpans[j]; + if (child.lane != parent.lane || child.begin >= parent.end) { + break; + } + if (child.level == parent.level + 1 && child.end <= parent.end) { + parent.self -= std::min(parent.self, child.end - child.begin); + } + } + } + + if (FILE *f = g_logFile) { + fprintf(f, "{\"f\":%llu,\"t\":%0.4f,\"ms\":%0.2f,\"typ\":%0.1f,%s\"th\":{", (unsigned long long)g_frameNumber, TicksToMs(frameEnd - g_startTicks) * 0.001, frameMs, typicalMs, hitch ? "\"slow\":1," : ""); + bool firstLane = true; + for (int l = 0; l < numLanes; l++) { + bool firstCat = true; + for (int c = 0; c < numCategories; c++) { + if (frame.seconds[l][c] <= 0.0f && frame.counts[l][c] == 0) { + continue; + } + if (firstCat) { + fputs(firstLane ? "" : ",", f); + WriteJsonString(f, names[l]); + fputs(":{", f); + firstLane = false; + } + fputs(firstCat ? "" : ",", f); + WriteJsonString(f, g_categories[c]); + fprintf(f, ":[%0.3f,%d]", frame.seconds[l][c] * 1000.0f, frame.counts[l][c]); + firstCat = false; + } + if (!firstCat) { + fputc('}', f); + } + } + fputs("}}\n", f); + if (hitch) { + for (const Span &s : g_frameSpans) { + fprintf(f, "{\"span\":%llu,\"th\":", (unsigned long long)g_frameNumber); + WriteJsonString(f, names[s.lane]); + fputs(",\"c\":", f); + WriteJsonString(f, g_categories[s.category]); + if (s.label) { + fputs(",\"n\":", f); + WriteJsonString(f, s.label); + } + fprintf(f, ",\"l\":%d,\"t\":%0.4f,\"ms\":%0.3f,\"self\":%0.3f}\n", s.level, TicksToMs(s.begin - g_startTicks) * 0.001, TicksToMs(s.end - s.begin), TicksToMs(s.self)); + } + fflush(f); + } else if ((g_frameNumber & 63) == 0) { + fflush(f); + } + } + + if (!hitch) { + return; + } + // The log line: the spans that took the most time themselves, leaving out plain sleeps. + std::vector worst; + for (const Span &s : g_frameSpans) { + if (strcmp(g_categories[s.category], "sleep") != 0) { + worst.push_back(&s); + } + } + std::sort(worst.begin(), worst.end(), [](const Span *a, const Span *b) { return a->self > b->self; }); + int vblanks = 0; + for (int c = 0; c < numCategories; c++) { + if (!strcmp(g_categories[c], "vblank")) { + for (int l = 0; l < numLanes; l++) { + vblanks += frame.counts[l][c]; + } + } + } + char buf[512]; + size_t len = snprintf(buf, sizeof(buf), "Profiler: Frame %llu took %0.1f ms (typically %0.1f), %d vblanks.", (unsigned long long)g_frameNumber, frameMs, typicalMs, vblanks); + for (size_t i = 0; i < worst.size() && i < 6 && len < sizeof(buf); i++) { + const Span &s = *worst[i]; + len += snprintf(buf + len, sizeof(buf) - len, " %s %s%s%s %0.1f/%0.1f ms.", names[s.lane], g_categories[s.category], s.label ? ":" : "", s.label ? s.label : "", TicksToMs(s.self), TicksToMs(s.end - s.begin)); + } + INFO_LOG(Log::System, "%s", buf); +} + } // namespace +std::atomic g_profilerEnabled; + void internal_profiler_init() { if (g_initialized) { return; @@ -226,10 +415,37 @@ void Profiler_Enter(int category) { if (!lane) { return; } - const int level = lane->depth++; - if (!g_paused.load(std::memory_order_relaxed)) { - Record(lane, ProfilerEventType::Begin, category, level); + const int level = std::min(lane->depth++, MAX_DEPTH - 1); + lane->openBegin[level] = Record(lane, ProfilerEventType::Begin, category, level); +} + +void Profiler_Label(const char *text) { + if (!g_initialized || !text) { + return; } + Lane *lane = CurrentLane(); + if (!lane || lane->depth == 0) { + return; + } + const uint64_t index = lane->openBegin[std::min(lane->depth - 1, MAX_DEPTH - 1)]; + // Unless the ring has moved past it. + if (index != NO_EVENT && index + LANE_EVENTS > lane->written.load(std::memory_order_relaxed)) { + lane->events[index & (LANE_EVENTS - 1)].label = text; + } +} + +void Profiler_LabelCopy(const char *text) { + if (!g_initialized || !text) { + return; + } + Lane *lane = CurrentLane(); + if (!lane || lane->depth == 0) { + return; + } + char *copy = lane->labelCopies[lane->labelCopyPos]; + lane->labelCopyPos = (lane->labelCopyPos + 1) % LABEL_COPIES; + truncate_cpy(copy, sizeof(lane->labelCopies[0]), text); + Profiler_Label(copy); } void Profiler_Leave(int category) { @@ -241,14 +457,17 @@ void Profiler_Leave(int category) { if (!lane || lane->depth == 0) { return; } - const int level = --lane->depth; - if (!g_paused.load(std::memory_order_relaxed)) { - Record(lane, ProfilerEventType::End, category, level); - } + const int level = std::min(--lane->depth, MAX_DEPTH - 1); + Record(lane, ProfilerEventType::End, category, level); } void internal_profiler_end_frame() { - if (!g_initialized || g_paused.load(std::memory_order_relaxed)) { + if (!g_initialized) { + return; + } + if (!Profiler_IsEnabled()) { + // So the first frame after turning it back on isn't taken for a long one. + g_lastFrameEnd = 0; return; } const uint64_t now = time_now_fast_ticks(); @@ -256,11 +475,39 @@ void internal_profiler_end_frame() { FrameTotals &frame = g_history[pos]; memset(&frame, 0, sizeof(frame)); const int numLanes = g_numLanes.load(std::memory_order_acquire); + const bool logging = g_hitchMs > 0.0; + const uint64_t spanTicks = logging ? std::max(1, (uint64_t)(g_spanMs * 0.001 / g_secondsPerTick)) : 0; + g_frameSpans.clear(); for (int i = 0; i < numLanes; i++) { - AggregateLane(g_lanes[i], i, frame, now); + AggregateLane(g_lanes[i], i, frame, now, spanTicks); } g_frameEnds[pos] = now; g_historyPos = (pos + 1) & (HISTORY_SIZE - 1); + + if (logging && g_lastFrameEnd != 0) { + LogFrame(frame, numLanes, g_lastFrameEnd, now); + } + g_lastFrameEnd = now; + g_frameNumber++; +} + +void Profiler_SetLogging(const char *path, double hitchMs, double spanMs) { + if (g_logFile) { + fclose(g_logFile); + g_logFile = nullptr; + } + g_hitchMs = hitchMs; + g_spanMs = spanMs; + if (hitchMs > 0.0) { + Profiler_SetEnabled(true); + } + if (hitchMs > 0.0 && path && path[0]) { + g_logFile = File::OpenCFile(Path(path), "wb"); + if (!g_logFile) { + ERROR_LOG(Log::System, "Profiler: Couldn't open %s for the log", path); + } + } + INFO_LOG(Log::System, "Profiler: Logging frames of %0.1f ms or more%s%s", hitchMs, g_logFile ? " to " : "", g_logFile ? path : ""); } const char *Profiler_GetCategoryName(int i) { @@ -329,12 +576,8 @@ void Profiler_GetHistoryCounts(int category, int thread, int *data, int count) { } } -void Profiler_SetPaused(bool paused) { - g_paused.store(paused, std::memory_order_relaxed); -} - -bool Profiler_IsPaused() { - return g_paused.load(std::memory_order_relaxed); +void Profiler_SetEnabled(bool enabled) { + g_profilerEnabled.store(enabled, std::memory_order_relaxed); } uint64_t Profiler_Now() { diff --git a/Common/Profiler/Profiler.h b/Common/Profiler/Profiler.h index de40165f3f..4fe35f51ed 100644 --- a/Common/Profiler/Profiler.h +++ b/Common/Profiler/Profiler.h @@ -1,9 +1,17 @@ #pragma once +#include #include #include -// #define USE_PROFILER +#include "ppsspp_config.h" + +// Compiled in everywhere but iOS, UWP, libretro and builds that define NO_PROFILER (Android release +// builds), and off until something turns it on (Profiler_SetEnabled), which costs a scope one load and +// a branch. Check USE_PROFILER only after including this file. +#if !defined(USE_PROFILER) && !defined(NO_PROFILER) && !PPSSPP_PLATFORM(IOS) && !PPSSPP_PLATFORM(UWP) && !defined(__LIBRETRO__) +#define USE_PROFILER +#endif #ifdef USE_PROFILER @@ -14,6 +22,13 @@ void internal_profiler_init(); void internal_profiler_end_frame(); +extern std::atomic g_profilerEnabled; +// While off, nothing is recorded and the history stands still. +void Profiler_SetEnabled(bool enabled); +inline bool Profiler_IsEnabled() { + return g_profilerEnabled.load(std::memory_order_relaxed); +} + // Returns the category number for a name, adding it if new. Names are compared by pointer first, so // a string literal is cheap to look up. int Profiler_GetCategory(const char *name); @@ -22,11 +37,16 @@ void Profiler_Leave(int category); class ProfileThis { public: - explicit ProfileThis(int category) : category_(category) { - Profiler_Enter(category); + explicit ProfileThis(int category) : category_(Profiler_IsEnabled() ? category : -1) { + if (category_ >= 0) { + Profiler_Enter(category_); + } } ~ProfileThis() { - Profiler_Leave(category_); + // Left even if the profiler was turned off meanwhile, so Begins and Ends pair up. + if (category_ >= 0) { + Profiler_Leave(category_); + } } private: int category_; @@ -44,6 +64,22 @@ private: #define PROFILE_THIS_SCOPE_DYNAMIC(cat) ProfileThis PROFILE_CONCAT(_profile_scoped, __LINE__)(Profiler_GetCategory(cat)); #define PROFILE_END_FRAME() internal_profiler_end_frame(); +// Names this thread's innermost open scope (which syscall, which file) in the log and the timeline. +// The text must outlive the profiler, like a literal or a name in a static table. +void Profiler_Label(const char *text); +// The same for text that won't live that long: copies the first 47 characters into the thread's lane, +// which keeps the last 64 copies. +void Profiler_LabelCopy(const char *text); +#define PROFILE_LABEL(text) do { if (Profiler_IsEnabled()) { Profiler_Label(text); } } while (false) +#define PROFILE_LABEL_COPY(text) do { if (Profiler_IsEnabled()) { Profiler_LabelCopy(text); } } while (false) + +// Logs stalls without the UI. Any frame (between two PROFILE_END_FRAME calls) of at least hitchMs gets +// a line in the log naming its longest scopes. With a path, every frame's per-thread totals go to that +// file as JSON lines, and for each frame of at least hitchMs, every scope of at least spanMs that ended +// in it. Tools/profile_report.py summarizes the file. Turns the profiler on, unless hitchMs <= 0, which +// turns the logging off. +void Profiler_SetLogging(const char *path, double hitchMs, double spanMs); + // Per-frame totals, in seconds of time spent in each category itself (not in scopes nested in it). // A frame is the time between two PROFILE_END_FRAME() calls; history index count - 1 is the last one. const char *Profiler_GetCategoryName(int i); @@ -57,10 +93,6 @@ void Profiler_GetHistory(int category, int thread, float *data, int count); // How many times each scope of the category was left in the frame. void Profiler_GetHistoryCounts(int category, int thread, int *data, int count); -// While paused, nothing is recorded and the history stands still. -void Profiler_SetPaused(bool paused); -bool Profiler_IsPaused(); - // For the timeline. enum class ProfilerEventType : uint8_t { Begin, @@ -69,6 +101,8 @@ enum class ProfilerEventType : uint8_t { struct ProfilerEvent { uint64_t ticks; + // From PROFILE_LABEL, on a Begin. Null if none. + const char *label; uint16_t category; ProfilerEventType type; // Nesting depth, 0 at the outermost scope. Begin and End of one scope have the same level. @@ -93,5 +127,7 @@ void Profiler_GetFrameEnds(std::vector *ends); #define PROFILE_THIS_SCOPE(cat) #define PROFILE_THIS_SCOPE_DYNAMIC(cat) #define PROFILE_END_FRAME() +#define PROFILE_LABEL(text) +#define PROFILE_LABEL_COPY(text) #endif diff --git a/Common/TimeUtil.cpp b/Common/TimeUtil.cpp index f0d1d4c70f..9599351d38 100644 --- a/Common/TimeUtil.cpp +++ b/Common/TimeUtil.cpp @@ -5,6 +5,7 @@ #include #include "Common/TimeUtil.h" +#include "Common/Profiler/Profiler.h" #include "Common/Data/Random/Rng.h" #include "Common/Log.h" @@ -290,6 +291,8 @@ void sleep_ms(int ms, const char *reason) { if (ms <= 0) { return; } + PROFILE_THIS_SCOPE("sleep"); + PROFILE_LABEL(reason); #if SLEEP_LOG_ENABLED INFO_LOG(Log::System, "Sleep %d ms: %s", ms, reason); #endif @@ -308,6 +311,8 @@ void sleep_us(int us, const char *reason) { if (us <= 0) { return; } + PROFILE_THIS_SCOPE("sleep"); + PROFILE_LABEL(reason); #if SLEEP_LOG_ENABLED INFO_LOG(Log::System, "Sleep %d us: %s", us, reason); #endif @@ -327,6 +332,8 @@ void sleep_precise(double seconds, const char *reason) { if (seconds <= 0.0) { return; } + PROFILE_THIS_SCOPE("sleep"); + PROFILE_LABEL(reason); #if SLEEP_LOG_ENABLED INFO_LOG(Log::System, "Sleep precise %f s: %s", seconds, reason); #endif @@ -384,6 +391,8 @@ void sleep_approx(double seconds, const char *reason) { if (seconds <= 0.0) { return; } + PROFILE_THIS_SCOPE("sleep"); + PROFILE_LABEL(reason); #if SLEEP_LOG_ENABLED INFO_LOG(Log::System, "Sleep approx %f s: %s", seconds, reason); #endif diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 1ab2a44079..41c2951d41 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -10,6 +10,7 @@ #include "Core/WebServer.h" #include "Core/Util/PathUtil.h" #include "Common/File/FileUtil.h" +#include "Common/Profiler/Profiler.h" #include "Common/StringUtils.h" #include "Common/Log/LogManager.h" @@ -267,6 +268,9 @@ static const CommandLineParam g_autoParams[] = { {POFF(adhocDelayMs), CmdParamType::Int, "adhoc-delay", '\0', "Delay ad hoc datagrams received by MS milliseconds of emulated time, to simulate a distant player"}, {POFF(adhocJitterMs), CmdParamType::Int, "adhoc-jitter", '\0', "With --adhoc-delay, up to MS more at random"}, {POFF(adhocLossPercent), CmdParamType::Int, "adhoc-loss", '\0', "Drop this percentage of the ad hoc datagrams received"}, + {POFF(profileLog), CmdParamType::String, "profile-log", '\0', "Profiler builds: write each frame's profile, and the long scopes of slow frames, to FILE as JSON lines (see Tools/profile_report.py)"}, + {POFF(profileHitchMs), CmdParamType::Double, "profile-hitch-ms", '\0', "Profiler builds: log frames taking at least MS milliseconds (default 25)"}, + {POFF(profileSpanMs), CmdParamType::Double, "profile-span-ms", '\0', "Profiler builds: in slow frames, log the scopes of at least MS milliseconds (default 1)"}, {POFF(lagSync), CmdParamType::Bool, "lag-sync", '\0', "Keep emulated time in step with the wall clock every millisecond, not every frame (the Force real clock sync setting)"}, {POFF(frameStats), CmdParamType::Bool, "frame-stats", '\0', "Print flips per emulated second at the end of the run, counted from --state's load", CmdLineMode::Headless}, {POFF(audioSinkPeriod), CmdParamType::Int, "audio-sink", '\0', "Pull audio like an output device, FRAMES at 48 kHz at a time, and print the buffer's stats at the end (with --realtime)", CmdLineMode::Headless}, @@ -593,6 +597,13 @@ CommandLineParseResult CommandLineOptions::Parse(int argc, const char *argv[], C } void CommandLineOptions::ApplyToConfig() const { + if (profileLog.has_value() || profileHitchMs.has_value()) { +#ifdef USE_PROFILER + Profiler_SetLogging(profileLog.value_or("").c_str(), profileHitchMs.value_or(25.0), profileSpanMs.value_or(1.0)); +#else + WARN_LOG(Log::System, "--profile-log and --profile-hitch-ms need a build with USE_PROFILER"); +#endif + } if (adhocServer.has_value()) { g_Config.bEnableAdhocServer = adhocServer.value(); g_Config.DoNotSaveSetting(&g_Config.bEnableAdhocServer); diff --git a/Core/CmdLine.h b/Core/CmdLine.h index 5e6c88cf0f..9c8c680adf 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -219,6 +219,10 @@ struct CommandLineOptions { std::optional adhocDelayMs; std::optional adhocJitterMs; std::optional adhocLossPercent; + // Frame profiler logging (builds with USE_PROFILER). See Profiler_SetLogging. + std::optional profileLog; + std::optional profileHitchMs; + std::optional profileSpanMs; std::optional screenshotFilename; std::optional screenshotFilenameSave; diff --git a/Core/ConfigValues.h b/Core/ConfigValues.h index 64d2baae98..0e87547407 100644 --- a/Core/ConfigValues.h +++ b/Core/ConfigValues.h @@ -290,9 +290,7 @@ enum class DebugOverlay : int { DEBUG_STATS, FRAME_GRAPH, FRAME_TIMING, -#ifdef USE_PROFILER - FRAME_PROFILE, -#endif + FRAME_PROFILE, // Draws nothing in builds without USE_PROFILER. CONTROL, Audio, GPU_PROFILE, diff --git a/Core/CoreTiming.cpp b/Core/CoreTiming.cpp index 04aef5f5bd..7083385e9e 100644 --- a/Core/CoreTiming.cpp +++ b/Core/CoreTiming.cpp @@ -409,6 +409,8 @@ void ProcessEvents() { Event *evt = first; first = first->next; if (evt->type >= 0 && evt->type < (int)event_types.size()) { + PROFILE_THIS_SCOPE("event"); + PROFILE_LABEL(event_types[evt->type].name); event_types[evt->type].callback(evt->userdata, (int)(GetTicks(currentMIPS) - evt->time)); } else { _dbg_assert_msg_(false, "Bad event type %d", evt->type); diff --git a/Core/HLE/HLE.cpp b/Core/HLE/HLE.cpp index f6076c9a4d..2e62931bb1 100644 --- a/Core/HLE/HLE.cpp +++ b/Core/HLE/HLE.cpp @@ -1197,6 +1197,11 @@ const HLEFunction *GetSyscallFunctionData(MIPSOpcode op, u32 pcForDiagnostics) { void *GetQuickSyscallFunc(const HLEFunction *info, MIPSOpcode op) { if (g_coreCollectDebugStats) return nullptr; +#ifdef USE_PROFILER + // CallSyscall has the profiler scope. Blocks compiled before the profiler was turned on skip it. + if (Profiler_IsEnabled()) + return nullptr; +#endif if (!info || !info->func) return nullptr; @@ -1230,6 +1235,7 @@ void CallSyscallWithPC(MIPSState *mips, MIPSOpcode op, u32 pc) { RETURN(SCE_KERNEL_ERROR_LIBRARY_NOT_YET_LINKED); return; } + PROFILE_LABEL(info->name); if (info->func) { if (op == g_idleOp) diff --git a/Core/HLE/sceDisplay.cpp b/Core/HLE/sceDisplay.cpp index 3890eefb59..eb3cd5aab0 100644 --- a/Core/HLE/sceDisplay.cpp +++ b/Core/HLE/sceDisplay.cpp @@ -660,7 +660,7 @@ static void DoFrameIdleTiming() { #else const double left = goal - cur_time; if (left > 0.0f && left < 1.0f) { // Sanity check - usleep((long)(left * 1000000)); + sleep_us((int)(left * 1000000), "frame-idle"); } #endif } @@ -723,6 +723,8 @@ LagSyncMode __DisplayGetLagSyncMode() { } void hleEnterVblank(u64 userdata, int cyclesLate) { + // Counted per frame: Tools/profile_report.py compares a frame's length with the vblanks in it. + PROFILE_THIS_SCOPE("vblank"); int vbCount = userdata; UpdateLagSyncMode(); @@ -808,6 +810,8 @@ void __DisplaySetDisplayLayoutConfig(const DisplayLayoutConfig &config) { } void __DisplayFlip(int cyclesLate) { + // Runs on each vblank that hasn't flipped yet, and holds the frame-idle sleep. + PROFILE_THIS_SCOPE("flip"); if (!gpu) { _dbg_assert_(gpu); // Something has gone wrong. diff --git a/Core/HLE/sceKernelModule.cpp b/Core/HLE/sceKernelModule.cpp index b1602aed2c..fb559d1208 100644 --- a/Core/HLE/sceKernelModule.cpp +++ b/Core/HLE/sceKernelModule.cpp @@ -22,6 +22,7 @@ #include "zlib.h" #include "Common/Data/Convert/SmallDataConvert.h" +#include "Common/Profiler/Profiler.h" #include "Common/Serialize/Serializer.h" #include "Common/Serialize/SerializeFuncs.h" #include "Common/Serialize/SerializeMap.h" @@ -2392,6 +2393,8 @@ SceUID KernelLoadModule(const std::string &filename, std::string *error_string, // filename is only used for dumping etc. static PSPModule *__KernelLoadModule(u8 *fileptr, size_t fileSize, SceKernelLMOption *options, std::string_view filename, std::string *error_string) { + PROFILE_THIS_SCOPE("module_load"); + PROFILE_LABEL_COPY(std::string(filename).c_str()); PSPModule *module = nullptr; // Check for PBP if (fileSize >= sizeof(PSP_Header) && memcmp(fileptr, "\0PBP", 4) == 0) { diff --git a/Core/MIPS/ARM/ArmIRCompSystem.cpp b/Core/MIPS/ARM/ArmIRCompSystem.cpp index 0ccd4ab8f1..517c578c55 100644 --- a/Core/MIPS/ARM/ArmIRCompSystem.cpp +++ b/Core/MIPS/ARM/ArmIRCompSystem.cpp @@ -188,13 +188,7 @@ void ArmJitBackend::CompIR_System(IRInst inst) { SaveStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::SYSCALL); -#ifdef USE_PROFILER - // When profiling, we can't skip CallSyscall, since it times syscalls. - MOVP2R(R0, mips_); - MOVI2R(R1, inst.constant); - QuickCallFunction(R12, &CallSyscall); -#else - // Skip the CallSyscall where possible. + // Skip the CallSyscall where possible (GetQuickSyscallFunc says when not). { MIPSOpcode op(inst.constant); const HLEFunction *func = GetSyscallFunctionData(op, 0); @@ -216,7 +210,6 @@ void ArmJitBackend::CompIR_System(IRInst inst) { QuickCallFunction(R12, &CallSyscallUnresolvedAtPC); } } -#endif WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); LoadStaticRegisters(); diff --git a/Core/MIPS/ARM64/Arm64IRCompSystem.cpp b/Core/MIPS/ARM64/Arm64IRCompSystem.cpp index 2e3386a50b..cad25bdf8a 100644 --- a/Core/MIPS/ARM64/Arm64IRCompSystem.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompSystem.cpp @@ -213,13 +213,7 @@ void Arm64JitBackend::CompIR_System(IRInst inst) { SaveStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::SYSCALL); -#ifdef USE_PROFILER - // When profiling, we can't skip CallSyscall, since it times syscalls. - MOVP2R(X0, mips_); - MOVI2R(W1, inst.constant); - QuickCallFunction(SCRATCH2_64, &CallSyscall); -#else - // Skip the CallSyscall where possible. + // Skip the CallSyscall where possible (GetQuickSyscallFunc says when not). { MIPSOpcode op(inst.constant); const HLEFunction *func = GetSyscallFunctionData(op, 0); @@ -241,7 +235,6 @@ void Arm64JitBackend::CompIR_System(IRInst inst) { QuickCallFunction(SCRATCH2_64, &CallSyscallUnresolvedAtPC); } } -#endif WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); LoadStaticRegisters(); diff --git a/Core/MIPS/LoongArch64/LoongArch64CompSystem.cpp b/Core/MIPS/LoongArch64/LoongArch64CompSystem.cpp index 9f9a60051f..a0e1a49fda 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompSystem.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompSystem.cpp @@ -177,13 +177,7 @@ void LoongArch64JitBackend::CompIR_System(IRInst inst) { SaveStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::SYSCALL); -#ifdef USE_PROFILER - // When profiling, we can't skip CallSyscall, since it times syscalls. - LI(R4, (uintptr_t)mips_); - LI(R5, (int32_t)inst.constant); - QuickCallFunction(&CallSyscall, SCRATCH2); -#else - // Skip the CallSyscall where possible. + // Skip the CallSyscall where possible (GetQuickSyscallFunc says when not). { MIPSOpcode op(inst.constant); const HLEFunction *func = GetSyscallFunctionData(op, 0); @@ -205,7 +199,6 @@ void LoongArch64JitBackend::CompIR_System(IRInst inst) { QuickCallFunction(&CallSyscallUnresolvedAtPC, SCRATCH2); } } -#endif WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); LoadStaticRegisters(); // This is always followed by an ExitToPC, where we check coreState. diff --git a/Core/MIPS/RiscV/RiscVCompSystem.cpp b/Core/MIPS/RiscV/RiscVCompSystem.cpp index fdfe23a31d..c6be8738b1 100644 --- a/Core/MIPS/RiscV/RiscVCompSystem.cpp +++ b/Core/MIPS/RiscV/RiscVCompSystem.cpp @@ -185,13 +185,7 @@ void RiscVJitBackend::CompIR_System(IRInst inst) { SaveStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::SYSCALL); -#ifdef USE_PROFILER - // When profiling, we can't skip CallSyscall, since it times syscalls. - LI(X10, (uintptr_t)mips_); - LI(X11, (int32_t)inst.constant); - QuickCallFunction(&CallSyscall); -#else - // Skip the CallSyscall where possible. + // Skip the CallSyscall where possible (GetQuickSyscallFunc says when not). { MIPSOpcode op(inst.constant); const HLEFunction *func = GetSyscallFunctionData(op, 0); @@ -213,7 +207,6 @@ void RiscVJitBackend::CompIR_System(IRInst inst) { QuickCallFunction(&CallSyscallUnresolvedAtPC); } } -#endif WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); LoadStaticRegisters(); diff --git a/Core/MIPS/x86/X64IRCompSystem.cpp b/Core/MIPS/x86/X64IRCompSystem.cpp index 5db8e72338..69d1ebb3e2 100644 --- a/Core/MIPS/x86/X64IRCompSystem.cpp +++ b/Core/MIPS/x86/X64IRCompSystem.cpp @@ -204,11 +204,7 @@ void X64JitBackend::CompIR_System(IRInst inst) { SaveStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::SYSCALL); -#ifdef USE_PROFILER - // When profiling, we can't skip CallSyscall, since it times syscalls. - ABI_CallFunctionPA((const u8 *)&CallSyscall, mips_, Imm32(inst.constant)); -#else - // Skip the CallSyscall where possible. + // Skip the CallSyscall where possible (GetQuickSyscallFunc says when not). { MIPSOpcode op(inst.constant); const HLEFunction *func = GetSyscallFunctionData(op, 0); @@ -225,7 +221,6 @@ void X64JitBackend::CompIR_System(IRInst inst) { ABI_CallFunctionPA((const u8 *)&CallSyscallUnresolvedAtPC, mips_, Imm32(0)); } } -#endif WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); LoadStaticRegisters(); diff --git a/GPU/Common/FramebufferManagerCommon.cpp b/GPU/Common/FramebufferManagerCommon.cpp index 8c2f380281..fd1d513927 100644 --- a/GPU/Common/FramebufferManagerCommon.cpp +++ b/GPU/Common/FramebufferManagerCommon.cpp @@ -20,6 +20,7 @@ #include #include "Common/GPU/thin3d.h" +#include "Common/Profiler/Profiler.h" #include "Common/Data/Collections/TinySet.h" #include "Common/Data/Convert/ColorConv.h" #include "Common/LogReporting.h" @@ -3508,6 +3509,7 @@ bool FramebufferManagerCommon::GetOutputFramebuffer(GPUDebugBuffer &buffer) { // dithering behavior and games that expect exact colors like Danganronpa, so we // can't entirely be rid of the CPU path.) -- unknown void FramebufferManagerCommon::ReadbackFramebuffer(VirtualFramebuffer *vfb, int x, int y, int w, int h, RasterChannel channel, Draw::ReadbackMode mode, u8 *shadow) { + PROFILE_THIS_SCOPE("fb_readback"); if (w <= 0 || h <= 0) { ERROR_LOG(Log::FrameBuf, "Bad inputs to ReadbackFramebufferSync: %d %d %d %d", x, y, w, h); return; @@ -3738,6 +3740,7 @@ void FramebufferManagerCommon::FlushBeforeCopy() { // TODO: Replace with with depal, reading the palette from the texture on the GPU directly. void FramebufferManagerCommon::DownloadFramebufferForClut(u32 fb_address, u32 loadBytes) { + PROFILE_THIS_SCOPE("fb_clut_download"); VirtualFramebuffer *vfb = GetVFBAt(fb_address); // Without an fbo there's nothing to read back (ReadbackFramebuffer would read the backbuffer instead). if (vfb && vfb->fb_stride != 0 && vfb->fbo) { diff --git a/GPU/GPUCommon.cpp b/GPU/GPUCommon.cpp index 93aa816aec..b336cb9ed4 100644 --- a/GPU/GPUCommon.cpp +++ b/GPU/GPUCommon.cpp @@ -2611,6 +2611,7 @@ void GPUCommon::NotifyVideoCopy(u32 dest, u32 src, int size) { } bool GPUCommon::PerformMemoryCopy(u32 dest, u32 src, int size, GPUCopyFlag flags) { + PROFILE_THIS_SCOPE("gpu_memcpy"); if (size == 0) { _dbg_assert_msg_(false, "Zero-sized PerformMemoryCopy: %08x -> %08x, size %d (flag: %d)", src, dest, size, (int)flags); return false; @@ -2657,6 +2658,7 @@ bool GPUCommon::PerformMemoryCopy(u32 dest, u32 src, int size, GPUCopyFlag flags } bool GPUCommon::PerformMemorySet(u32 dest, u8 v, int size) { + PROFILE_THIS_SCOPE("gpu_memset"); if (size == 0) { _dbg_assert_msg_(false, "Zero-sized PerformMemorySet: %08x, value %02x, size %d", dest, v, size); return false; @@ -2698,6 +2700,7 @@ bool GPUCommon::PerformReadbackToMemory(u32 dest, int size) { } bool GPUCommon::PerformWriteColorFromMemory(u32 dest, int size) { + PROFILE_THIS_SCOPE("gpu_upload"); if (Memory::IsVRAMAddress(dest)) { recorder_.NotifyUpload(dest, size); return PerformMemoryCopy(dest, dest, size, GPUCopyFlag::FORCE_SRC_MATCH_MEM | GPUCopyFlag::DEBUG_NOTIFIED); diff --git a/Tools/profile_report.py b/Tools/profile_report.py new file mode 100644 index 0000000000..bf4ce4971e --- /dev/null +++ b/Tools/profile_report.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Summarizes a frame profiler log, as written with --profile-log=FILE. + +The log has a JSON line per frame, with each thread's time and count per category, and for each slow +frame (--profile-hitch-ms) a line per scope of at least --profile-span-ms that ended in it, with the +scope's own time (minus the logged scopes inside it) and its label (which syscall, event, module...). + + python Tools/profile_report.py profile.jsonl # summary, causes and the worst frames + python Tools/profile_report.py profile.jsonl --frame 1234 # one frame's scopes +""" + +import argparse +import json +import sys +from collections import defaultdict + +# Waiting, not working. Left out when naming what made a frame slow, but shown in the details. +WAIT_CATEGORIES = ("sleep",) + + +def load(path): + frames = {} + spans = defaultdict(list) + with open(path, encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue # A line cut short at the end of a run. + if "span" in rec: + spans[rec["span"]].append(rec) + elif "f" in rec: + frames[rec["f"]] = rec + return frames, spans + + +def percentile(values, p): + if not values: + return 0.0 + values = sorted(values) + return values[min(len(values) - 1, int(len(values) * p / 100))] + + +def span_name(s): + return s["c"] + (":" + s["n"] if s.get("n") else "") + + +def is_wait(s): + return s["c"] in WAIT_CATEGORIES + + +def emu_thread(frame): + """The thread running the game: the one with JIT (or interpreter) time.""" + for thread, cats in frame.get("th", {}).items(): + if "jit" in cats or "advance" in cats: + return thread + return None + + +VBLANK_MS = 1000.0 / 59.94 +# A frame this far behind the emulated time it covered is a stall: half a vblank. +STALL_MS = 8.0 +# Frame details show the scopes with this much work of their own. +DETAIL_MS = 1.0 +# The threads that a frame waits for, other than the emulator thread. Vulkan's is "VulkanRenderMan". +RENDER_THREAD_PREFIX = "VulkanRender" + + +def behind_ms(frame): + """How much longer the frame took than the emulated time it covered: the PSP takes a vblank + (16.7 ms) per vblank, so a game that doesn't flip for ten of them (loading, say) shows a 167 ms + frame on the PSP too. A frame without vblanks covered no emulated time (a menu, say).""" + thread = emu_thread(frame) + vblanks = frame["th"][thread].get("vblank", [0.0, 0])[1] if thread else 0 + return frame["ms"] - vblanks * VBLANK_MS + + +def culprits(frame_spans): + """The frame's working scopes, by their own time, longest first.""" + return sorted((s for s in frame_spans if not is_wait(s)), key=lambda s: -s["self"]) + + +def print_frame(frame, frame_spans, out): + out.write("Frame %d at %.2f s: %.1f ms (typically %.1f), %.1f ms behind emulated time\n" % (frame["f"], frame["t"], frame["ms"], frame.get("typ", 0.0), behind_ms(frame))) + for thread, cats in frame.get("th", {}).items(): + busy = sorted(cats.items(), key=lambda kv: -kv[1][0]) + parts = ["%s %.1f%s" % (c, v[0], " x%d" % v[1] if v[1] > 1 else "") for c, v in busy if v[0] >= 0.1] + if parts: + out.write(" %-16s %s\n" % (thread, ", ".join(parts))) + by_thread = defaultdict(list) + for s in frame_spans: + by_thread[s["th"]].append(s) + # Times from the frame's start. A scope that began in an earlier frame starts below zero. + start = frame["t"] - frame["ms"] / 1000.0 + for thread, ss in by_thread.items(): + ss = sorted(ss, key=lambda s: (s["t"], s["l"])) + # The scopes that did work themselves, and the ones around them. + keep = set(i for i, s in enumerate(ss) if s["self"] >= DETAIL_MS and not is_wait(s)) + for i in list(keep): + inner = ss[i] + for j, outer in enumerate(ss): + if outer["l"] < inner["l"] and outer["t"] <= inner["t"] and outer["t"] + outer["ms"] / 1000.0 >= inner["t"] + inner["ms"] / 1000.0 - 1e-6: + keep.add(j) + if not keep: + continue + out.write(" %s scopes with %.0f ms or more of their own work (start, then length and own time):\n" % (thread, DETAIL_MS)) + for i, s in enumerate(ss): + if i in keep: + out.write(" %+8.1f ms %s%s %.1f ms (own %.1f)\n" % ((s["t"] - start) * 1000.0, " " * s["l"], span_name(s), s["ms"], s["self"])) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("log") + ap.add_argument("--hitch-ms", type=float, default=None, help="Count frames of at least this long as slow (default: the logged threshold)") + ap.add_argument("--since", type=float, default=0.0, help="Leave out frames before this many seconds") + ap.add_argument("--worst", type=int, default=5, help="Show the scopes of this many of the slowest frames") + ap.add_argument("--causes", type=int, default=25, help="Show this many causes") + ap.add_argument("--frame", type=int, action="append", help="Show this frame's scopes (repeatable)") + args = ap.parse_args() + + frames, spans = load(args.log) + out = sys.stdout + if args.frame: + for n in args.frame: + if n in frames: + print_frame(frames[n], spans.get(n, []), out) + else: + out.write("No frame %d\n" % n) + return + + chosen = [f for f in frames.values() if f["t"] >= args.since] + if not chosen: + out.write("No frames.\n") + return + ms = [f["ms"] for f in chosen] + if args.hitch_ms is not None: + slow = [f for f in chosen if f["ms"] >= args.hitch_ms] + rule = "of %.1f ms or more" % args.hitch_ms + else: + # As logged: slow in absolute terms, and at least 1.5x the frames around it ("typ"). + slow = [f for f in chosen if f.get("slow")] + rule = "logged as slow" + # A slow frame that covered as much emulated time is the game not flipping, as slow on a PSP + # (a loading screen, say). The rest are stalls: emulation fell behind real time. + paced = [f for f in slow if behind_ms(f) < STALL_MS] + slow = [f for f in slow if behind_ms(f) >= STALL_MS] + slow.sort(key=lambda f: -f["ms"]) + duration = chosen[-1]["t"] - chosen[0]["t"] + out.write("%d frames over %.1f s. Frame time median %.1f ms, 95%% %.1f, 99%% %.1f, max %.1f.\n" % ( + len(chosen), duration, percentile(ms, 50), percentile(ms, 95), percentile(ms, 99), max(ms))) + stalled = sum(behind_ms(f) for f in slow) + out.write("%d stalls (frames %s that fell %.0f ms or more behind emulated time), %.2f s behind in all.\n" % (len(slow), rule, STALL_MS, stalled / 1000.0)) + out.write("%d more slow frames took as long as the vblanks they covered (the game not flipping, as on a PSP).\n" % len(paced)) + + # What the stalls' time went to on the threads a frame waits for: each category's own time, from + # the exact per-frame totals. Worker threads run alongside, and show up here as wait_* when waited + # for. A stall's "cause" is its biggest category. + cat_ms = defaultdict(float) + cat_frames = defaultdict(int) + first_cause = defaultdict(int) + for f in slow: + here = defaultdict(float) + for thread, cats in f.get("th", {}).items(): + if thread != emu_thread(f) and not thread.startswith(RENDER_THREAD_PREFIX): + continue + for c, v in cats.items(): + if c not in WAIT_CATEGORIES: + here["%s %s" % (thread, c)] += v[0] + for k, v in here.items(): + cat_ms[k] += v + cat_frames[k] += 1 + if here: + first_cause[max(here.items(), key=lambda kv: kv[1])[0]] += 1 + if cat_ms: + out.write("\nTime in stalls by thread and category (own time):\n") + out.write(" %9s %6s %6s %s\n" % ("ms", "frames", "worst", "category")) + for name, total in sorted(cat_ms.items(), key=lambda kv: -kv[1])[:args.causes]: + out.write(" %9.1f %6d %6d %s\n" % (total, cat_frames[name], first_cause.get(name, 0), name)) + out.write(" (worst: how many stalls it was the biggest category of)\n") + + # Which syscall, event, module or pipeline: the labelled scopes, on any thread. + own = defaultdict(float) + count = defaultdict(int) + for f in slow: + for s in spans.get(f["f"], []): + if s.get("n") and not is_wait(s): + own[span_name(s)] += s["self"] + count[span_name(s)] += 1 + if own: + out.write("\nLabelled scopes in stalls (own time, scopes of at least the logged length, any thread):\n") + out.write(" %9s %6s %s\n" % ("ms", "times", "scope")) + for name, total in sorted(own.items(), key=lambda kv: -kv[1])[:args.causes]: + out.write(" %9.1f %6d %s\n" % (total, count[name], name)) + + for f in slow[:args.worst]: + out.write("\n") + print_frame(f, spans.get(f["f"], []), out) + + +if __name__ == "__main__": + main() diff --git a/UI/DevScreens.cpp b/UI/DevScreens.cpp index 90a0377514..c13f17400f 100644 --- a/UI/DevScreens.cpp +++ b/UI/DevScreens.cpp @@ -99,9 +99,7 @@ static const char *g_debugOverlayList[] = { "Debug stats", "Draw Frametimes Graph", "Frame timing", -#ifdef USE_PROFILER "Frame profile", -#endif "Control Debug", "Audio Debug", "GPU Profile", diff --git a/UI/EmuScreen.cpp b/UI/EmuScreen.cpp index 075d1bdfd1..dc3314a935 100644 --- a/UI/EmuScreen.cpp +++ b/UI/EmuScreen.cpp @@ -2049,11 +2049,9 @@ void EmuScreen::renderUI() { } } -#ifdef USE_PROFILER if ((DebugOverlay)g_Config.iDebugOverlay == DebugOverlay::FRAME_PROFILE && PSP_IsInited()) { DrawProfile(*ctx); } -#endif if (g_Config.bShowGPOLEDs) { // Draw a vertical strip of LEDs at the right side of the screen. diff --git a/UI/ImDebugger/ImProfiler.cpp b/UI/ImDebugger/ImProfiler.cpp index e621ec51a1..77b3f2486c 100644 --- a/UI/ImDebugger/ImProfiler.cpp +++ b/UI/ImDebugger/ImProfiler.cpp @@ -72,12 +72,21 @@ struct TimelineView { }; TimelineView g_view; + +// The first time a profiler window shows. After that, its Record checkbox decides. +void TurnOnOnce() { + static bool done = false; + if (!done) { + Profiler_SetEnabled(true); + done = true; + } +} std::vector g_events; void DrawTimeline() { TimelineView &view = g_view; const uint64_t now = Profiler_Now(); - if (view.follow && !Profiler_IsPaused()) { + if (view.follow && Profiler_IsEnabled()) { view.end = now; } const int64_t span = std::max(Profiler_SecondsToTicks(view.spanSeconds), 1); @@ -118,7 +127,7 @@ void DrawTimeline() { const double newSpan = std::clamp(view.spanSeconds * pow(0.85, io.MouseWheel), 0.0001, 2.0); const int64_t newSpanTicks = Profiler_SecondsToTicks(newSpan); view.spanSeconds = newSpan; - if (!view.follow || Profiler_IsPaused()) { + if (!view.follow || !Profiler_IsEnabled()) { view.end = anchor + (uint64_t)((1.0 - frac) * newSpanTicks); } } @@ -142,9 +151,11 @@ void DrawTimeline() { struct Open { uint64_t start; + const char *label; bool valid; }; const char *hoveredName = nullptr; + const char *hoveredLabel = nullptr; double hoveredMs = 0.0; float y = origin.y + rowH; for (int t = 0; t < numThreads; t++) { @@ -157,7 +168,7 @@ void DrawTimeline() { Open open[32]{}; int maxLevel = 0; - auto drawBar = [&](int level, int category, uint64_t from, uint64_t to) { + auto drawBar = [&](int level, int category, uint64_t from, uint64_t to, const char *label) { maxLevel = std::max(maxLevel, level); if (level >= view.laneDepth[t]) { return; @@ -175,15 +186,16 @@ void DrawTimeline() { } if (hovered && io.MousePos.x >= a.x && io.MousePos.x < b.x && io.MousePos.y >= a.y && io.MousePos.y < b.y) { hoveredName = name; + hoveredLabel = label; hoveredMs = Profiler_TicksToSeconds((int64_t)(to - from)) * 1000.0; } }; for (const ProfilerEvent &e : g_events) { if (e.type == ProfilerEventType::Begin) { - open[e.level] = Open{ e.ticks, true }; + open[e.level] = Open{ e.ticks, e.label, true }; } else { // Without its Begin, the scope started before the view. - drawBar(e.level, e.category, open[e.level].valid ? open[e.level].start : viewStart, e.ticks); + drawBar(e.level, e.category, open[e.level].valid ? open[e.level].start : viewStart, e.ticks, open[e.level].valid ? open[e.level].label : nullptr); open[e.level].valid = false; } } @@ -197,7 +209,7 @@ void DrawTimeline() { break; } } - drawBar(level, category, open[level].start, std::min(viewEnd, now)); + drawBar(level, category, open[level].start, std::min(viewEnd, now), open[level].label); } } if (maxLevel + 1 > view.laneDepth[t]) { @@ -218,7 +230,7 @@ void DrawTimeline() { dl->PopClipRect(); if (hoveredName) { - ImGui::SetTooltip("%s\n%0.3f ms", hoveredName, hoveredMs); + ImGui::SetTooltip("%s%s%s\n%0.3f ms", hoveredName, hoveredLabel ? ": " : "", hoveredLabel ? hoveredLabel : "", hoveredMs); } } @@ -283,6 +295,7 @@ void DrawFrameProfile(ImConfig &cfg) { return; } #ifdef USE_PROFILER + TurnOnOnce(); const int numCategories = Profiler_GetNumCategories(); const int numThreads = Profiler_GetNumThreads(); const int historyLength = Profiler_GetHistoryLength(); @@ -391,9 +404,10 @@ void DrawCpuProfiler(ImConfig &cfg) { return; } #ifdef USE_PROFILER - bool paused = Profiler_IsPaused(); - if (ImGui::Checkbox("Pause", &paused)) { - Profiler_SetPaused(paused); + TurnOnOnce(); + bool record = Profiler_IsEnabled(); + if (ImGui::Checkbox("Record", &record)) { + Profiler_SetEnabled(record); } ImGui::SameLine(); ImGui::Checkbox("Follow", &g_view.follow); diff --git a/UI/ProfilerDraw.cpp b/UI/ProfilerDraw.cpp index 52efda7379..85b9006e17 100644 --- a/UI/ProfilerDraw.cpp +++ b/UI/ProfilerDraw.cpp @@ -55,6 +55,7 @@ enum ProfileCatStatus { void DrawProfile(UIContext &ui) { #ifdef USE_PROFILER + Profiler_SetEnabled(true); PROFILE_THIS_SCOPE("timing"); int numCategories = Profiler_GetNumCategories(); int numThreads = Profiler_GetNumThreads(); diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 2f2bce04d0..2d05543d8e 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -157,6 +157,13 @@ android { } else { println("WARNING: RELEASE_STORE_FILE is missing. Release builds will be unusable.") } + // Debug and optimized builds have the frame profiler compiled in (off until turned on). + externalNativeBuild { + cmake { + cFlags.add("-DNO_PROFILER") + cppFlags.add("-DNO_PROFILER") + } + } } } externalNativeBuild { diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp index 5398fae595..b7005db92a 100644 --- a/unittest/UnitTest.cpp +++ b/unittest/UnitTest.cpp @@ -1917,15 +1917,22 @@ bool TestSymbolMap() { // than there are lanes) while frames are being totalled. Only does anything with USE_PROFILER. bool TestProfiler() { #ifdef USE_PROFILER + // Busy, not sleeping: sleeps have a scope of their own. + auto spin = [](int ms) { + const double end = time_now_d() + ms * 0.001; + while (time_now_d() < end) { + } + }; PROFILE_INIT(); + Profiler_SetEnabled(true); PROFILE_END_FRAME(); { PROFILE_THIS_SCOPE("test_outer"); - sleep_ms(20, "test"); + spin(20); { PROFILE_THIS_SCOPE("test_inner"); - sleep_ms(30, "test"); + spin(30); } } PROFILE_END_FRAME(); @@ -1952,7 +1959,7 @@ bool TestProfiler() { Profiler_GetHistory(outerCat, lane, outer.data(), len); Profiler_GetHistory(innerCat, lane, inner.data(), len); Profiler_GetHistoryCounts(innerCat, lane, counts.data(), len); - // Outer gets only its own time, not the inner scope's. Sleeps run long, never short. + // Outer gets only its own time, not the inner scope's. EXPECT_TRUE(outer[len - 1] >= 0.019f && outer[len - 1] < 0.035f); EXPECT_TRUE(inner[len - 1] >= 0.029f && inner[len - 1] < 0.045f); EXPECT_EQ_INT(counts[len - 1], 1); @@ -1960,9 +1967,9 @@ bool TestProfiler() { // A scope open across a frame end is split between the frames. { PROFILE_THIS_SCOPE("test_outer"); - sleep_ms(10, "test"); + spin(10); PROFILE_END_FRAME(); - sleep_ms(10, "test"); + spin(10); } PROFILE_END_FRAME(); Profiler_GetHistory(outerCat, lane, outer.data(), len); @@ -2003,6 +2010,7 @@ bool TestProfiler() { stop = true; reader.join(); EXPECT_TRUE(Profiler_GetNumThreads() <= 32); + Profiler_SetEnabled(false); #endif return true; }