From 0fa7349f5aa38c9ad2a57b06f8426663f6c9f46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 30 May 2025 21:54:16 +0200 Subject: [PATCH 01/10] Integrate Dolphin's granule based audio resampler. Removed parts of it that were not relevant. Working, it seems. Not sure about the buffer size thing. Not defaulting it for now See #20146 and https://github.com/dolphin-emu/dolphin/pull/13352 .. --- CMakeLists.txt | 2 + Core/Config.cpp | 3 + Core/Config.h | 3 + Core/ConfigValues.h | 5 + Core/Core.vcxproj | 2 + Core/Core.vcxproj.filters | 8 +- Core/HW/GranularMixer.cpp | 285 ++++++++++++++++++++++++++++ Core/HW/GranularMixer.h | 108 +++++++++++ Core/HW/StereoResampler.cpp | 2 +- GPU/Common/PresentationCommon.cpp | 2 + UI/AudioCommon.cpp | 41 +++- UI/GameSettingsScreen.cpp | 2 + UWP/CoreUWP/CoreUWP.vcxproj | 2 + UWP/CoreUWP/CoreUWP.vcxproj.filters | 7 +- android/jni/Android.mk | 1 + ios/iOSCoreAudio.mm | 2 +- 16 files changed, 466 insertions(+), 9 deletions(-) create mode 100644 Core/HW/GranularMixer.cpp create mode 100644 Core/HW/GranularMixer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e638c47aab..5f25235bb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2358,6 +2358,8 @@ add_library(${CoreLibName} ${CoreLinkType} Core/HW/Camera.h Core/HW/Display.cpp Core/HW/Display.h + Core/HW/GranularMixer.cpp + Core/HW/GranularMixer.h Core/HW/MediaEngine.cpp Core/HW/MediaEngine.h Core/HW/MpegDemux.cpp diff --git a/Core/Config.cpp b/Core/Config.cpp index 6c9c1830e1..511e2fb370 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -784,6 +784,9 @@ static const ConfigSetting soundSettings[] = { ConfigSetting("ExtraAudioBuffering", &g_Config.bExtraAudioBuffering, false, CfgFlag::DEFAULT), ConfigSetting("AudioBufferSize", &g_Config.iSDLAudioBufferSize, 256, CfgFlag::DEFAULT), + ConfigSetting("FillAudioGaps", &g_Config.bFillAudioGaps, true, CfgFlag::DEFAULT), + ConfigSetting("AudioSyncMode", &g_Config.iAudioSyncMode, (int)AudioSyncMode::CLASSIC_PITCH, CfgFlag::DEFAULT), + // Legacy volume settings, these get auto upgraded through default handlers on the new settings. NOTE: Must be before the new ones in the order here. // The default settings here are still relevant, they will get propagated into the new ones. ConfigSetting("GlobalVolume", &g_Config.iLegacyGameVolume, VOLUME_FULL, CfgFlag::PER_GAME | CfgFlag::DONT_SAVE), diff --git a/Core/Config.h b/Core/Config.h index 1852e6c9bc..efd69c9288 100644 --- a/Core/Config.h +++ b/Core/Config.h @@ -289,6 +289,9 @@ public: // Sound bool bEnableSound; int iSDLAudioBufferSize; + int iAudioBufferSize; + bool bFillAudioGaps; + int iAudioSyncMode; // Legacy volume settings, 0-10. These get auto-upgraded and should not be used. int iLegacyGameVolume; diff --git a/Core/ConfigValues.h b/Core/ConfigValues.h index ce6f30c369..efea63c002 100644 --- a/Core/ConfigValues.h +++ b/Core/ConfigValues.h @@ -117,6 +117,11 @@ enum class DepthRasterMode { FORCE_ON = 3, }; +enum class AudioSyncMode { + GRANULAR = 0, + CLASSIC_PITCH = 1, +}; + enum class RestoreSettingsBits : int { SETTINGS = 1, CONTROLS = 2, diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index 41b0645c51..abe294c4a8 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -491,6 +491,7 @@ + @@ -1050,6 +1051,7 @@ + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index 4dfaa70ccb..b53609f8b3 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -1390,6 +1390,9 @@ Ext + + HW + @@ -2244,6 +2247,9 @@ Debugger + + HW + @@ -2272,4 +2278,4 @@ Ext - \ No newline at end of file + diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp new file mode 100644 index 0000000000..aa3a16fe57 --- /dev/null +++ b/Core/HW/GranularMixer.cpp @@ -0,0 +1,285 @@ +// Copyright 2008 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "Core/HW/GranularMixer.h" + +#include +#include +#include +#include + +#include "Common/CommonTypes.h" +#include "Common/Log.h" +#include "Common/Math/math_util.h" +#include "Common/Swap.h" +#include "Core/Core.h" +#include "Core/System.h" +#include "Core/Util/AudioFormat.h" // for clamp_u16 + +using Clock = std::chrono::steady_clock; +using TimePoint = Clock::time_point; +using DT = Clock::duration; +using DT_us = std::chrono::duration; +using DT_ms = std::chrono::duration; +using DT_s = std::chrono::duration>; + +GranularMixer::GranularMixer() { + RefreshConfig(); + + INFO_LOG(Log::Audio, "Mixer is initialized"); +} + +// Executed from sound stream thread +void GranularMixer::MixerFifo::Mix(s16* samples, std::size_t num_samples, int outSampleRate) +{ + constexpr u32 INDEX_HALF = 0x80000000; + constexpr DT_s FADE_IN_RC = DT_s(0.008); + constexpr DT_s FADE_OUT_RC = DT_s(0.064); + + // We need at least a double because the index jump has 24 bits of fractional precision. + const double out_sample_rate = outSampleRate; + double in_sample_rate = 44100; + + const double emulation_speed = 1.0f; // TODO: Change when we're in slow-motion mode etc. + if (0 < emulation_speed && emulation_speed != 1.0) + in_sample_rate *= emulation_speed; + + const double base = static_cast(1 << GRANULE_FRAC_BITS); + const u32 index_jump = std::lround(base * in_sample_rate / out_sample_rate); + + // These fade in / out multiplier are tuned to match a constant + // fade speed regardless of the input or the output sample rate. + const float fade_in_mul = -std::expm1(-DT_s(1.0) / (out_sample_rate * FADE_IN_RC)); + const float fade_out_mul = -std::expm1(-DT_s(1.0) / (out_sample_rate * FADE_OUT_RC)); + + // Calculate the ideal length of the granule queue. + const std::size_t buffer_size_ms = 40; // TODO: Actually take from the audio backend. + const std::size_t buffer_size_samples = std::llround(buffer_size_ms * in_sample_rate / 1000.0); + + // Limit the possible queue sizes to any number between 4 and 64. + const std::size_t buffer_size_granules = + std::clamp((buffer_size_samples) / (GRANULE_SIZE >> 1), static_cast(4), + static_cast(MAX_GRANULE_QUEUE_SIZE)); + + m_granule_queue_size.store(buffer_size_granules, std::memory_order_relaxed); + + while (num_samples-- > 0) + { + // The indexes for the front and back buffers are offset by 50% of the granule size. + // We use the modular nature of 32-bit integers to wrap around the granule size. + m_current_index += index_jump; + const u32 front_index = m_current_index; + const u32 back_index = m_current_index + INDEX_HALF; + + // If either index is less than the index jump, that means we reached + // the end of the of the buffer and need to load the next granule. + if (front_index < index_jump) + Dequeue(&m_front); + else if (back_index < index_jump) + Dequeue(&m_back); + + // The Granules are pre-windowed, so we can just add them together + const std::size_t ft = front_index >> GRANULE_FRAC_BITS; + const std::size_t bt = back_index >> GRANULE_FRAC_BITS; + const StereoPair s0 = m_front[(ft - 2) & GRANULE_MASK] + m_back[(bt - 2) & GRANULE_MASK]; + const StereoPair s1 = m_front[(ft - 1) & GRANULE_MASK] + m_back[(bt - 1) & GRANULE_MASK]; + const StereoPair s2 = m_front[(ft + 0) & GRANULE_MASK] + m_back[(bt + 0) & GRANULE_MASK]; + const StereoPair s3 = m_front[(ft + 1) & GRANULE_MASK] + m_back[(bt + 1) & GRANULE_MASK]; + const StereoPair s4 = m_front[(ft + 2) & GRANULE_MASK] + m_back[(bt + 2) & GRANULE_MASK]; + const StereoPair s5 = m_front[(ft + 3) & GRANULE_MASK] + m_back[(bt + 3) & GRANULE_MASK]; + + // Polynomial Interpolators for High-Quality Resampling of + // Over Sampled Audio by Olli Niemitalo, October 2001. + // Page 43 -- 6-point, 3rd-order Hermite: + // https://yehar.com/blog/wp-content/uploads/2009/08/deip.pdf + const u32 t_frac = m_current_index & ((1 << GRANULE_FRAC_BITS) - 1); + const float t1 = t_frac / static_cast(1 << GRANULE_FRAC_BITS); + const float t2 = t1 * t1; + const float t3 = t2 * t1; + + StereoPair sample = (s0 * StereoPair{ (+0.0f + 1.0f * t1 - 2.0f * t2 + 1.0f * t3) / 12.0f } + + s1 * StereoPair{ (+0.0f - 8.0f * t1 + 15.0f * t2 - 7.0f * t3) / 12.0f } + + s2 * StereoPair{ (+3.0f + 0.0f * t1 - 7.0f * t2 + 4.0f * t3) / 3.0f } + + s3 * StereoPair{ (+0.0f + 2.0f * t1 + 5.0f * t2 - 4.0f * t3) / 3.0f } + + s4 * StereoPair{ (+0.0f - 1.0f * t1 - 6.0f * t2 + 7.0f * t3) / 12.0f } + + s5 * StereoPair{ (+0.0f + 0.0f * t1 + 1.0f * t2 - 1.0f * t3) / 12.0f }); + + // Apply Fade In / Fade Out depending on if we are looping + if (m_queue_looping.load(std::memory_order_relaxed)) + m_fade_volume += fade_out_mul * (0.0f - m_fade_volume); + else + m_fade_volume += fade_in_mul * (1.0f - m_fade_volume); + + // Apply the fade volume to the sample + sample = sample * StereoPair{ m_fade_volume }; + + // This quantization method prevents accumulated error but does not do noise shaping. + sample.l += samples[0] - m_quantization_error.l; + samples[0] = (int16_t)clamp_value(sample.l, -32767.0f, 32767.0f); + m_quantization_error.l = clamp_value(samples[0] - sample.l, -1.0f, 1.0f); + + sample.r += samples[1] - m_quantization_error.r; + samples[1] = (int16_t)clamp_value(sample.r, -32767.0f, 32767.0f); + m_quantization_error.r = std::clamp(samples[1] - sample.r, -1.0f, 1.0f); + + samples += 2; + } +} + +std::size_t GranularMixer::Mix(s16* samples, std::size_t num_samples, int outSampleRate) +{ + if (!samples) + return 0; + memset(samples, 0, num_samples * 2 * sizeof(s16)); + m_dma_mixer.Mix(samples, num_samples, outSampleRate); + return num_samples; +} + +inline s16 clampfloat_s16(float f) { + if (f <= -32767.0f) return -32767; + if (f >= 32767.0f) return 32767; + return (s16)f; +} + +void GranularMixer::MixerFifo::PushSamples(const s32 *samples, std::size_t num_samples, float volume) +{ + // TODO: This can be massively sped up. Although hardly likely to be a bottleneck. + while (num_samples-- > 0) + { + const s16 l = clampfloat_s16(samples[0] * volume); + const s16 r = clampfloat_s16(samples[1] * volume); + samples += 2; + + m_next_buffer[m_next_buffer_index] = StereoPair(l, r); + m_next_buffer_index = (m_next_buffer_index + 1) & GRANULE_MASK; + + // The granules overlap by 50%, so we need to enqueue the + // next buffer every time we fill half of the samples. + if (m_next_buffer_index == 0 || m_next_buffer_index == m_next_buffer.size() / 2) + Enqueue(); + } +} + +void GranularMixer::PushSamples(const s32 *samples, std::size_t num_samples, float volume) +{ + m_dma_mixer.PushSamples(samples, num_samples, volume); +} + +void GranularMixer::RefreshConfig() { + + // m_config_audio_buffer_ms = Config::Get(Config::MAIN_AUDIO_BUFFER_SIZE); +} + +void GranularMixer::MixerFifo::Enqueue() +{ + // import numpy as np + // import scipy.signal as signal + // window = np.convolve(np.ones(128), signal.windows.dpss(128 + 1, 4)) + // window /= (window[:len(window) // 2] + window[len(window) // 2:]).max() + // elements = ", ".join([f"{x:.10f}f" for x in window]) + // print(f'constexpr std::array GRANULE_WINDOW = {{ {elements} + // }};') + constexpr std::array GRANULE_WINDOW = { + 0.0000016272f, 0.0000050749f, 0.0000113187f, 0.0000216492f, 0.0000377350f, 0.0000616906f, + 0.0000961509f, 0.0001443499f, 0.0002102045f, 0.0002984010f, 0.0004144844f, 0.0005649486f, + 0.0007573262f, 0.0010002765f, 0.0013036694f, 0.0016786636f, 0.0021377783f, 0.0026949534f, + 0.0033656000f, 0.0041666352f, 0.0051165029f, 0.0062351752f, 0.0075441359f, 0.0090663409f, + 0.0108261579f, 0.0128492811f, 0.0151626215f, 0.0177941726f, 0.0207728499f, 0.0241283062f, + 0.0278907219f, 0.0320905724f, 0.0367583739f, 0.0419244083f, 0.0476184323f, 0.0538693708f, + 0.0607049996f, 0.0681516192f, 0.0762337261f, 0.0849736833f, 0.0943913952f, 0.1045039915f, + 0.1153255250f, 0.1268666867f, 0.1391345431f, 0.1521323012f, 0.1658591025f, 0.1803098534f, + 0.1954750915f, 0.2113408944f, 0.2278888303f, 0.2450959552f, 0.2629348550f, 0.2813737361f, + 0.3003765625f, 0.3199032396f, 0.3399098438f, 0.3603488941f, 0.3811696664f, 0.4023185434f, + 0.4237393998f, 0.4453740162f, 0.4671625177f, 0.4890438330f, 0.5109561670f, 0.5328374823f, + 0.5546259838f, 0.5762606002f, 0.5976814566f, 0.6188303336f, 0.6396511059f, 0.6600901562f, + 0.6800967604f, 0.6996234375f, 0.7186262639f, 0.7370651450f, 0.7549040448f, 0.7721111697f, + 0.7886591056f, 0.8045249085f, 0.8196901466f, 0.8341408975f, 0.8478676988f, 0.8608654569f, + 0.8731333133f, 0.8846744750f, 0.8954960085f, 0.9056086048f, 0.9150263167f, 0.9237662739f, + 0.9318483808f, 0.9392950004f, 0.9461306292f, 0.9523815677f, 0.9580755917f, 0.9632416261f, + 0.9679094276f, 0.9721092781f, 0.9758716938f, 0.9792271501f, 0.9822058274f, 0.9848373785f, + 0.9871507189f, 0.9891738421f, 0.9909336591f, 0.9924558641f, 0.9937648248f, 0.9948834971f, + 0.9958333648f, 0.9966344000f, 0.9973050466f, 0.9978622217f, 0.9983213364f, 0.9986963306f, + 0.9989997235f, 0.9992426738f, 0.9994350514f, 0.9995855156f, 0.9997015990f, 0.9997897955f, + 0.9998556501f, 0.9999038491f, 0.9999383094f, 0.9999622650f, 0.9999783508f, 0.9999886813f, + 0.9999949251f, 0.9999983728f, 0.9999983728f, 0.9999949251f, 0.9999886813f, 0.9999783508f, + 0.9999622650f, 0.9999383094f, 0.9999038491f, 0.9998556501f, 0.9997897955f, 0.9997015990f, + 0.9995855156f, 0.9994350514f, 0.9992426738f, 0.9989997235f, 0.9986963306f, 0.9983213364f, + 0.9978622217f, 0.9973050466f, 0.9966344000f, 0.9958333648f, 0.9948834971f, 0.9937648248f, + 0.9924558641f, 0.9909336591f, 0.9891738421f, 0.9871507189f, 0.9848373785f, 0.9822058274f, + 0.9792271501f, 0.9758716938f, 0.9721092781f, 0.9679094276f, 0.9632416261f, 0.9580755917f, + 0.9523815677f, 0.9461306292f, 0.9392950004f, 0.9318483808f, 0.9237662739f, 0.9150263167f, + 0.9056086048f, 0.8954960085f, 0.8846744750f, 0.8731333133f, 0.8608654569f, 0.8478676988f, + 0.8341408975f, 0.8196901466f, 0.8045249085f, 0.7886591056f, 0.7721111697f, 0.7549040448f, + 0.7370651450f, 0.7186262639f, 0.6996234375f, 0.6800967604f, 0.6600901562f, 0.6396511059f, + 0.6188303336f, 0.5976814566f, 0.5762606002f, 0.5546259838f, 0.5328374823f, 0.5109561670f, + 0.4890438330f, 0.4671625177f, 0.4453740162f, 0.4237393998f, 0.4023185434f, 0.3811696664f, + 0.3603488941f, 0.3399098438f, 0.3199032396f, 0.3003765625f, 0.2813737361f, 0.2629348550f, + 0.2450959552f, 0.2278888303f, 0.2113408944f, 0.1954750915f, 0.1803098534f, 0.1658591025f, + 0.1521323012f, 0.1391345431f, 0.1268666867f, 0.1153255250f, 0.1045039915f, 0.0943913952f, + 0.0849736833f, 0.0762337261f, 0.0681516192f, 0.0607049996f, 0.0538693708f, 0.0476184323f, + 0.0419244083f, 0.0367583739f, 0.0320905724f, 0.0278907219f, 0.0241283062f, 0.0207728499f, + 0.0177941726f, 0.0151626215f, 0.0128492811f, 0.0108261579f, 0.0090663409f, 0.0075441359f, + 0.0062351752f, 0.0051165029f, 0.0041666352f, 0.0033656000f, 0.0026949534f, 0.0021377783f, + 0.0016786636f, 0.0013036694f, 0.0010002765f, 0.0007573262f, 0.0005649486f, 0.0004144844f, + 0.0002984010f, 0.0002102045f, 0.0001443499f, 0.0000961509f, 0.0000616906f, 0.0000377350f, + 0.0000216492f, 0.0000113187f, 0.0000050749f, 0.0000016272f }; + + const std::size_t head = m_queue_head.load(std::memory_order_acquire); + + // Check if we run out of space in the circular queue. (rare) + std::size_t next_head = (head + 1) & GRANULE_QUEUE_MASK; + if (next_head == m_queue_tail.load(std::memory_order_acquire)) + { + WARN_LOG(Log::Audio, + "Granule Queue has completely filled and audio samples are being dropped. " + "This should not happen unless the audio backend has stopped requesting audio."); + return; + } + + // By preconstructing the granule window, we have the best chance of + // the compiler optimizing this loop using SIMD instructions. + const std::size_t start_index = m_next_buffer_index; + for (std::size_t i = 0; i < GRANULE_SIZE; ++i) + m_queue[head][i] = m_next_buffer[(i + start_index) & GRANULE_MASK] * GRANULE_WINDOW[i]; + + m_queue_head.store(next_head, std::memory_order_release); + m_queue_looping.store(false, std::memory_order_relaxed); +} + +void GranularMixer::MixerFifo::Dequeue(Granule* granule) +{ + const std::size_t granule_queue_size = m_granule_queue_size.load(std::memory_order_relaxed); + const std::size_t head = m_queue_head.load(std::memory_order_acquire); + std::size_t tail = m_queue_tail.load(std::memory_order_acquire); + + // Checks to see if the queue has gotten too long. + if (granule_queue_size < ((head - tail) & GRANULE_QUEUE_MASK)) + { + // Jump the playhead to half the queue size behind the head. + const std::size_t gap = (granule_queue_size >> 1) + 1; + tail = (head - gap) & GRANULE_QUEUE_MASK; + } + + // Checks to see if the queue is empty. + std::size_t next_tail = (tail + 1) & GRANULE_QUEUE_MASK; + if (next_tail == head) + { + // Only fill gaps when running to prevent stutter on pause. + CoreState state = coreState; + const bool is_running = state == CORE_RUNNING_CPU || state == CORE_RUNNING_GE; + if (g_Config.bFillAudioGaps && is_running) { + // Jump the playhead to half the queue size behind the head. + // This provides smoother audio playback than suddenly stopping. + const std::size_t gap = std::max(2, granule_queue_size >> 1) - 1; + next_tail = (head - gap) & GRANULE_QUEUE_MASK; + m_queue_looping.store(true, std::memory_order_relaxed); + } else { + std::fill(granule->begin(), granule->end(), StereoPair{ 0.0f, 0.0f }); + m_queue_looping.store(false, std::memory_order_relaxed); + return; + } + } + + *granule = m_queue[tail]; + m_queue_tail.store(next_tail, std::memory_order_release); +} diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h new file mode 100644 index 0000000000..7fe308b5eb --- /dev/null +++ b/Core/HW/GranularMixer.h @@ -0,0 +1,108 @@ +// Copyright 2009 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include + +#include "Common/CommonTypes.h" +#include "Core/Config.h" + +class PointerWrap; + +// Replacement for std::countr_one +constexpr std::size_t countr_one_replacement(std::size_t x) { + std::size_t count = 0; + while (x & 1) { + ++count; + x >>= 1; + } + return count; +} + +class GranularMixer final { +public: + explicit GranularMixer(); + + // Called from audio threads + std::size_t Mix(s16* samples, std::size_t numSamples, int outSampleRate); + + // Called from emulation thread + void PushSamples(const s32* samples, std::size_t num_samples, float volume); + +private: + const std::size_t SURROUND_CHANNELS = 6; + + class MixerFifo final { + static constexpr std::size_t MAX_GRANULE_QUEUE_SIZE = 256; + static constexpr std::size_t GRANULE_QUEUE_MASK = MAX_GRANULE_QUEUE_SIZE - 1; + + struct StereoPair final { + float l = 0.f; + float r = 0.f; + + constexpr StereoPair() = default; + constexpr StereoPair(const StereoPair&) = default; + constexpr StereoPair& operator=(const StereoPair&) = default; + constexpr StereoPair(StereoPair&&) = default; + constexpr StereoPair& operator=(StereoPair&&) = default; + + constexpr StereoPair(float mono) : l(mono), r(mono) {} + constexpr StereoPair(float left, float right) : l(left), r(right) {} + constexpr StereoPair(s16 left, s16 right) : l(left), r(right) {} + + StereoPair operator+(const StereoPair& other) const + { + return StereoPair(l + other.l, r + other.r); + } + + StereoPair operator*(const StereoPair& other) const + { + return StereoPair(l * other.l, r * other.r); + } + }; + + static constexpr std::size_t GRANULE_SIZE = 256; + static constexpr std::size_t GRANULE_OVERLAP = GRANULE_SIZE / 2; + static constexpr std::size_t GRANULE_MASK = GRANULE_SIZE - 1; + static constexpr std::size_t GRANULE_BITS = countr_one_replacement(GRANULE_MASK); + static constexpr std::size_t GRANULE_FRAC_BITS = 32 - GRANULE_BITS; + + using Granule = std::array; + + public: + MixerFifo(GranularMixer* mixer, bool little_endian) : m_mixer(mixer) {} + void PushSamples(const s32* samples, std::size_t num_samples, float volume); + void Mix(s16* samples, std::size_t num_samples, int outSampleRate); + + private: + GranularMixer* m_mixer; + + Granule m_next_buffer{}; + std::size_t m_next_buffer_index = 0; + + u32 m_current_index = 0; + Granule m_front, m_back; + + std::atomic m_granule_queue_size{ 20 }; + std::array m_queue; + std::atomic m_queue_head{ 0 }; + std::atomic m_queue_tail{ 0 }; + std::atomic m_queue_looping{ false }; + float m_fade_volume = 1.0; + + void Enqueue(); + void Dequeue(Granule* granule); + + StereoPair m_quantization_error; + }; + + void RefreshConfig(); + + MixerFifo m_dma_mixer{ this, false }; + + //int m_config_audio_buffer_ms; +}; diff --git a/Core/HW/StereoResampler.cpp b/Core/HW/StereoResampler.cpp index f5e4be4054..ec44ae1971 100644 --- a/Core/HW/StereoResampler.cpp +++ b/Core/HW/StereoResampler.cpp @@ -46,7 +46,7 @@ #include "Core/Config.h" #include "Core/ConfigValues.h" #include "Core/HW/StereoResampler.h" -#include "Core/Util/AudioFormat.h" // for clamp_u8 +#include "Core/Util/AudioFormat.h" // for clamp_u16 #include "Core/System.h" StereoResampler::StereoResampler() noexcept diff --git a/GPU/Common/PresentationCommon.cpp b/GPU/Common/PresentationCommon.cpp index af8316b685..a5d72daff9 100644 --- a/GPU/Common/PresentationCommon.cpp +++ b/GPU/Common/PresentationCommon.cpp @@ -528,6 +528,8 @@ Draw::Pipeline *PresentationCommon::CreatePipeline(std::vectorSetEnabledPtr(&g_Config.bEnableSound); #endif + audioSettings->Add(new ItemHeader(a->T("Audio"))); + audioSettings->Add(new CheckBox(&g_Config.bFillAudioGaps, a->T("Fill audio gaps"))); audioSettings->Add(new ItemHeader(a->T("Game volume"))); diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index 2a4284a601..8254121cf3 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -161,6 +161,7 @@ + @@ -429,6 +430,7 @@ + diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index ccb9d16891..27029b01ad 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -165,6 +165,7 @@ + @@ -679,8 +680,12 @@ +<<<<<<< HEAD +======= + +>>>>>>> cfb162d436 (Integrate Dolphin's granule based audio resampler.) - \ No newline at end of file + diff --git a/android/jni/Android.mk b/android/jni/Android.mk index 8cb25082ba..55f67d7d15 100644 --- a/android/jni/Android.mk +++ b/android/jni/Android.mk @@ -585,6 +585,7 @@ EXEC_AND_LIB_FILES := \ $(SRC)/Core/HW/SasAudio.cpp.arm \ $(SRC)/Core/HW/SasReverb.cpp.arm \ $(SRC)/Core/HW/StereoResampler.cpp.arm \ + $(SRC)/Core/HW/GranularMixer.cpp.arm \ $(SRC)/Core/ControlMapper.cpp \ $(SRC)/Core/Core.cpp \ $(SRC)/Core/Compatibility.cpp \ diff --git a/ios/iOSCoreAudio.mm b/ios/iOSCoreAudio.mm index 3216ec0635..e8f7619583 100644 --- a/ios/iOSCoreAudio.mm +++ b/ios/iOSCoreAudio.mm @@ -74,7 +74,7 @@ void iOSCoreAudioSetDisplayConnected(bool connected) { iOSCoreAudioUpdateSession(); } -void NativeMix(short *audio, int numSamples, int sampleRate, void *userdata); +void NativeMix(short *audio, int numSamples, int sampleRateHz, void *userdata); OSStatus iOSCoreAudioCallback(void *inRefCon, AudioUnitRenderActionFlags *ioActionFlags, From 878975ad804cb85c9e91dbc60f6d0682455f9ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 13 Jun 2025 15:45:25 +0200 Subject: [PATCH 02/10] Rework granule mixer code style a little, optimize. --- Core/HW/GranularMixer.cpp | 186 ++++++++++++++++++-------------------- Core/HW/GranularMixer.h | 38 ++++---- 2 files changed, 103 insertions(+), 121 deletions(-) diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp index aa3a16fe57..4760452212 100644 --- a/Core/HW/GranularMixer.cpp +++ b/Core/HW/GranularMixer.cpp @@ -19,19 +19,67 @@ using Clock = std::chrono::steady_clock; using TimePoint = Clock::time_point; using DT = Clock::duration; -using DT_us = std::chrono::duration; -using DT_ms = std::chrono::duration; using DT_s = std::chrono::duration>; -GranularMixer::GranularMixer() { - RefreshConfig(); +// Something like a gaussian. +static const float g_GranuleWindow[256] = { + 0.0000016272f, 0.0000050749f, 0.0000113187f, 0.0000216492f, 0.0000377350f, 0.0000616906f, + 0.0000961509f, 0.0001443499f, 0.0002102045f, 0.0002984010f, 0.0004144844f, 0.0005649486f, + 0.0007573262f, 0.0010002765f, 0.0013036694f, 0.0016786636f, 0.0021377783f, 0.0026949534f, + 0.0033656000f, 0.0041666352f, 0.0051165029f, 0.0062351752f, 0.0075441359f, 0.0090663409f, + 0.0108261579f, 0.0128492811f, 0.0151626215f, 0.0177941726f, 0.0207728499f, 0.0241283062f, + 0.0278907219f, 0.0320905724f, 0.0367583739f, 0.0419244083f, 0.0476184323f, 0.0538693708f, + 0.0607049996f, 0.0681516192f, 0.0762337261f, 0.0849736833f, 0.0943913952f, 0.1045039915f, + 0.1153255250f, 0.1268666867f, 0.1391345431f, 0.1521323012f, 0.1658591025f, 0.1803098534f, + 0.1954750915f, 0.2113408944f, 0.2278888303f, 0.2450959552f, 0.2629348550f, 0.2813737361f, + 0.3003765625f, 0.3199032396f, 0.3399098438f, 0.3603488941f, 0.3811696664f, 0.4023185434f, + 0.4237393998f, 0.4453740162f, 0.4671625177f, 0.4890438330f, 0.5109561670f, 0.5328374823f, + 0.5546259838f, 0.5762606002f, 0.5976814566f, 0.6188303336f, 0.6396511059f, 0.6600901562f, + 0.6800967604f, 0.6996234375f, 0.7186262639f, 0.7370651450f, 0.7549040448f, 0.7721111697f, + 0.7886591056f, 0.8045249085f, 0.8196901466f, 0.8341408975f, 0.8478676988f, 0.8608654569f, + 0.8731333133f, 0.8846744750f, 0.8954960085f, 0.9056086048f, 0.9150263167f, 0.9237662739f, + 0.9318483808f, 0.9392950004f, 0.9461306292f, 0.9523815677f, 0.9580755917f, 0.9632416261f, + 0.9679094276f, 0.9721092781f, 0.9758716938f, 0.9792271501f, 0.9822058274f, 0.9848373785f, + 0.9871507189f, 0.9891738421f, 0.9909336591f, 0.9924558641f, 0.9937648248f, 0.9948834971f, + 0.9958333648f, 0.9966344000f, 0.9973050466f, 0.9978622217f, 0.9983213364f, 0.9986963306f, + 0.9989997235f, 0.9992426738f, 0.9994350514f, 0.9995855156f, 0.9997015990f, 0.9997897955f, + 0.9998556501f, 0.9999038491f, 0.9999383094f, 0.9999622650f, 0.9999783508f, 0.9999886813f, + 0.9999949251f, 0.9999983728f, 0.9999983728f, 0.9999949251f, 0.9999886813f, 0.9999783508f, + 0.9999622650f, 0.9999383094f, 0.9999038491f, 0.9998556501f, 0.9997897955f, 0.9997015990f, + 0.9995855156f, 0.9994350514f, 0.9992426738f, 0.9989997235f, 0.9986963306f, 0.9983213364f, + 0.9978622217f, 0.9973050466f, 0.9966344000f, 0.9958333648f, 0.9948834971f, 0.9937648248f, + 0.9924558641f, 0.9909336591f, 0.9891738421f, 0.9871507189f, 0.9848373785f, 0.9822058274f, + 0.9792271501f, 0.9758716938f, 0.9721092781f, 0.9679094276f, 0.9632416261f, 0.9580755917f, + 0.9523815677f, 0.9461306292f, 0.9392950004f, 0.9318483808f, 0.9237662739f, 0.9150263167f, + 0.9056086048f, 0.8954960085f, 0.8846744750f, 0.8731333133f, 0.8608654569f, 0.8478676988f, + 0.8341408975f, 0.8196901466f, 0.8045249085f, 0.7886591056f, 0.7721111697f, 0.7549040448f, + 0.7370651450f, 0.7186262639f, 0.6996234375f, 0.6800967604f, 0.6600901562f, 0.6396511059f, + 0.6188303336f, 0.5976814566f, 0.5762606002f, 0.5546259838f, 0.5328374823f, 0.5109561670f, + 0.4890438330f, 0.4671625177f, 0.4453740162f, 0.4237393998f, 0.4023185434f, 0.3811696664f, + 0.3603488941f, 0.3399098438f, 0.3199032396f, 0.3003765625f, 0.2813737361f, 0.2629348550f, + 0.2450959552f, 0.2278888303f, 0.2113408944f, 0.1954750915f, 0.1803098534f, 0.1658591025f, + 0.1521323012f, 0.1391345431f, 0.1268666867f, 0.1153255250f, 0.1045039915f, 0.0943913952f, + 0.0849736833f, 0.0762337261f, 0.0681516192f, 0.0607049996f, 0.0538693708f, 0.0476184323f, + 0.0419244083f, 0.0367583739f, 0.0320905724f, 0.0278907219f, 0.0241283062f, 0.0207728499f, + 0.0177941726f, 0.0151626215f, 0.0128492811f, 0.0108261579f, 0.0090663409f, 0.0075441359f, + 0.0062351752f, 0.0051165029f, 0.0041666352f, 0.0033656000f, 0.0026949534f, 0.0021377783f, + 0.0016786636f, 0.0013036694f, 0.0010002765f, 0.0007573262f, 0.0005649486f, 0.0004144844f, + 0.0002984010f, 0.0002102045f, 0.0001443499f, 0.0000961509f, 0.0000616906f, 0.0000377350f, + 0.0000216492f, 0.0000113187f, 0.0000050749f, 0.0000016272f +}; +inline s16 clampfloat_s16(float f) { + if (f <= -32767.0f) return -32767; + if (f >= 32767.0f) return 32767; + return (s16)f; +} + +GranularMixer::GranularMixer() { INFO_LOG(Log::Audio, "Mixer is initialized"); } // Executed from sound stream thread -void GranularMixer::MixerFifo::Mix(s16* samples, std::size_t num_samples, int outSampleRate) -{ +void GranularMixer::MixerFifo::Mix(s16* samples, u32 num_samples, int outSampleRate) { constexpr u32 INDEX_HALF = 0x80000000; constexpr DT_s FADE_IN_RC = DT_s(0.008); constexpr DT_s FADE_OUT_RC = DT_s(0.064); @@ -57,14 +105,13 @@ void GranularMixer::MixerFifo::Mix(s16* samples, std::size_t num_samples, int ou const std::size_t buffer_size_samples = std::llround(buffer_size_ms * in_sample_rate / 1000.0); // Limit the possible queue sizes to any number between 4 and 64. - const std::size_t buffer_size_granules = + const u32 buffer_size_granules = std::clamp((buffer_size_samples) / (GRANULE_SIZE >> 1), static_cast(4), static_cast(MAX_GRANULE_QUEUE_SIZE)); m_granule_queue_size.store(buffer_size_granules, std::memory_order_relaxed); - while (num_samples-- > 0) - { + while (num_samples-- > 0) { // The indexes for the front and back buffers are offset by 50% of the granule size. // We use the modular nature of 32-bit integers to wrap around the granule size. m_current_index += index_jump; @@ -88,6 +135,7 @@ void GranularMixer::MixerFifo::Mix(s16* samples, std::size_t num_samples, int ou const StereoPair s4 = m_front[(ft + 2) & GRANULE_MASK] + m_back[(bt + 2) & GRANULE_MASK]; const StereoPair s5 = m_front[(ft + 3) & GRANULE_MASK] + m_back[(bt + 3) & GRANULE_MASK]; + // Probably an overkill interpolator, but let's go with it for now. // Polynomial Interpolators for High-Quality Resampling of // Over Sampled Audio by Olli Niemitalo, October 2001. // Page 43 -- 6-point, 3rd-order Hermite: @@ -96,13 +144,14 @@ void GranularMixer::MixerFifo::Mix(s16* samples, std::size_t num_samples, int ou const float t1 = t_frac / static_cast(1 << GRANULE_FRAC_BITS); const float t2 = t1 * t1; const float t3 = t2 * t1; - - StereoPair sample = (s0 * StereoPair{ (+0.0f + 1.0f * t1 - 2.0f * t2 + 1.0f * t3) / 12.0f } + - s1 * StereoPair{ (+0.0f - 8.0f * t1 + 15.0f * t2 - 7.0f * t3) / 12.0f } + - s2 * StereoPair{ (+3.0f + 0.0f * t1 - 7.0f * t2 + 4.0f * t3) / 3.0f } + - s3 * StereoPair{ (+0.0f + 2.0f * t1 + 5.0f * t2 - 4.0f * t3) / 3.0f } + - s4 * StereoPair{ (+0.0f - 1.0f * t1 - 6.0f * t2 + 7.0f * t3) / 12.0f } + - s5 * StereoPair{ (+0.0f + 0.0f * t1 + 1.0f * t2 - 1.0f * t3) / 12.0f }); + StereoPair sample = ( + s0 * ((+0.0f + 1.0f * t1 - 2.0f * t2 + 1.0f * t3) * (1.0f / 12.0f)) + + s1 * ((+0.0f - 8.0f * t1 + 15.0f * t2 - 7.0f * t3) * (1.0f / 12.0f)) + + s2 * ((+3.0f + 0.0f * t1 - 7.0f * t2 + 4.0f * t3) * (1.0f / 3.0f)) + + s3 * ((+0.0f + 2.0f * t1 + 5.0f * t2 - 4.0f * t3) * (1.0f / 3.0f)) + + s4 * ((+0.0f - 1.0f * t1 - 6.0f * t2 + 7.0f * t3) * (1.0f / 12.0f)) + + s5 * ((+0.0f + 0.0f * t1 + 1.0f * t2 - 1.0f * t3) * (1.0f / 12.0f)) + ); // Apply Fade In / Fade Out depending on if we are looping if (m_queue_looping.load(std::memory_order_relaxed)) @@ -111,23 +160,23 @@ void GranularMixer::MixerFifo::Mix(s16* samples, std::size_t num_samples, int ou m_fade_volume += fade_in_mul * (1.0f - m_fade_volume); // Apply the fade volume to the sample - sample = sample * StereoPair{ m_fade_volume }; + sample = sample * m_fade_volume; // This quantization method prevents accumulated error but does not do noise shaping. sample.l += samples[0] - m_quantization_error.l; - samples[0] = (int16_t)clamp_value(sample.l, -32767.0f, 32767.0f); - m_quantization_error.l = clamp_value(samples[0] - sample.l, -1.0f, 1.0f); - sample.r += samples[1] - m_quantization_error.r; + + samples[0] = (int16_t)clamp_value(sample.l, -32767.0f, 32767.0f); samples[1] = (int16_t)clamp_value(sample.r, -32767.0f, 32767.0f); - m_quantization_error.r = std::clamp(samples[1] - sample.r, -1.0f, 1.0f); + + m_quantization_error.l = clamp_value(samples[0] - sample.l, -1.0f, 1.0f); + m_quantization_error.r = clamp_value(samples[1] - sample.r, -1.0f, 1.0f); samples += 2; } } -std::size_t GranularMixer::Mix(s16* samples, std::size_t num_samples, int outSampleRate) -{ +std::size_t GranularMixer::Mix(s16* samples, u32 num_samples, int outSampleRate) { if (!samples) return 0; memset(samples, 0, num_samples * 2 * sizeof(s16)); @@ -135,17 +184,9 @@ std::size_t GranularMixer::Mix(s16* samples, std::size_t num_samples, int outSam return num_samples; } -inline s16 clampfloat_s16(float f) { - if (f <= -32767.0f) return -32767; - if (f >= 32767.0f) return 32767; - return (s16)f; -} - -void GranularMixer::MixerFifo::PushSamples(const s32 *samples, std::size_t num_samples, float volume) -{ +void GranularMixer::MixerFifo::PushSamples(const s32 *samples, u32 num_samples, float volume) { // TODO: This can be massively sped up. Although hardly likely to be a bottleneck. - while (num_samples-- > 0) - { + while (num_samples-- > 0) { const s16 l = clampfloat_s16(samples[0] * volume); const s16 r = clampfloat_s16(samples[1] * volume); samples += 2; @@ -155,23 +196,17 @@ void GranularMixer::MixerFifo::PushSamples(const s32 *samples, std::size_t num_s // The granules overlap by 50%, so we need to enqueue the // next buffer every time we fill half of the samples. - if (m_next_buffer_index == 0 || m_next_buffer_index == m_next_buffer.size() / 2) + if (m_next_buffer_index == 0 || m_next_buffer_index == m_next_buffer.size() / 2) { Enqueue(); + } } } -void GranularMixer::PushSamples(const s32 *samples, std::size_t num_samples, float volume) -{ +void GranularMixer::PushSamples(const s32 *samples, u32 num_samples, float volume) { m_dma_mixer.PushSamples(samples, num_samples, volume); } -void GranularMixer::RefreshConfig() { - - // m_config_audio_buffer_ms = Config::Get(Config::MAIN_AUDIO_BUFFER_SIZE); -} - -void GranularMixer::MixerFifo::Enqueue() -{ +void GranularMixer::MixerFifo::Enqueue() { // import numpy as np // import scipy.signal as signal // window = np.convolve(np.ones(128), signal.windows.dpss(128 + 1, 4)) @@ -179,82 +214,34 @@ void GranularMixer::MixerFifo::Enqueue() // elements = ", ".join([f"{x:.10f}f" for x in window]) // print(f'constexpr std::array GRANULE_WINDOW = {{ {elements} // }};') - constexpr std::array GRANULE_WINDOW = { - 0.0000016272f, 0.0000050749f, 0.0000113187f, 0.0000216492f, 0.0000377350f, 0.0000616906f, - 0.0000961509f, 0.0001443499f, 0.0002102045f, 0.0002984010f, 0.0004144844f, 0.0005649486f, - 0.0007573262f, 0.0010002765f, 0.0013036694f, 0.0016786636f, 0.0021377783f, 0.0026949534f, - 0.0033656000f, 0.0041666352f, 0.0051165029f, 0.0062351752f, 0.0075441359f, 0.0090663409f, - 0.0108261579f, 0.0128492811f, 0.0151626215f, 0.0177941726f, 0.0207728499f, 0.0241283062f, - 0.0278907219f, 0.0320905724f, 0.0367583739f, 0.0419244083f, 0.0476184323f, 0.0538693708f, - 0.0607049996f, 0.0681516192f, 0.0762337261f, 0.0849736833f, 0.0943913952f, 0.1045039915f, - 0.1153255250f, 0.1268666867f, 0.1391345431f, 0.1521323012f, 0.1658591025f, 0.1803098534f, - 0.1954750915f, 0.2113408944f, 0.2278888303f, 0.2450959552f, 0.2629348550f, 0.2813737361f, - 0.3003765625f, 0.3199032396f, 0.3399098438f, 0.3603488941f, 0.3811696664f, 0.4023185434f, - 0.4237393998f, 0.4453740162f, 0.4671625177f, 0.4890438330f, 0.5109561670f, 0.5328374823f, - 0.5546259838f, 0.5762606002f, 0.5976814566f, 0.6188303336f, 0.6396511059f, 0.6600901562f, - 0.6800967604f, 0.6996234375f, 0.7186262639f, 0.7370651450f, 0.7549040448f, 0.7721111697f, - 0.7886591056f, 0.8045249085f, 0.8196901466f, 0.8341408975f, 0.8478676988f, 0.8608654569f, - 0.8731333133f, 0.8846744750f, 0.8954960085f, 0.9056086048f, 0.9150263167f, 0.9237662739f, - 0.9318483808f, 0.9392950004f, 0.9461306292f, 0.9523815677f, 0.9580755917f, 0.9632416261f, - 0.9679094276f, 0.9721092781f, 0.9758716938f, 0.9792271501f, 0.9822058274f, 0.9848373785f, - 0.9871507189f, 0.9891738421f, 0.9909336591f, 0.9924558641f, 0.9937648248f, 0.9948834971f, - 0.9958333648f, 0.9966344000f, 0.9973050466f, 0.9978622217f, 0.9983213364f, 0.9986963306f, - 0.9989997235f, 0.9992426738f, 0.9994350514f, 0.9995855156f, 0.9997015990f, 0.9997897955f, - 0.9998556501f, 0.9999038491f, 0.9999383094f, 0.9999622650f, 0.9999783508f, 0.9999886813f, - 0.9999949251f, 0.9999983728f, 0.9999983728f, 0.9999949251f, 0.9999886813f, 0.9999783508f, - 0.9999622650f, 0.9999383094f, 0.9999038491f, 0.9998556501f, 0.9997897955f, 0.9997015990f, - 0.9995855156f, 0.9994350514f, 0.9992426738f, 0.9989997235f, 0.9986963306f, 0.9983213364f, - 0.9978622217f, 0.9973050466f, 0.9966344000f, 0.9958333648f, 0.9948834971f, 0.9937648248f, - 0.9924558641f, 0.9909336591f, 0.9891738421f, 0.9871507189f, 0.9848373785f, 0.9822058274f, - 0.9792271501f, 0.9758716938f, 0.9721092781f, 0.9679094276f, 0.9632416261f, 0.9580755917f, - 0.9523815677f, 0.9461306292f, 0.9392950004f, 0.9318483808f, 0.9237662739f, 0.9150263167f, - 0.9056086048f, 0.8954960085f, 0.8846744750f, 0.8731333133f, 0.8608654569f, 0.8478676988f, - 0.8341408975f, 0.8196901466f, 0.8045249085f, 0.7886591056f, 0.7721111697f, 0.7549040448f, - 0.7370651450f, 0.7186262639f, 0.6996234375f, 0.6800967604f, 0.6600901562f, 0.6396511059f, - 0.6188303336f, 0.5976814566f, 0.5762606002f, 0.5546259838f, 0.5328374823f, 0.5109561670f, - 0.4890438330f, 0.4671625177f, 0.4453740162f, 0.4237393998f, 0.4023185434f, 0.3811696664f, - 0.3603488941f, 0.3399098438f, 0.3199032396f, 0.3003765625f, 0.2813737361f, 0.2629348550f, - 0.2450959552f, 0.2278888303f, 0.2113408944f, 0.1954750915f, 0.1803098534f, 0.1658591025f, - 0.1521323012f, 0.1391345431f, 0.1268666867f, 0.1153255250f, 0.1045039915f, 0.0943913952f, - 0.0849736833f, 0.0762337261f, 0.0681516192f, 0.0607049996f, 0.0538693708f, 0.0476184323f, - 0.0419244083f, 0.0367583739f, 0.0320905724f, 0.0278907219f, 0.0241283062f, 0.0207728499f, - 0.0177941726f, 0.0151626215f, 0.0128492811f, 0.0108261579f, 0.0090663409f, 0.0075441359f, - 0.0062351752f, 0.0051165029f, 0.0041666352f, 0.0033656000f, 0.0026949534f, 0.0021377783f, - 0.0016786636f, 0.0013036694f, 0.0010002765f, 0.0007573262f, 0.0005649486f, 0.0004144844f, - 0.0002984010f, 0.0002102045f, 0.0001443499f, 0.0000961509f, 0.0000616906f, 0.0000377350f, - 0.0000216492f, 0.0000113187f, 0.0000050749f, 0.0000016272f }; - const std::size_t head = m_queue_head.load(std::memory_order_acquire); // Check if we run out of space in the circular queue. (rare) std::size_t next_head = (head + 1) & GRANULE_QUEUE_MASK; - if (next_head == m_queue_tail.load(std::memory_order_acquire)) - { + if (next_head == m_queue_tail.load(std::memory_order_acquire)) { WARN_LOG(Log::Audio, "Granule Queue has completely filled and audio samples are being dropped. " "This should not happen unless the audio backend has stopped requesting audio."); return; } - // By preconstructing the granule window, we have the best chance of - // the compiler optimizing this loop using SIMD instructions. + // The compiler (at least MSVC) fails at optimizing this loop using SIMD instructions. const std::size_t start_index = m_next_buffer_index; - for (std::size_t i = 0; i < GRANULE_SIZE; ++i) - m_queue[head][i] = m_next_buffer[(i + start_index) & GRANULE_MASK] * GRANULE_WINDOW[i]; + for (std::size_t i = 0; i < GRANULE_SIZE; ++i) { + m_queue[head][i] = m_next_buffer[(i + start_index) & GRANULE_MASK] * g_GranuleWindow[i]; + } m_queue_head.store(next_head, std::memory_order_release); m_queue_looping.store(false, std::memory_order_relaxed); } -void GranularMixer::MixerFifo::Dequeue(Granule* granule) -{ +void GranularMixer::MixerFifo::Dequeue(Granule* granule) { const std::size_t granule_queue_size = m_granule_queue_size.load(std::memory_order_relaxed); const std::size_t head = m_queue_head.load(std::memory_order_acquire); std::size_t tail = m_queue_tail.load(std::memory_order_acquire); // Checks to see if the queue has gotten too long. - if (granule_queue_size < ((head - tail) & GRANULE_QUEUE_MASK)) - { + if (granule_queue_size < ((head - tail) & GRANULE_QUEUE_MASK)) { // Jump the playhead to half the queue size behind the head. const std::size_t gap = (granule_queue_size >> 1) + 1; tail = (head - gap) & GRANULE_QUEUE_MASK; @@ -262,8 +249,7 @@ void GranularMixer::MixerFifo::Dequeue(Granule* granule) // Checks to see if the queue is empty. std::size_t next_tail = (tail + 1) & GRANULE_QUEUE_MASK; - if (next_tail == head) - { + if (next_tail == head) { // Only fill gaps when running to prevent stutter on pause. CoreState state = coreState; const bool is_running = state == CORE_RUNNING_CPU || state == CORE_RUNNING_GE; diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h index 7fe308b5eb..39fae20893 100644 --- a/Core/HW/GranularMixer.h +++ b/Core/HW/GranularMixer.h @@ -13,7 +13,7 @@ class PointerWrap; -// Replacement for std::countr_one +// Replacement for std::countr_one from a later version of C++ constexpr std::size_t countr_one_replacement(std::size_t x) { std::size_t count = 0; while (x & 1) { @@ -28,16 +28,16 @@ public: explicit GranularMixer(); // Called from audio threads - std::size_t Mix(s16* samples, std::size_t numSamples, int outSampleRate); + std::size_t Mix(s16* samples, u32 numSamples, int outSampleRate); // Called from emulation thread - void PushSamples(const s32* samples, std::size_t num_samples, float volume); + void PushSamples(const s32* samples, u32 num_samples, float volume); private: const std::size_t SURROUND_CHANNELS = 6; class MixerFifo final { - static constexpr std::size_t MAX_GRANULE_QUEUE_SIZE = 256; + static constexpr std::size_t MAX_GRANULE_QUEUE_SIZE = 128; static constexpr std::size_t GRANULE_QUEUE_MASK = MAX_GRANULE_QUEUE_SIZE - 1; struct StereoPair final { @@ -54,14 +54,12 @@ private: constexpr StereoPair(float left, float right) : l(left), r(right) {} constexpr StereoPair(s16 left, s16 right) : l(left), r(right) {} - StereoPair operator+(const StereoPair& other) const - { + StereoPair operator+(const StereoPair& other) const { return StereoPair(l + other.l, r + other.r); } - StereoPair operator*(const StereoPair& other) const - { - return StereoPair(l * other.l, r * other.r); + StereoPair operator*(float f) const { + return StereoPair(l * f, r * f); } }; @@ -75,22 +73,24 @@ private: public: MixerFifo(GranularMixer* mixer, bool little_endian) : m_mixer(mixer) {} - void PushSamples(const s32* samples, std::size_t num_samples, float volume); - void Mix(s16* samples, std::size_t num_samples, int outSampleRate); + void PushSamples(const s32* samples, u32 num_samples, float volume); + void Mix(s16* samples, u32 num_samples, int outSampleRate); private: GranularMixer* m_mixer; Granule m_next_buffer{}; - std::size_t m_next_buffer_index = 0; + u32 m_next_buffer_index = 0; u32 m_current_index = 0; - Granule m_front, m_back; + Granule m_front; + Granule m_back; - std::atomic m_granule_queue_size{ 20 }; - std::array m_queue; - std::atomic m_queue_head{ 0 }; - std::atomic m_queue_tail{ 0 }; + std::atomic m_granule_queue_size{ 20 }; + Granule m_queue[MAX_GRANULE_QUEUE_SIZE]; + + std::atomic m_queue_head{}; + std::atomic m_queue_tail{}; std::atomic m_queue_looping{ false }; float m_fade_volume = 1.0; @@ -100,9 +100,5 @@ private: StereoPair m_quantization_error; }; - void RefreshConfig(); - MixerFifo m_dma_mixer{ this, false }; - - //int m_config_audio_buffer_ms; }; From db24efcfa42e1b4f75845899c50bceb6b0a08665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 13 Jun 2025 16:18:00 +0200 Subject: [PATCH 03/10] Remove the inner class, since we're only working with a single stereo stream. --- Core/HW/GranularMixer.cpp | 30 ++++------- Core/HW/GranularMixer.h | 106 +++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 79 deletions(-) diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp index 4760452212..761f4a57db 100644 --- a/Core/HW/GranularMixer.cpp +++ b/Core/HW/GranularMixer.cpp @@ -79,7 +79,10 @@ GranularMixer::GranularMixer() { } // Executed from sound stream thread -void GranularMixer::MixerFifo::Mix(s16* samples, u32 num_samples, int outSampleRate) { +std::size_t GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { + if (!samples) + return 0; + memset(samples, 0, num_samples * 2 * sizeof(s16)); constexpr u32 INDEX_HALF = 0x80000000; constexpr DT_s FADE_IN_RC = DT_s(0.008); constexpr DT_s FADE_OUT_RC = DT_s(0.064); @@ -101,13 +104,13 @@ void GranularMixer::MixerFifo::Mix(s16* samples, u32 num_samples, int outSampleR const float fade_out_mul = -std::expm1(-DT_s(1.0) / (out_sample_rate * FADE_OUT_RC)); // Calculate the ideal length of the granule queue. - const std::size_t buffer_size_ms = 40; // TODO: Actually take from the audio backend. - const std::size_t buffer_size_samples = std::llround(buffer_size_ms * in_sample_rate / 1000.0); + const u32 buffer_size_ms = 40; // TODO: Actually take from the audio backend. + const u32 buffer_size_samples = std::llround(buffer_size_ms * in_sample_rate / 1000.0); // Limit the possible queue sizes to any number between 4 and 64. const u32 buffer_size_granules = - std::clamp((buffer_size_samples) / (GRANULE_SIZE >> 1), static_cast(4), - static_cast(MAX_GRANULE_QUEUE_SIZE)); + std::clamp((buffer_size_samples) / (GRANULE_SIZE >> 1), static_cast(4), + static_cast(MAX_GRANULE_QUEUE_SIZE)); m_granule_queue_size.store(buffer_size_granules, std::memory_order_relaxed); @@ -174,17 +177,10 @@ void GranularMixer::MixerFifo::Mix(s16* samples, u32 num_samples, int outSampleR samples += 2; } -} - -std::size_t GranularMixer::Mix(s16* samples, u32 num_samples, int outSampleRate) { - if (!samples) - return 0; - memset(samples, 0, num_samples * 2 * sizeof(s16)); - m_dma_mixer.Mix(samples, num_samples, outSampleRate); return num_samples; } -void GranularMixer::MixerFifo::PushSamples(const s32 *samples, u32 num_samples, float volume) { +void GranularMixer::PushSamples(const s32 *samples, u32 num_samples, float volume) { // TODO: This can be massively sped up. Although hardly likely to be a bottleneck. while (num_samples-- > 0) { const s16 l = clampfloat_s16(samples[0] * volume); @@ -202,11 +198,7 @@ void GranularMixer::MixerFifo::PushSamples(const s32 *samples, u32 num_samples, } } -void GranularMixer::PushSamples(const s32 *samples, u32 num_samples, float volume) { - m_dma_mixer.PushSamples(samples, num_samples, volume); -} - -void GranularMixer::MixerFifo::Enqueue() { +void GranularMixer::Enqueue() { // import numpy as np // import scipy.signal as signal // window = np.convolve(np.ones(128), signal.windows.dpss(128 + 1, 4)) @@ -235,7 +227,7 @@ void GranularMixer::MixerFifo::Enqueue() { m_queue_looping.store(false, std::memory_order_relaxed); } -void GranularMixer::MixerFifo::Dequeue(Granule* granule) { +void GranularMixer::Dequeue(Granule* granule) { const std::size_t granule_queue_size = m_granule_queue_size.load(std::memory_order_relaxed); const std::size_t head = m_queue_head.load(std::memory_order_acquire); std::size_t tail = m_queue_tail.load(std::memory_order_acquire); diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h index 39fae20893..57bf53be7e 100644 --- a/Core/HW/GranularMixer.h +++ b/Core/HW/GranularMixer.h @@ -34,71 +34,57 @@ public: void PushSamples(const s32* samples, u32 num_samples, float volume); private: - const std::size_t SURROUND_CHANNELS = 6; + static constexpr std::size_t MAX_GRANULE_QUEUE_SIZE = 128; + static constexpr std::size_t GRANULE_QUEUE_MASK = MAX_GRANULE_QUEUE_SIZE - 1; - class MixerFifo final { - static constexpr std::size_t MAX_GRANULE_QUEUE_SIZE = 128; - static constexpr std::size_t GRANULE_QUEUE_MASK = MAX_GRANULE_QUEUE_SIZE - 1; + struct StereoPair final { + float l = 0.f; + float r = 0.f; - struct StereoPair final { - float l = 0.f; - float r = 0.f; + constexpr StereoPair() = default; + constexpr StereoPair(const StereoPair&) = default; + constexpr StereoPair& operator=(const StereoPair&) = default; + constexpr StereoPair(StereoPair&&) = default; + constexpr StereoPair& operator=(StereoPair&&) = default; - constexpr StereoPair() = default; - constexpr StereoPair(const StereoPair&) = default; - constexpr StereoPair& operator=(const StereoPair&) = default; - constexpr StereoPair(StereoPair&&) = default; - constexpr StereoPair& operator=(StereoPair&&) = default; + constexpr StereoPair(float mono) : l(mono), r(mono) {} + constexpr StereoPair(float left, float right) : l(left), r(right) {} + constexpr StereoPair(s16 left, s16 right) : l(left), r(right) {} - constexpr StereoPair(float mono) : l(mono), r(mono) {} - constexpr StereoPair(float left, float right) : l(left), r(right) {} - constexpr StereoPair(s16 left, s16 right) : l(left), r(right) {} + StereoPair operator+(const StereoPair& other) const { + return StereoPair(l + other.l, r + other.r); + } - StereoPair operator+(const StereoPair& other) const { - return StereoPair(l + other.l, r + other.r); - } - - StereoPair operator*(float f) const { - return StereoPair(l * f, r * f); - } - }; - - static constexpr std::size_t GRANULE_SIZE = 256; - static constexpr std::size_t GRANULE_OVERLAP = GRANULE_SIZE / 2; - static constexpr std::size_t GRANULE_MASK = GRANULE_SIZE - 1; - static constexpr std::size_t GRANULE_BITS = countr_one_replacement(GRANULE_MASK); - static constexpr std::size_t GRANULE_FRAC_BITS = 32 - GRANULE_BITS; - - using Granule = std::array; - - public: - MixerFifo(GranularMixer* mixer, bool little_endian) : m_mixer(mixer) {} - void PushSamples(const s32* samples, u32 num_samples, float volume); - void Mix(s16* samples, u32 num_samples, int outSampleRate); - - private: - GranularMixer* m_mixer; - - Granule m_next_buffer{}; - u32 m_next_buffer_index = 0; - - u32 m_current_index = 0; - Granule m_front; - Granule m_back; - - std::atomic m_granule_queue_size{ 20 }; - Granule m_queue[MAX_GRANULE_QUEUE_SIZE]; - - std::atomic m_queue_head{}; - std::atomic m_queue_tail{}; - std::atomic m_queue_looping{ false }; - float m_fade_volume = 1.0; - - void Enqueue(); - void Dequeue(Granule* granule); - - StereoPair m_quantization_error; + StereoPair operator*(float f) const { + return StereoPair(l * f, r * f); + } }; - MixerFifo m_dma_mixer{ this, false }; + static constexpr u32 GRANULE_SIZE = 256; + static constexpr u32 GRANULE_OVERLAP = GRANULE_SIZE / 2; + static constexpr u32 GRANULE_MASK = GRANULE_SIZE - 1; + static constexpr u32 GRANULE_BITS = countr_one_replacement(GRANULE_MASK); + static constexpr u32 GRANULE_FRAC_BITS = 32 - GRANULE_BITS; + + using Granule = std::array; + + Granule m_next_buffer{}; + u32 m_next_buffer_index = 0; + + u32 m_current_index = 0; + Granule m_front; + Granule m_back; + + std::atomic m_granule_queue_size{ 20 }; + Granule m_queue[MAX_GRANULE_QUEUE_SIZE]; + + std::atomic m_queue_head{}; + std::atomic m_queue_tail{}; + std::atomic m_queue_looping{ false }; + float m_fade_volume = 1.0; + + void Enqueue(); + void Dequeue(Granule* granule); + + StereoPair m_quantization_error; }; From a1d0f405b12046850e8d9417b9485f469c55d574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 13 Jun 2025 16:29:41 +0200 Subject: [PATCH 04/10] Remove quantization error thingy. Other cleanup --- Core/HW/GranularMixer.cpp | 64 +++++++++++++++++++-------------------- Core/HW/GranularMixer.h | 8 ++--- UI/AudioCommon.cpp | 2 +- libretro/libretro.cpp | 6 ++-- 4 files changed, 38 insertions(+), 42 deletions(-) diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp index 761f4a57db..a6d5930c25 100644 --- a/Core/HW/GranularMixer.cpp +++ b/Core/HW/GranularMixer.cpp @@ -16,11 +16,6 @@ #include "Core/System.h" #include "Core/Util/AudioFormat.h" // for clamp_u16 -using Clock = std::chrono::steady_clock; -using TimePoint = Clock::time_point; -using DT = Clock::duration; -using DT_s = std::chrono::duration>; - // Something like a gaussian. static const float g_GranuleWindow[256] = { 0.0000016272f, 0.0000050749f, 0.0000113187f, 0.0000216492f, 0.0000377350f, 0.0000616906f, @@ -79,13 +74,14 @@ GranularMixer::GranularMixer() { } // Executed from sound stream thread -std::size_t GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { +void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { + _dbg_assert_(samples); if (!samples) - return 0; + return; memset(samples, 0, num_samples * 2 * sizeof(s16)); constexpr u32 INDEX_HALF = 0x80000000; - constexpr DT_s FADE_IN_RC = DT_s(0.008); - constexpr DT_s FADE_OUT_RC = DT_s(0.064); + constexpr double FADE_IN_RC = 0.008; + constexpr double FADE_OUT_RC = 0.064; // We need at least a double because the index jump has 24 bits of fractional precision. const double out_sample_rate = outSampleRate; @@ -100,8 +96,8 @@ std::size_t GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) // These fade in / out multiplier are tuned to match a constant // fade speed regardless of the input or the output sample rate. - const float fade_in_mul = -std::expm1(-DT_s(1.0) / (out_sample_rate * FADE_IN_RC)); - const float fade_out_mul = -std::expm1(-DT_s(1.0) / (out_sample_rate * FADE_OUT_RC)); + const float fade_in_mul = -std::expm1(-1.0 / (out_sample_rate * FADE_IN_RC)); + const float fade_out_mul = -std::expm1(-1.0 / (out_sample_rate * FADE_OUT_RC)); // Calculate the ideal length of the granule queue. const u32 buffer_size_ms = 40; // TODO: Actually take from the audio backend. @@ -112,8 +108,15 @@ std::size_t GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) std::clamp((buffer_size_samples) / (GRANULE_SIZE >> 1), static_cast(4), static_cast(MAX_GRANULE_QUEUE_SIZE)); + if (buffer_size_granules != m_granule_queue_size.load(std::memory_order_relaxed)) { + INFO_LOG(Log::Audio, "Granule buffer size changed to %d", buffer_size_granules); + } + m_granule_queue_size.store(buffer_size_granules, std::memory_order_relaxed); + // TODO: The performance of this could be greatly enhanced with SIMD but it won't be easy + // due to wrapping of various buffers. + bool queue_looping = m_queue_looping.load(std::memory_order_relaxed); while (num_samples-- > 0) { // The indexes for the front and back buffers are offset by 50% of the granule size. // We use the modular nature of 32-bit integers to wrap around the granule size. @@ -129,8 +132,8 @@ std::size_t GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) Dequeue(&m_back); // The Granules are pre-windowed, so we can just add them together - const std::size_t ft = front_index >> GRANULE_FRAC_BITS; - const std::size_t bt = back_index >> GRANULE_FRAC_BITS; + const u32 ft = front_index >> GRANULE_FRAC_BITS; + const u32 bt = back_index >> GRANULE_FRAC_BITS; const StereoPair s0 = m_front[(ft - 2) & GRANULE_MASK] + m_back[(bt - 2) & GRANULE_MASK]; const StereoPair s1 = m_front[(ft - 1) & GRANULE_MASK] + m_back[(bt - 1) & GRANULE_MASK]; const StereoPair s2 = m_front[(ft + 0) & GRANULE_MASK] + m_back[(bt + 0) & GRANULE_MASK]; @@ -156,8 +159,13 @@ std::size_t GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) s5 * ((+0.0f + 0.0f * t1 + 1.0f * t2 - 1.0f * t3) * (1.0f / 12.0f)) ); + // Update the looping flag occasionally. + if (!(num_samples & 31)) { + queue_looping = m_queue_looping.load(std::memory_order_relaxed); + } + // Apply Fade In / Fade Out depending on if we are looping - if (m_queue_looping.load(std::memory_order_relaxed)) + if (queue_looping) m_fade_volume += fade_out_mul * (0.0f - m_fade_volume); else m_fade_volume += fade_in_mul * (1.0f - m_fade_volume); @@ -165,19 +173,11 @@ std::size_t GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) // Apply the fade volume to the sample sample = sample * m_fade_volume; - // This quantization method prevents accumulated error but does not do noise shaping. - sample.l += samples[0] - m_quantization_error.l; - sample.r += samples[1] - m_quantization_error.r; - samples[0] = (int16_t)clamp_value(sample.l, -32767.0f, 32767.0f); samples[1] = (int16_t)clamp_value(sample.r, -32767.0f, 32767.0f); - m_quantization_error.l = clamp_value(samples[0] - sample.l, -1.0f, 1.0f); - m_quantization_error.r = clamp_value(samples[1] - sample.r, -1.0f, 1.0f); - samples += 2; } - return num_samples; } void GranularMixer::PushSamples(const s32 *samples, u32 num_samples, float volume) { @@ -206,10 +206,10 @@ void GranularMixer::Enqueue() { // elements = ", ".join([f"{x:.10f}f" for x in window]) // print(f'constexpr std::array GRANULE_WINDOW = {{ {elements} // }};') - const std::size_t head = m_queue_head.load(std::memory_order_acquire); + const u32 head = m_queue_head.load(std::memory_order_acquire); // Check if we run out of space in the circular queue. (rare) - std::size_t next_head = (head + 1) & GRANULE_QUEUE_MASK; + u32 next_head = (head + 1) & GRANULE_QUEUE_MASK; if (next_head == m_queue_tail.load(std::memory_order_acquire)) { WARN_LOG(Log::Audio, "Granule Queue has completely filled and audio samples are being dropped. " @@ -218,8 +218,8 @@ void GranularMixer::Enqueue() { } // The compiler (at least MSVC) fails at optimizing this loop using SIMD instructions. - const std::size_t start_index = m_next_buffer_index; - for (std::size_t i = 0; i < GRANULE_SIZE; ++i) { + const u32 start_index = m_next_buffer_index; + for (u32 i = 0; i < GRANULE_SIZE; ++i) { m_queue[head][i] = m_next_buffer[(i + start_index) & GRANULE_MASK] * g_GranuleWindow[i]; } @@ -228,19 +228,19 @@ void GranularMixer::Enqueue() { } void GranularMixer::Dequeue(Granule* granule) { - const std::size_t granule_queue_size = m_granule_queue_size.load(std::memory_order_relaxed); - const std::size_t head = m_queue_head.load(std::memory_order_acquire); - std::size_t tail = m_queue_tail.load(std::memory_order_acquire); + const u32 granule_queue_size = m_granule_queue_size.load(std::memory_order_relaxed); + const u32 head = m_queue_head.load(std::memory_order_acquire); + u32 tail = m_queue_tail.load(std::memory_order_acquire); // Checks to see if the queue has gotten too long. if (granule_queue_size < ((head - tail) & GRANULE_QUEUE_MASK)) { // Jump the playhead to half the queue size behind the head. - const std::size_t gap = (granule_queue_size >> 1) + 1; + const u32 gap = (granule_queue_size >> 1) + 1; tail = (head - gap) & GRANULE_QUEUE_MASK; } // Checks to see if the queue is empty. - std::size_t next_tail = (tail + 1) & GRANULE_QUEUE_MASK; + u32 next_tail = (tail + 1) & GRANULE_QUEUE_MASK; if (next_tail == head) { // Only fill gaps when running to prevent stutter on pause. CoreState state = coreState; @@ -248,7 +248,7 @@ void GranularMixer::Dequeue(Granule* granule) { if (g_Config.bFillAudioGaps && is_running) { // Jump the playhead to half the queue size behind the head. // This provides smoother audio playback than suddenly stopping. - const std::size_t gap = std::max(2, granule_queue_size >> 1) - 1; + const u32 gap = std::max(2, granule_queue_size >> 1) - 1; next_tail = (head - gap) & GRANULE_QUEUE_MASK; m_queue_looping.store(true, std::memory_order_relaxed); } else { diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h index 57bf53be7e..ab688294eb 100644 --- a/Core/HW/GranularMixer.h +++ b/Core/HW/GranularMixer.h @@ -14,8 +14,8 @@ class PointerWrap; // Replacement for std::countr_one from a later version of C++ -constexpr std::size_t countr_one_replacement(std::size_t x) { - std::size_t count = 0; +constexpr u32 countr_one_replacement(u32 x) { + u32 count = 0; while (x & 1) { ++count; x >>= 1; @@ -28,7 +28,7 @@ public: explicit GranularMixer(); // Called from audio threads - std::size_t Mix(s16* samples, u32 numSamples, int outSampleRate); + void Mix(s16* samples, u32 numSamples, int outSampleRate); // Called from emulation thread void PushSamples(const s32* samples, u32 num_samples, float volume); @@ -85,6 +85,4 @@ private: void Enqueue(); void Dequeue(Granule* granule); - - StereoPair m_quantization_error; }; diff --git a/UI/AudioCommon.cpp b/UI/AudioCommon.cpp index 87adf95ce0..585ec4605a 100644 --- a/UI/AudioCommon.cpp +++ b/UI/AudioCommon.cpp @@ -30,7 +30,7 @@ GranularMixer g_granular; void NativeMix(int16_t *outStereo, int numFrames, int sampleRateHz, void *userdata) { // Mix UI sound effects on top. if (g_Config.iAudioSyncMode == (int)AudioSyncMode::GRANULAR) { - (int)g_granular.Mix(outStereo, numFrames, sampleRateHz); + g_granular.Mix(outStereo, numFrames, sampleRateHz); } else { g_resampler.Mix(outStereo, numFrames, false, sampleRateHz); } diff --git a/libretro/libretro.cpp b/libretro/libretro.cpp index 9364dc4539..8f9246fa1c 100644 --- a/libretro/libretro.cpp +++ b/libretro/libretro.cpp @@ -1934,10 +1934,8 @@ void retro_cheat_set(unsigned index, bool enabled, const char *code) { } } -int64_t System_GetPropertyInt(SystemProperty prop) -{ - switch (prop) - { +int64_t System_GetPropertyInt(SystemProperty prop) { + switch (prop) { case SYSPROP_AUDIO_SAMPLE_RATE: return SAMPLERATE; #if PPSSPP_PLATFORM(ANDROID) From 5b0d8c06c6fd1082f86ce6e17cdc994973132dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 13 Jun 2025 18:48:15 +0200 Subject: [PATCH 05/10] Add debug UI for granule mixer --- Core/HW/GranularMixer.cpp | 19 +++++++++++++++++-- Core/HW/GranularMixer.h | 12 +++++++++++- UI/AudioCommon.h | 6 ++++++ UI/ImDebugger/ImDebugger.cpp | 29 +++++++++++++++++++++++++++++ UI/ImDebugger/ImDebugger.h | 1 + 5 files changed, 64 insertions(+), 3 deletions(-) diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp index a6d5930c25..8352d398d0 100644 --- a/Core/HW/GranularMixer.cpp +++ b/Core/HW/GranularMixer.cpp @@ -100,7 +100,11 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { const float fade_out_mul = -std::expm1(-1.0 / (out_sample_rate * FADE_OUT_RC)); // Calculate the ideal length of the granule queue. - const u32 buffer_size_ms = 40; // TODO: Actually take from the audio backend. + // NOTE: We must have enough room here for 20fps games, generating all their audio + // in a burst each frame (since we can't force real clock sync). That means 16*3 = 48 or rather 50ms. + // However, in case of faster framerates, we should apply some pressure to reduce this. And if real clock sync + // is on, we should also be able to get away with a shorter buffer here. + const u32 buffer_size_ms = 55; const u32 buffer_size_samples = std::llround(buffer_size_ms * in_sample_rate / 1000.0); // Limit the possible queue sizes to any number between 4 and 64. @@ -233,7 +237,7 @@ void GranularMixer::Dequeue(Granule* granule) { u32 tail = m_queue_tail.load(std::memory_order_acquire); // Checks to see if the queue has gotten too long. - if (granule_queue_size < ((head - tail) & GRANULE_QUEUE_MASK)) { + if (((head - tail) & GRANULE_QUEUE_MASK) > granule_queue_size) { // Jump the playhead to half the queue size behind the head. const u32 gap = (granule_queue_size >> 1) + 1; tail = (head - gap) & GRANULE_QUEUE_MASK; @@ -261,3 +265,14 @@ void GranularMixer::Dequeue(Granule* granule) { *granule = m_queue[tail]; m_queue_tail.store(next_tail, std::memory_order_release); } + +void GranularMixer::GetStats(GranularStats *stats) { + int queuedGranules = ((int)m_queue_head.load() - (int)m_queue_tail.load()) & (MAX_GRANULE_QUEUE_SIZE - 1); + if (queuedGranules < 0) { + queuedGranules = 0; + } + stats->queuedGranules = queuedGranules; + stats->targetQueueSize = m_granule_queue_size.load(std::memory_order_relaxed); + stats->maxQueuedGranules = MAX_GRANULE_QUEUE_SIZE; + stats->fadeVolume = m_fade_volume; +} diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h index ab688294eb..d5e6e7a44b 100644 --- a/Core/HW/GranularMixer.h +++ b/Core/HW/GranularMixer.h @@ -23,6 +23,13 @@ constexpr u32 countr_one_replacement(u32 x) { return count; } +struct GranularStats { + int queuedGranules; + int targetQueueSize; + int maxQueuedGranules; + float fadeVolume; +}; + class GranularMixer final { public: explicit GranularMixer(); @@ -33,6 +40,10 @@ public: // Called from emulation thread void PushSamples(const s32* samples, u32 num_samples, float volume); + void GetStats(GranularStats *stats); + + static constexpr u32 GRANULE_SIZE = 256; + private: static constexpr std::size_t MAX_GRANULE_QUEUE_SIZE = 128; static constexpr std::size_t GRANULE_QUEUE_MASK = MAX_GRANULE_QUEUE_SIZE - 1; @@ -60,7 +71,6 @@ private: } }; - static constexpr u32 GRANULE_SIZE = 256; static constexpr u32 GRANULE_OVERLAP = GRANULE_SIZE / 2; static constexpr u32 GRANULE_MASK = GRANULE_SIZE - 1; static constexpr u32 GRANULE_BITS = countr_one_replacement(GRANULE_MASK); diff --git a/UI/AudioCommon.h b/UI/AudioCommon.h index 88cb40dec8..8811895119 100644 --- a/UI/AudioCommon.h +++ b/UI/AudioCommon.h @@ -1,3 +1,9 @@ #pragma once #include + +#include "Core/HW/GranularMixer.h" +#include "Core/HW/StereoResampler.h" + +extern StereoResampler g_resampler; +extern GranularMixer g_granular; diff --git a/UI/ImDebugger/ImDebugger.cpp b/UI/ImDebugger/ImDebugger.cpp index 7720281559..2b8f3b45dc 100644 --- a/UI/ImDebugger/ImDebugger.cpp +++ b/UI/ImDebugger/ImDebugger.cpp @@ -64,6 +64,7 @@ #include "UI/ImDebugger/ImDebugger.h" #include "UI/ImDebugger/ImGe.h" +#include "UI/AudioCommon.h" extern bool g_TakeScreenshot; static ImVec4 g_normalTextColor; @@ -1339,6 +1340,27 @@ void DrawMediaDecodersView(ImConfig &cfg, ImControl &control) { ImGui::End(); } + +void DrawAudioOut(ImConfig &cfg, ImControl &control) { + if (!ImGui::Begin("Audio output", &cfg.audioOutOpen)) { + ImGui::End(); + return; + } + + if (g_Config.iAudioSyncMode == (int)AudioSyncMode::GRANULAR) { + // Show granular stats + GranularStats stats; + g_granular.GetStats(&stats); + ImGui::Text("Granules"); + ImGui::Text("Queued: %d", stats.queuedGranules); + ImGui::Text("Target: %d", stats.targetQueueSize); + ImGui::Text("Max: %d", stats.maxQueuedGranules); + ImGui::Text("Computed queue latency: %d ms", stats.queuedGranules * GranularMixer::GRANULE_SIZE * 1000 / 44100); + } + + ImGui::End(); +} + void DrawAudioChannels(ImConfig &cfg, ImControl &control) { if (!ImGui::Begin("Raw audio channels", &cfg.audioChannelsOpen)) { ImGui::End(); @@ -2261,6 +2283,8 @@ void ImDebugger::Frame(MIPSDebugInterface *mipsDebug, GPUDebugInterface *gpuDebu ImGui::MenuItem("SasAudio mixer", nullptr, &cfg_.sasAudioOpen); ImGui::MenuItem("Raw audio channels", nullptr, &cfg_.audioChannelsOpen); ImGui::MenuItem("AV Decoder contexts", nullptr, &cfg_.mediaDecodersOpen); + ImGui::Separator(); + ImGui::MenuItem("Audio output / buffer", nullptr, &cfg_.audioOutOpen); ImGui::EndMenu(); } if (ImGui::BeginMenu("Network")) { @@ -2340,6 +2364,10 @@ void ImDebugger::Frame(MIPSDebugInterface *mipsDebug, GPUDebugInterface *gpuDebu DrawAudioChannels(cfg_, control); } + if (cfg_.audioOutOpen) { + DrawAudioOut(cfg_, control); + } + if (cfg_.sasAudioOpen) { DrawSasAudio(cfg_); } @@ -2589,6 +2617,7 @@ void ImConfig::SyncConfig(IniFile *ini, bool save) { sync.Sync("filesystemBrowserOpen", &filesystemBrowserOpen, false); sync.Sync("kernelObjectsOpen", &kernelObjectsOpen, false); sync.Sync("audioChannelsOpen", &audioChannelsOpen, false); + sync.Sync("audioOutOpen", &audioOutOpen, false); sync.Sync("texturesOpen", &texturesOpen, false); sync.Sync("debugStatsOpen", &debugStatsOpen, false); sync.Sync("geDebuggerOpen", &geDebuggerOpen, false); diff --git a/UI/ImDebugger/ImDebugger.h b/UI/ImDebugger/ImDebugger.h index e898325338..d069feca16 100644 --- a/UI/ImDebugger/ImDebugger.h +++ b/UI/ImDebugger/ImDebugger.h @@ -87,6 +87,7 @@ struct ImConfig { bool atracToolOpen; bool memViewOpen[4]; bool luaConsoleOpen; + bool audioOutOpen; // HLE explorer settings // bool filterByUsed = true; From 8a76ac74fc96bf47dab5845f1330fcf707881129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 14 Jun 2025 10:50:36 +0200 Subject: [PATCH 06/10] Granule queue: Move masking to access instead of increment --- Core/HW/GranularMixer.cpp | 16 +++++++++------- Core/HW/GranularMixer.h | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp index 8352d398d0..0bcd7bcd67 100644 --- a/Core/HW/GranularMixer.cpp +++ b/Core/HW/GranularMixer.cpp @@ -213,7 +213,7 @@ void GranularMixer::Enqueue() { const u32 head = m_queue_head.load(std::memory_order_acquire); // Check if we run out of space in the circular queue. (rare) - u32 next_head = (head + 1) & GRANULE_QUEUE_MASK; + u32 next_head = head + 1; if (next_head == m_queue_tail.load(std::memory_order_acquire)) { WARN_LOG(Log::Audio, "Granule Queue has completely filled and audio samples are being dropped. " @@ -223,8 +223,10 @@ void GranularMixer::Enqueue() { // The compiler (at least MSVC) fails at optimizing this loop using SIMD instructions. const u32 start_index = m_next_buffer_index; + + const u32 maskedHead = head & GRANULE_QUEUE_MASK; for (u32 i = 0; i < GRANULE_SIZE; ++i) { - m_queue[head][i] = m_next_buffer[(i + start_index) & GRANULE_MASK] * g_GranuleWindow[i]; + m_queue[maskedHead][i] = m_next_buffer[(i + start_index) & GRANULE_MASK] * g_GranuleWindow[i]; } m_queue_head.store(next_head, std::memory_order_release); @@ -237,14 +239,14 @@ void GranularMixer::Dequeue(Granule* granule) { u32 tail = m_queue_tail.load(std::memory_order_acquire); // Checks to see if the queue has gotten too long. - if (((head - tail) & GRANULE_QUEUE_MASK) > granule_queue_size) { + if ((head - tail) > granule_queue_size) { // Jump the playhead to half the queue size behind the head. const u32 gap = (granule_queue_size >> 1) + 1; - tail = (head - gap) & GRANULE_QUEUE_MASK; + tail = (head - gap); } // Checks to see if the queue is empty. - u32 next_tail = (tail + 1) & GRANULE_QUEUE_MASK; + u32 next_tail = tail + 1; if (next_tail == head) { // Only fill gaps when running to prevent stutter on pause. CoreState state = coreState; @@ -253,7 +255,7 @@ void GranularMixer::Dequeue(Granule* granule) { // Jump the playhead to half the queue size behind the head. // This provides smoother audio playback than suddenly stopping. const u32 gap = std::max(2, granule_queue_size >> 1) - 1; - next_tail = (head - gap) & GRANULE_QUEUE_MASK; + next_tail = head - gap; m_queue_looping.store(true, std::memory_order_relaxed); } else { std::fill(granule->begin(), granule->end(), StereoPair{ 0.0f, 0.0f }); @@ -262,7 +264,7 @@ void GranularMixer::Dequeue(Granule* granule) { } } - *granule = m_queue[tail]; + *granule = m_queue[tail & GRANULE_QUEUE_MASK]; m_queue_tail.store(next_tail, std::memory_order_release); } diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h index d5e6e7a44b..f6cf2335bf 100644 --- a/Core/HW/GranularMixer.h +++ b/Core/HW/GranularMixer.h @@ -88,6 +88,7 @@ private: std::atomic m_granule_queue_size{ 20 }; Granule m_queue[MAX_GRANULE_QUEUE_SIZE]; + // These are permitted to grow indefinitely and masked on use. std::atomic m_queue_head{}; std::atomic m_queue_tail{}; std::atomic m_queue_looping{ false }; From 07ad5ff3725769be2647fa68b1878b1afcb90612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 14 Jun 2025 11:42:49 +0200 Subject: [PATCH 07/10] Add more measurements to the granular mixer queue --- Core/HW/GranularMixer.cpp | 67 ++++++++++++++++++++++++------------ Core/HW/GranularMixer.h | 17 +++++++-- UI/ImDebugger/ImDebugger.cpp | 9 +++-- 3 files changed, 67 insertions(+), 26 deletions(-) diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp index 0bcd7bcd67..befdb684cb 100644 --- a/Core/HW/GranularMixer.cpp +++ b/Core/HW/GranularMixer.cpp @@ -12,6 +12,7 @@ #include "Common/Log.h" #include "Common/Math/math_util.h" #include "Common/Swap.h" +#include "Core/HW/Display.h" #include "Core/Core.h" #include "Core/System.h" #include "Core/Util/AudioFormat.h" // for clamp_u16 @@ -79,6 +80,9 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { if (!samples) return; memset(samples, 0, num_samples * 2 * sizeof(s16)); + + smoothedReadSize_ = smoothedReadSize_ == 0 ? num_samples : (smoothedReadSize_ * 0.95f + num_samples * 0.05f); + constexpr u32 INDEX_HALF = 0x80000000; constexpr double FADE_IN_RC = 0.008; constexpr double FADE_OUT_RC = 0.064; @@ -104,7 +108,7 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { // in a burst each frame (since we can't force real clock sync). That means 16*3 = 48 or rather 50ms. // However, in case of faster framerates, we should apply some pressure to reduce this. And if real clock sync // is on, we should also be able to get away with a shorter buffer here. - const u32 buffer_size_ms = 55; + const u32 buffer_size_ms = 100; const u32 buffer_size_samples = std::llround(buffer_size_ms * in_sample_rate / 1000.0); // Limit the possible queue sizes to any number between 4 and 64. @@ -118,6 +122,20 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { m_granule_queue_size.store(buffer_size_granules, std::memory_order_relaxed); + int actualQueueSize = m_queue_head - m_queue_tail; + if (smoothedQueueSize_ == 0) { + smoothedQueueSize_ = actualQueueSize; + } else { + constexpr float factor = 0.95f; + smoothedQueueSize_ = factor * smoothedQueueSize_ + (1.0f - factor) * (float)actualQueueSize; + } + if (actualQueueSize < queuedGranulesMin_) { + queuedGranulesMin_ = actualQueueSize; + } + if (actualQueueSize > queuedGranulesMax_) { + queuedGranulesMax_ = actualQueueSize; + } + // TODO: The performance of this could be greatly enhanced with SIMD but it won't be easy // due to wrapping of various buffers. bool queue_looping = m_queue_looping.load(std::memory_order_relaxed); @@ -135,7 +153,8 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { else if (back_index < index_jump) Dequeue(&m_back); - // The Granules are pre-windowed, so we can just add them together + // The Granules are pre-windowed, so we can just add them together. A bit of accidental wrapping doesn't matter + // either since the tails are so weak. const u32 ft = front_index >> GRANULE_FRAC_BITS; const u32 bt = back_index >> GRANULE_FRAC_BITS; const StereoPair s0 = m_front[(ft - 2) & GRANULE_MASK] + m_back[(bt - 2) & GRANULE_MASK]; @@ -174,11 +193,8 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { else m_fade_volume += fade_in_mul * (1.0f - m_fade_volume); - // Apply the fade volume to the sample - sample = sample * m_fade_volume; - - samples[0] = (int16_t)clamp_value(sample.l, -32767.0f, 32767.0f); - samples[1] = (int16_t)clamp_value(sample.r, -32767.0f, 32767.0f); + samples[0] = (int16_t)clamp_value(sample.l * m_fade_volume, -32767.0f, 32767.0f); + samples[1] = (int16_t)clamp_value(sample.r * m_fade_volume, -32767.0f, 32767.0f); samples += 2; } @@ -203,18 +219,11 @@ void GranularMixer::PushSamples(const s32 *samples, u32 num_samples, float volum } void GranularMixer::Enqueue() { - // import numpy as np - // import scipy.signal as signal - // window = np.convolve(np.ones(128), signal.windows.dpss(128 + 1, 4)) - // window /= (window[:len(window) // 2] + window[len(window) // 2:]).max() - // elements = ", ".join([f"{x:.10f}f" for x in window]) - // print(f'constexpr std::array GRANULE_WINDOW = {{ {elements} - // }};') const u32 head = m_queue_head.load(std::memory_order_acquire); // Check if we run out of space in the circular queue. (rare) u32 next_head = head + 1; - if (next_head == m_queue_tail.load(std::memory_order_acquire)) { + if ((next_head & GRANULE_QUEUE_MASK) == (m_queue_tail.load(std::memory_order_acquire) & GRANULE_QUEUE_MASK)) { WARN_LOG(Log::Audio, "Granule Queue has completely filled and audio samples are being dropped. " "This should not happen unless the audio backend has stopped requesting audio."); @@ -233,7 +242,7 @@ void GranularMixer::Enqueue() { m_queue_looping.store(false, std::memory_order_relaxed); } -void GranularMixer::Dequeue(Granule* granule) { +void GranularMixer::Dequeue(Granule *granule) { const u32 granule_queue_size = m_granule_queue_size.load(std::memory_order_relaxed); const u32 head = m_queue_head.load(std::memory_order_acquire); u32 tail = m_queue_tail.load(std::memory_order_acquire); @@ -243,21 +252,31 @@ void GranularMixer::Dequeue(Granule* granule) { // Jump the playhead to half the queue size behind the head. const u32 gap = (granule_queue_size >> 1) + 1; tail = (head - gap); + overruns_++; } // Checks to see if the queue is empty. u32 next_tail = tail + 1; - if (next_tail == head) { + + bool looping = m_queue_looping.load(); + + /*if (!looping && !smoothedQueueSize_ < granule_queue_size / 2) { + // Repeat a single block occasionally to make sure we have a reasonably sized queue. + next_tail = tail; + } else*/ if (next_tail == head) { // Only fill gaps when running to prevent stutter on pause. CoreState state = coreState; const bool is_running = state == CORE_RUNNING_CPU || state == CORE_RUNNING_GE; if (g_Config.bFillAudioGaps && is_running) { // Jump the playhead to half the queue size behind the head. + // This will repeat a few past granules I guess? They still contain sensible data. // This provides smoother audio playback than suddenly stopping. const u32 gap = std::max(2, granule_queue_size >> 1) - 1; next_tail = head - gap; + underruns_++; m_queue_looping.store(true, std::memory_order_relaxed); } else { + // Send a zero granule. std::fill(granule->begin(), granule->end(), StereoPair{ 0.0f, 0.0f }); m_queue_looping.store(false, std::memory_order_relaxed); return; @@ -269,12 +288,16 @@ void GranularMixer::Dequeue(Granule* granule) { } void GranularMixer::GetStats(GranularStats *stats) { - int queuedGranules = ((int)m_queue_head.load() - (int)m_queue_tail.load()) & (MAX_GRANULE_QUEUE_SIZE - 1); - if (queuedGranules < 0) { - queuedGranules = 0; - } - stats->queuedGranules = queuedGranules; + stats->queuedGranulesMin = queuedGranulesMin_; + stats->queuedGranulesMax = queuedGranulesMax_; + stats->smoothedQueuedGranules = smoothedQueueSize_; stats->targetQueueSize = m_granule_queue_size.load(std::memory_order_relaxed); stats->maxQueuedGranules = MAX_GRANULE_QUEUE_SIZE; stats->fadeVolume = m_fade_volume; + stats->looping = m_queue_looping; + stats->overruns = overruns_; + stats->underruns = underruns_; + stats->smoothedReadSize = smoothedReadSize_; + queuedGranulesMin_ = 10000; + queuedGranulesMax_ = 0; } diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h index f6cf2335bf..8ab4c50408 100644 --- a/Core/HW/GranularMixer.h +++ b/Core/HW/GranularMixer.h @@ -24,10 +24,16 @@ constexpr u32 countr_one_replacement(u32 x) { } struct GranularStats { - int queuedGranules; + int queuedGranulesMin; + int queuedGranulesMax; + float smoothedQueuedGranules; int targetQueueSize; int maxQueuedGranules; float fadeVolume; + bool looping; + int overruns; + int underruns; + float smoothedReadSize; }; class GranularMixer final { @@ -86,13 +92,20 @@ private: Granule m_back; std::atomic m_granule_queue_size{ 20 }; + float smoothedQueueSize_ = 0.0f; Granule m_queue[MAX_GRANULE_QUEUE_SIZE]; // These are permitted to grow indefinitely and masked on use. std::atomic m_queue_head{}; std::atomic m_queue_tail{}; - std::atomic m_queue_looping{ false }; + std::atomic m_queue_looping{}; + float m_fade_volume = 1.0; + int underruns_ = 0; + int overruns_ = 0; + int queuedGranulesMin_ = 10000; + int queuedGranulesMax_ = 0; + float smoothedReadSize_ = 0.0f; void Enqueue(); void Dequeue(Granule* granule); diff --git a/UI/ImDebugger/ImDebugger.cpp b/UI/ImDebugger/ImDebugger.cpp index 2b8f3b45dc..0b95cb9d54 100644 --- a/UI/ImDebugger/ImDebugger.cpp +++ b/UI/ImDebugger/ImDebugger.cpp @@ -1352,10 +1352,15 @@ void DrawAudioOut(ImConfig &cfg, ImControl &control) { GranularStats stats; g_granular.GetStats(&stats); ImGui::Text("Granules"); - ImGui::Text("Queued: %d", stats.queuedGranules); + ImGui::Text("Read size: %0.2f", stats.smoothedReadSize); + ImGui::Text("Queued min/max: %d / %d", stats.queuedGranulesMin, stats.queuedGranulesMax); + ImGui::Text("Queued (smooth): %0.3f", stats.smoothedQueuedGranules); ImGui::Text("Target: %d", stats.targetQueueSize); ImGui::Text("Max: %d", stats.maxQueuedGranules); - ImGui::Text("Computed queue latency: %d ms", stats.queuedGranules * GranularMixer::GRANULE_SIZE * 1000 / 44100); + ImGui::Text("Computed queue latency: %d ms", (int)(stats.smoothedQueuedGranules * (GranularMixer::GRANULE_SIZE * 1000.0 / 44100.0))); + ImGui::Text("Fade volume: %0.2f", stats.fadeVolume); + ImGui::Text("Looping: %s", BoolStr(stats.looping)); + ImGui::Text("Under/Over: %d / %d", stats.underruns, stats.overruns); } ImGui::End(); From 3b683a8e43a23875b5762dce03bfbc42c66b699e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Tue, 17 Jun 2025 15:19:15 +0200 Subject: [PATCH 08/10] Add UI to pick resampler mode --- UI/GameSettingsScreen.cpp | 4 ++++ UI/ImDebugger/ImDebugger.cpp | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/UI/GameSettingsScreen.cpp b/UI/GameSettingsScreen.cpp index 2f3ee6fa48..a36eb376a1 100644 --- a/UI/GameSettingsScreen.cpp +++ b/UI/GameSettingsScreen.cpp @@ -674,6 +674,10 @@ void GameSettingsScreen::CreateAudioSettings(UI::ViewGroup *audioSettings) { mixWithOthers->SetEnabledPtr(&g_Config.bEnableSound); #endif audioSettings->Add(new ItemHeader(a->T("Audio"))); + + static const char *syncModes[] = { "Smooth", "Classic" }; + + audioSettings->Add(new PopupMultiChoice(&g_Config.iAudioSyncMode, a->T("Audio sync mode"), syncModes, 0, ARRAY_SIZE(syncModes), I18NCat::AUDIO, screenManager())); audioSettings->Add(new CheckBox(&g_Config.bFillAudioGaps, a->T("Fill audio gaps"))); audioSettings->Add(new ItemHeader(a->T("Game volume"))); diff --git a/UI/ImDebugger/ImDebugger.cpp b/UI/ImDebugger/ImDebugger.cpp index 0b95cb9d54..3f9e57c0f3 100644 --- a/UI/ImDebugger/ImDebugger.cpp +++ b/UI/ImDebugger/ImDebugger.cpp @@ -1361,6 +1361,11 @@ void DrawAudioOut(ImConfig &cfg, ImControl &control) { ImGui::Text("Fade volume: %0.2f", stats.fadeVolume); ImGui::Text("Looping: %s", BoolStr(stats.looping)); ImGui::Text("Under/Over: %d / %d", stats.underruns, stats.overruns); + } else { + ImGui::Text("StereoResampler classic"); + char buf[1024]; + g_resampler.GetAudioDebugStats(buf, sizeof(buf)); + ImGui::Text("%s", buf); } ImGui::End(); From e00ef0479262f643319b12f227302c03c37a436b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Tue, 17 Jun 2025 16:16:39 +0200 Subject: [PATCH 09/10] Granular queue: Use the current FPS to get an estimate for how long the maximum queue size should be. --- Core/HW/GranularMixer.cpp | 16 ++++++++++------ Core/HW/GranularMixer.h | 8 ++++++-- UI/AudioCommon.cpp | 9 ++++++++- UI/ImDebugger/ImDebugger.cpp | 2 ++ 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/Core/HW/GranularMixer.cpp b/Core/HW/GranularMixer.cpp index befdb684cb..b12e32dfaf 100644 --- a/Core/HW/GranularMixer.cpp +++ b/Core/HW/GranularMixer.cpp @@ -75,11 +75,12 @@ GranularMixer::GranularMixer() { } // Executed from sound stream thread -void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { +void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate, float fpsEstimate) { _dbg_assert_(samples); if (!samples) return; memset(samples, 0, num_samples * 2 * sizeof(s16)); + frameTimeEstimate_ = 1.0f / fpsEstimate; smoothedReadSize_ = smoothedReadSize_ == 0 ? num_samples : (smoothedReadSize_ * 0.95f + num_samples * 0.05f); @@ -89,14 +90,14 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { // We need at least a double because the index jump has 24 bits of fractional precision. const double out_sample_rate = outSampleRate; - double in_sample_rate = 44100; + double inSampleRate = 44100; const double emulation_speed = 1.0f; // TODO: Change when we're in slow-motion mode etc. if (0 < emulation_speed && emulation_speed != 1.0) - in_sample_rate *= emulation_speed; + inSampleRate *= emulation_speed; const double base = static_cast(1 << GRANULE_FRAC_BITS); - const u32 index_jump = std::lround(base * in_sample_rate / out_sample_rate); + const u32 index_jump = std::lround(base * inSampleRate / out_sample_rate); // These fade in / out multiplier are tuned to match a constant // fade speed regardless of the input or the output sample rate. @@ -108,8 +109,9 @@ void GranularMixer::Mix(s16 *samples, u32 num_samples, int outSampleRate) { // in a burst each frame (since we can't force real clock sync). That means 16*3 = 48 or rather 50ms. // However, in case of faster framerates, we should apply some pressure to reduce this. And if real clock sync // is on, we should also be able to get away with a shorter buffer here. - const u32 buffer_size_ms = 100; - const u32 buffer_size_samples = std::llround(buffer_size_ms * in_sample_rate / 1000.0); + // const u32 buffer_size_ms = frameTimeEstimate_ * 44100.0f; + const u32 buffer_size_samples = smoothedReadSize_ * 4 + std::llround(frameTimeEstimate_ * inSampleRate); + queuedSamplesTarget_ = buffer_size_samples; // Limit the possible queue sizes to any number between 4 and 64. const u32 buffer_size_granules = @@ -298,6 +300,8 @@ void GranularMixer::GetStats(GranularStats *stats) { stats->overruns = overruns_; stats->underruns = underruns_; stats->smoothedReadSize = smoothedReadSize_; + stats->frameTimeEstimate = frameTimeEstimate_; + stats->queuedSamplesTarget = queuedSamplesTarget_; queuedGranulesMin_ = 10000; queuedGranulesMax_ = 0; } diff --git a/Core/HW/GranularMixer.h b/Core/HW/GranularMixer.h index 8ab4c50408..05daba77d1 100644 --- a/Core/HW/GranularMixer.h +++ b/Core/HW/GranularMixer.h @@ -26,6 +26,7 @@ constexpr u32 countr_one_replacement(u32 x) { struct GranularStats { int queuedGranulesMin; int queuedGranulesMax; + int queuedSamplesTarget; float smoothedQueuedGranules; int targetQueueSize; int maxQueuedGranules; @@ -34,6 +35,7 @@ struct GranularStats { int overruns; int underruns; float smoothedReadSize; + float frameTimeEstimate; }; class GranularMixer final { @@ -41,10 +43,10 @@ public: explicit GranularMixer(); // Called from audio threads - void Mix(s16* samples, u32 numSamples, int outSampleRate); + void Mix(s16 *samples, u32 numSamples, int outSampleRate, float vpsEstimate); // Called from emulation thread - void PushSamples(const s32* samples, u32 num_samples, float volume); + void PushSamples(const s32 *samples, u32 num_samples, float volume); void GetStats(GranularStats *stats); @@ -105,7 +107,9 @@ private: int overruns_ = 0; int queuedGranulesMin_ = 10000; int queuedGranulesMax_ = 0; + int queuedSamplesTarget_ = 0; float smoothedReadSize_ = 0.0f; + float frameTimeEstimate_ = 0.0f; void Enqueue(); void Dequeue(Granule* granule); diff --git a/UI/AudioCommon.cpp b/UI/AudioCommon.cpp index 585ec4605a..502f953117 100644 --- a/UI/AudioCommon.cpp +++ b/UI/AudioCommon.cpp @@ -21,6 +21,7 @@ #include "Core/HW/GranularMixer.h" #include "UI/AudioCommon.h" #include "UI/BackgroundAudio.h" +#include "Core/HW/Display.h" StereoResampler g_resampler; GranularMixer g_granular; @@ -30,7 +31,13 @@ GranularMixer g_granular; void NativeMix(int16_t *outStereo, int numFrames, int sampleRateHz, void *userdata) { // Mix UI sound effects on top. if (g_Config.iAudioSyncMode == (int)AudioSyncMode::GRANULAR) { - g_granular.Mix(outStereo, numFrames, sampleRateHz); + // We use the FPS estimate, because to maintain smooth audio even though our + // frame execution is very front (or back) heavy (as we can't count on "real time clock sync" + // to be enabled), we need at least one whole frame buffered, plus a bit of extra. + float fpsEstimate, vps, actualFps; + __DisplayGetFPS(&vps, &fpsEstimate, &actualFps); + + g_granular.Mix(outStereo, numFrames, sampleRateHz, fpsEstimate); } else { g_resampler.Mix(outStereo, numFrames, false, sampleRateHz); } diff --git a/UI/ImDebugger/ImDebugger.cpp b/UI/ImDebugger/ImDebugger.cpp index 3f9e57c0f3..d010a24f26 100644 --- a/UI/ImDebugger/ImDebugger.cpp +++ b/UI/ImDebugger/ImDebugger.cpp @@ -1353,6 +1353,8 @@ void DrawAudioOut(ImConfig &cfg, ImControl &control) { g_granular.GetStats(&stats); ImGui::Text("Granules"); ImGui::Text("Read size: %0.2f", stats.smoothedReadSize); + ImGui::Text("Frame time estimate: %0.1f ms", stats.frameTimeEstimate * 1000.0f); + ImGui::Text("Queued samples target: %d", stats.queuedSamplesTarget); ImGui::Text("Queued min/max: %d / %d", stats.queuedGranulesMin, stats.queuedGranulesMax); ImGui::Text("Queued (smooth): %0.3f", stats.smoothedQueuedGranules); ImGui::Text("Target: %d", stats.targetQueueSize); From 4b439f0108046faff193d0ce7b78d6cee855a463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 22 Aug 2025 21:21:57 +0200 Subject: [PATCH 10/10] Bump gradle plugin while at it --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 229c4c755b..c1bc539906 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:8.12.0' + classpath 'com.android.tools.build:gradle:8.12.1' } }