From 19f4eadeb1ffd81e5a4ec591635ef942a01d973e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 12 Jul 2023 15:37:16 +0200 Subject: [PATCH 1/2] Call the sound effect mixer directly from Mix instead of pushing the samples from background audio. --- UI/AudioCommon.cpp | 10 ++- UI/BackgroundAudio.cpp | 149 +++++++++++++++++++++++------------------ UI/BackgroundAudio.h | 62 ++++++++++------- UI/NativeApp.cpp | 2 +- 4 files changed, 130 insertions(+), 93 deletions(-) diff --git a/UI/AudioCommon.cpp b/UI/AudioCommon.cpp index 5cbe809f2b..082d80410f 100644 --- a/UI/AudioCommon.cpp +++ b/UI/AudioCommon.cpp @@ -1,13 +1,19 @@ #include "Common/System/System.h" #include "Core/HW/StereoResampler.h" // TODO: doesn't belong in Core/HW... #include "UI/AudioCommon.h" +#include "UI/BackgroundAudio.h" StereoResampler g_resampler; // numFrames is number of stereo frames. // This is called from *outside* the emulator thread. -int __AudioMix(int16_t *outstereo, int numFrames, int sampleRate) { - return g_resampler.Mix(outstereo, numFrames, false, sampleRate); +int __AudioMix(int16_t *outStereo, int numFrames, int sampleRateHz) { + int validFrames = g_resampler.Mix(outStereo, numFrames, false, sampleRateHz); + + // Mix sound effects on top. + g_BackgroundAudio.SFX().Mix(outStereo, validFrames, sampleRateHz); + + return validFrames; } void System_AudioGetDebugStats(char *buf, size_t bufSize) { diff --git a/UI/BackgroundAudio.cpp b/UI/BackgroundAudio.cpp index bc3fc19e0c..28ce9c36b2 100644 --- a/UI/BackgroundAudio.cpp +++ b/UI/BackgroundAudio.cpp @@ -257,49 +257,6 @@ BackgroundAudio::~BackgroundAudio() { delete[] buffer; } -BackgroundAudio::Sample *BackgroundAudio::LoadSample(const std::string &path) { - size_t bytes; - uint8_t *data = g_VFS.ReadFile(path.c_str(), &bytes); - if (!data) { - return nullptr; - } - - RIFFReader reader(data, (int)bytes); - - WavData wave; - wave.Read(reader); - - delete [] data; - - if (wave.num_channels != 2 || wave.sample_rate != 44100 || wave.raw_bytes_per_frame != 4) { - ERROR_LOG(AUDIO, "Wave format not supported for mixer playback. Must be 16-bit raw stereo. '%s'", path.c_str()); - return nullptr; - } - - int16_t *samples = new int16_t[2 * wave.numFrames]; - memcpy(samples, wave.raw_data, wave.numFrames * wave.raw_bytes_per_frame); - - return new BackgroundAudio::Sample(samples, wave.numFrames); -} - -void BackgroundAudio::LoadSamples() { - samples_.resize((size_t)UI::UISound::COUNT); - samples_[(size_t)UI::UISound::BACK] = std::unique_ptr(LoadSample("sfx_back.wav")); - samples_[(size_t)UI::UISound::SELECT] = std::unique_ptr(LoadSample("sfx_select.wav")); - samples_[(size_t)UI::UISound::CONFIRM] = std::unique_ptr(LoadSample("sfx_confirm.wav")); - samples_[(size_t)UI::UISound::TOGGLE_ON] = std::unique_ptr(LoadSample("sfx_toggle_on.wav")); - samples_[(size_t)UI::UISound::TOGGLE_OFF] = std::unique_ptr(LoadSample("sfx_toggle_off.wav")); - - UI::SetSoundCallback([](UI::UISound sound) { - g_BackgroundAudio.PlaySFX(sound); - }); -} - -void BackgroundAudio::PlaySFX(UI::UISound sfx) { - std::lock_guard lock(mutex_); - plays_.push_back(PlayInstance{ sfx, 0, 64, false }); -} - void BackgroundAudio::Clear(bool hard) { if (!hard) { fadingOut_ = true; @@ -372,28 +329,6 @@ bool BackgroundAudio::Play() { } } - // Mix in menu sound effects. Terribly slow mixer but meh. - if (!plays_.empty()) { - for (int i = 0; i < sz * 2; i += 2) { - std::vector::iterator iter = plays_.begin(); - while (iter != plays_.end()) { - PlayInstance inst = *iter; - auto sample = samples_[(int)inst.sound].get(); - if (!sample || iter->offset >= sample->length_) { - iter->done = true; - iter = plays_.erase(iter); - } else { - if (!iter->done) { - buffer[i] += sample->data_[inst.offset * 2] * inst.volume >> 8; - buffer[i + 1] += sample->data_[inst.offset * 2 + 1] * inst.volume >> 8; - } - iter->offset++; - iter++; - } - } - } - } - System_AudioPushSamples(buffer, sz); if (at3Reader_ && fadingOut_ && volume_ <= 0.0f) { @@ -431,3 +366,87 @@ void BackgroundAudio::Update() { sndLoadPending_ = false; } } + +static inline int16_t Clamp16(int32_t sample) { + if (sample < -32767) return -32767; + if (sample > 32767) return 32767; + return sample; +} + +void SoundEffectMixer::Mix(int16_t *buffer, int sz, int sampleRateHz) { + // Mix in menu sound effects. Terribly slow mixer but meh. + if (plays_.empty()) { + return; + } + + for (std::vector::iterator iter = plays_.begin(); iter != plays_.end(); ) { + auto sample = samples_[(int)iter->sound].get(); + for (int i = 0; i < sz * 2; i += 2) { + if (!sample || iter->offset >= sample->length_) { + iter->done = true; + break; + } + + // Clamping add on top. Not great, we should be mixing at higher bitrate instead. Oh well. + int left = buffer[i]; + int right = buffer[i + 1]; + + left = Clamp16(left + (sample->data_[iter->offset * 2] * iter->volume >> 8)); + right = Clamp16(right + (sample->data_[iter->offset * 2 + 1] * iter->volume >> 8)); + + buffer[i] = left; + buffer[i + 1] = right; + + iter->offset++; + } + + if (iter->done) { + iter = plays_.erase(iter); + } else { + iter++; + } + } +} + +void SoundEffectMixer::Play(UI::UISound sfx) { + std::lock_guard guard(mutex_); + plays_.push_back(PlayInstance{ sfx, 0, 64, false }); +} + +Sample *SoundEffectMixer::LoadSample(const std::string &path) { + size_t bytes; + uint8_t *data = g_VFS.ReadFile(path.c_str(), &bytes); + if (!data) { + return nullptr; + } + + RIFFReader reader(data, (int)bytes); + + WavData wave; + wave.Read(reader); + + delete[] data; + + if (wave.num_channels != 2 || wave.sample_rate != 44100 || wave.raw_bytes_per_frame != 4) { + ERROR_LOG(AUDIO, "Wave format not supported for mixer playback. Must be 16-bit raw stereo. '%s'", path.c_str()); + return nullptr; + } + + int16_t *samples = new int16_t[2 * wave.numFrames]; + memcpy(samples, wave.raw_data, wave.numFrames * wave.raw_bytes_per_frame); + + return new Sample(samples, wave.numFrames); +} + +void SoundEffectMixer::LoadSamples() { + samples_.resize((size_t)UI::UISound::COUNT); + samples_[(size_t)UI::UISound::BACK] = std::unique_ptr(LoadSample("sfx_back.wav")); + samples_[(size_t)UI::UISound::SELECT] = std::unique_ptr(LoadSample("sfx_select.wav")); + samples_[(size_t)UI::UISound::CONFIRM] = std::unique_ptr(LoadSample("sfx_confirm.wav")); + samples_[(size_t)UI::UISound::TOGGLE_ON] = std::unique_ptr(LoadSample("sfx_toggle_on.wav")); + samples_[(size_t)UI::UISound::TOGGLE_OFF] = std::unique_ptr(LoadSample("sfx_toggle_off.wav")); + + UI::SetSoundCallback([](UI::UISound sound) { + g_BackgroundAudio.SFX().Play(sound); + }); +} diff --git a/UI/BackgroundAudio.h b/UI/BackgroundAudio.h index 818b063a37..729f7024bd 100644 --- a/UI/BackgroundAudio.h +++ b/UI/BackgroundAudio.h @@ -10,6 +10,38 @@ class AT3PlusReader; +struct Sample { + // data must be new-ed. + Sample(int16_t *data, int length) : data_(data), length_(length) {} + ~Sample() { + delete[] data_; + } + int16_t *data_; + int length_; // stereo samples. +}; + +// Mixer for things played on top of everything. +class SoundEffectMixer { +public: + static Sample *LoadSample(const std::string &path); + void LoadSamples(); + + void Mix(int16_t *buffer, int sz, int sampleRateHz); + void Play(UI::UISound sfx); + + std::vector> samples_; + + struct PlayInstance { + UI::UISound sound; + int offset; + int volume; // 0..255 + bool done; + }; + + std::mutex mutex_; + std::vector plays_; +}; + class BackgroundAudio { public: BackgroundAudio(); @@ -19,8 +51,9 @@ public: void Update(); bool Play(); - void LoadSamples(); - void PlaySFX(UI::UISound sfx); + SoundEffectMixer &SFX() { + return sfxMixer_; + } private: void Clear(bool hard); @@ -33,35 +66,14 @@ private: Path bgGamePath_; std::atomic sndLoadPending_; int playbackOffset_ = 0; - AT3PlusReader *at3Reader_; + AT3PlusReader *at3Reader_ = nullptr; double gameLastChanged_ = 0.0; double lastPlaybackTime_ = 0.0; int *buffer = nullptr; bool fadingOut_ = true; float volume_ = 0.0f; float delta_ = -0.0001f; - - struct PlayInstance { - UI::UISound sound; - int offset; - int volume; // 0..255 - bool done; - }; - - struct Sample { - // data must be new-ed. - Sample(int16_t *data, int length) : data_(data), length_(length) {} - ~Sample() { - delete[] data_; - } - int16_t *data_; - int length_; // stereo samples. - }; - - static Sample *LoadSample(const std::string &path); - - std::vector plays_; - std::vector> samples_; + SoundEffectMixer sfxMixer_; }; extern BackgroundAudio g_BackgroundAudio; diff --git a/UI/NativeApp.cpp b/UI/NativeApp.cpp index 8e23a3ad93..b54d80140a 100644 --- a/UI/NativeApp.cpp +++ b/UI/NativeApp.cpp @@ -763,7 +763,7 @@ void NativeInit(int argc, const char *argv[], const char *savegame_dir, const ch #endif // TODO: Load these in the background instead of synchronously. - g_BackgroundAudio.LoadSamples(); + g_BackgroundAudio.SFX().LoadSamples(); if (!boot_filename.empty() && stateToLoad.Valid()) { SaveState::Load(stateToLoad, -1, [](SaveState::Status status, const std::string &message, void *) { From f1ff12784a4cfb98b0c5122c27e441cd83191124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 12 Jul 2023 17:54:41 +0200 Subject: [PATCH 2/2] Allow flexible sample rate for UI sound effects --- Common/UI/Root.cpp | 1 - UI/BackgroundAudio.cpp | 26 ++++++++++++++++---------- UI/BackgroundAudio.h | 5 +++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/Common/UI/Root.cpp b/Common/UI/Root.cpp index 123cd3376f..3c4311160c 100644 --- a/Common/UI/Root.cpp +++ b/Common/UI/Root.cpp @@ -226,7 +226,6 @@ static KeyEventResult KeyEventToFocusMoves(const KeyInput &key) { } heldKeys.insert(hk); - INFO_LOG(SCECTRL, "focus move: %d", key.keyCode); focusMoves.push_back(key.keyCode); retval = KeyEventResult::ACCEPT; } diff --git a/UI/BackgroundAudio.cpp b/UI/BackgroundAudio.cpp index 28ce9c36b2..1decb572c9 100644 --- a/UI/BackgroundAudio.cpp +++ b/UI/BackgroundAudio.cpp @@ -381,23 +381,30 @@ void SoundEffectMixer::Mix(int16_t *buffer, int sz, int sampleRateHz) { for (std::vector::iterator iter = plays_.begin(); iter != plays_.end(); ) { auto sample = samples_[(int)iter->sound].get(); + + int64_t rateOfSample = sample->rateInHz_; + int64_t stride = (rateOfSample << 32) / sampleRateHz; + for (int i = 0; i < sz * 2; i += 2) { - if (!sample || iter->offset >= sample->length_) { + if (!sample || (iter->offset >> 32) >= sample->length_ - 2) { iter->done = true; break; } - // Clamping add on top. Not great, we should be mixing at higher bitrate instead. Oh well. - int left = buffer[i]; - int right = buffer[i + 1]; + int wholeOffset = iter->offset >> 32; + int frac = (iter->offset >> 20) & 0xFFF; // Use a 12 bit fraction to get away with 32-bit multiplies - left = Clamp16(left + (sample->data_[iter->offset * 2] * iter->volume >> 8)); - right = Clamp16(right + (sample->data_[iter->offset * 2 + 1] * iter->volume >> 8)); + int interpolatedLeft = (sample->data_[wholeOffset * 2] * (0x1000 - frac) + sample->data_[(wholeOffset + 1) * 2] * frac) >> 12; + int interpolatedRight = (sample->data_[wholeOffset * 2 + 1] * (0x1000 - frac) + sample->data_[(wholeOffset + 1) * 2 + 1] * frac) >> 12; + + // Clamping add on top per sample. Not great, we should be mixing at higher bitrate instead. Oh well. + int left = Clamp16(buffer[i] + (interpolatedLeft * iter->volume >> 8)); + int right = Clamp16(buffer[i + 1] + (interpolatedRight * iter->volume >> 8)); buffer[i] = left; buffer[i + 1] = right; - iter->offset++; + iter->offset += stride; } if (iter->done) { @@ -427,15 +434,14 @@ Sample *SoundEffectMixer::LoadSample(const std::string &path) { delete[] data; - if (wave.num_channels != 2 || wave.sample_rate != 44100 || wave.raw_bytes_per_frame != 4) { + if (wave.num_channels != 2 || wave.raw_bytes_per_frame != 4) { ERROR_LOG(AUDIO, "Wave format not supported for mixer playback. Must be 16-bit raw stereo. '%s'", path.c_str()); return nullptr; } int16_t *samples = new int16_t[2 * wave.numFrames]; memcpy(samples, wave.raw_data, wave.numFrames * wave.raw_bytes_per_frame); - - return new Sample(samples, wave.numFrames); + return new Sample(samples, wave.numFrames, wave.sample_rate); } void SoundEffectMixer::LoadSamples() { diff --git a/UI/BackgroundAudio.h b/UI/BackgroundAudio.h index 729f7024bd..c03d4ea318 100644 --- a/UI/BackgroundAudio.h +++ b/UI/BackgroundAudio.h @@ -12,12 +12,13 @@ class AT3PlusReader; struct Sample { // data must be new-ed. - Sample(int16_t *data, int length) : data_(data), length_(length) {} + Sample(int16_t *data, int length, int rateInHz) : data_(data), length_(length), rateInHz_(rateInHz) {} ~Sample() { delete[] data_; } int16_t *data_; int length_; // stereo samples. + int rateInHz_; // sampleRate }; // Mixer for things played on top of everything. @@ -33,7 +34,7 @@ public: struct PlayInstance { UI::UISound sound; - int offset; + int64_t offset; // 32.32 fixed point int volume; // 0..255 bool done; };