diff --git a/Core/HW/SimpleAudioDec.cpp b/Core/HW/SimpleAudioDec.cpp
index d4b8003e54..317a046a42 100644
--- a/Core/HW/SimpleAudioDec.cpp
+++ b/Core/HW/SimpleAudioDec.cpp
@@ -337,9 +337,13 @@ size_t AuCtx::FindNextMp3Sync() {
 
 // return output pcm size, <0 error
 u32 AuCtx::AuDecode(u32 pcmAddr) {
-	auto outbuf = Memory::GetPointer(PCMBuf);
+	u32 outptr = PCMBuf + nextOutputHalf * PCMBufSize / 2;
+	auto outbuf = Memory::GetPointer(outptr);
 	int outpcmbufsize = 0;
 
+	if (pcmAddr)
+		Memory::Write_U32(outptr, pcmAddr);
+
 	// Decode a single frame in sourcebuff and output into PCMBuf.
 	if (!sourcebuff.empty()) {
 		// FFmpeg doesn't seem to search for a sync for us, so let's do that.
@@ -374,16 +378,18 @@ u32 AuCtx::AuDecode(u32 pcmAddr) {
 	}
 
 	if (outpcmbufsize == 0 && !end) {
-		outpcmbufsize = MaxOutputSample * 4;
-		memset(outbuf, 0, PCMBufSize);
+		// If we didn't decode anything, we fill this half of the buffer with zeros.
+		outpcmbufsize = PCMBufSize / 2;
+		memset(outbuf, 0, outpcmbufsize);
 	} else if ((u32)outpcmbufsize < PCMBufSize) {
-		// TODO: We probably should use a rolling buffer instead.
-		memset(outbuf + outpcmbufsize, 0, PCMBufSize - outpcmbufsize);
+		// TODO: Not sure it actually zeros this out.
+		memset(outbuf + outpcmbufsize, 0, PCMBufSize / 2 - outpcmbufsize);
 	}
 
-	NotifyMemInfo(MemBlockFlags::WRITE, PCMBuf, outpcmbufsize, "AuDecode");
-	if (pcmAddr)
-		Memory::Write_U32(PCMBuf, pcmAddr);
+	if (outpcmbufsize != 0)
+		NotifyMemInfo(MemBlockFlags::WRITE, outptr, outpcmbufsize, "AuDecode");
+
+	nextOutputHalf ^= 1;
 	return outpcmbufsize;
 }
 
@@ -504,7 +510,7 @@ u32 AuCtx::AuResetPlayPosition() {
 }
 
 void AuCtx::DoState(PointerWrap &p) {
-	auto s = p.Section("AuContext", 0, 1);
+	auto s = p.Section("AuContext", 0, 2);
 	if (!s)
 		return;
 
@@ -528,8 +534,17 @@ void AuCtx::DoState(PointerWrap &p) {
 	Do(p, dummy);
 	Do(p, FrameNum);
 
+	if (s < 2) {
+		AuBufAvailable = 0;
+		Version = 3;
+	} else {
+		Do(p, Version);
+		Do(p, AuBufAvailable);
+		Do(p, sourcebuff);
+		Do(p, nextOutputHalf);
+	}
+
 	if (p.mode == p.MODE_READ) {
 		decoder = new SimpleAudio(audioType);
-		AuBufAvailable = 0; // reset to read from file at position readPos
 	}
 }
diff --git a/Core/HW/SimpleAudioDec.h b/Core/HW/SimpleAudioDec.h
index 9f355e72f6..2bc2c566d8 100644
--- a/Core/HW/SimpleAudioDec.h
+++ b/Core/HW/SimpleAudioDec.h
@@ -162,6 +162,7 @@ private:
 	int AuBufAvailable = 0; // the available buffer of AuBuf to be able to recharge data
 	int readPos; // read position in audio source file
 	int askedReadSize = 0; // the size of data requied to be read from file by the game
+	int nextOutputHalf = 0;
 };