diff --git a/Core/HW/Atrac3Standalone.cpp b/Core/HW/Atrac3Standalone.cpp
index 4e8f4088b1..e4bc79a3d6 100644
--- a/Core/HW/Atrac3Standalone.cpp
+++ b/Core/HW/Atrac3Standalone.cpp
@@ -18,19 +18,13 @@ public:
 	Atrac3Audio(PSPAudioType audioType, int channels, size_t blockAlign, const uint8_t *extraData, size_t extraDataSize)
 		: audioType_(audioType), channels_(channels) {
 		blockAlign_ = (int)blockAlign;
-		if (audioType == PSP_CODEC_AT3PLUS) {
-			at3pCtx_ = atrac3p_alloc(channels, &blockAlign_);
-			if (at3pCtx_) {
-				codecOpen_ = true;
-			} else {
-				ERROR_LOG(Log::ME, "Failed to open atrac3+ context! (channels=%d blockAlign=%d ed=%d)", channels, (int)blockAlign, (int)extraDataSize);
-			}
-		} else if (audioType_ == PSP_CODEC_AT3) {
+		if (audioType_ == PSP_CODEC_AT3) {
 			at3Ctx_ = atrac3_alloc(channels, &blockAlign_, extraData, (int)extraDataSize);
 			if (at3Ctx_) {
 				codecOpen_ = true;
 			} else {
 				ERROR_LOG(Log::ME, "Failed to open atrac3 context! !channels=%d blockAlign=%d ed=%d)", channels, (int)blockAlign, (int)extraDataSize);
+				codecFailed_ = true;
 			}
 		}
 		for (int i = 0; i < 2; i++) {
@@ -63,17 +57,34 @@ public:
 	}
 
 	bool Decode(const uint8_t *inbuf, int inbytes, int *inbytesConsumed, int outputChannels, int16_t *outbuf, int *outSamples) override {
+		if (outSamples)
+			*outSamples = 0;
+		if (inbytesConsumed)
+			*inbytesConsumed = 0;
 		if (!codecOpen_) {
-			WARN_LOG_N_TIMES(codecNotOpen, 5, Log::ME, "Atrac3Audio:Decode: Codec not open, not decoding");
-			if (outSamples)
-				*outSamples = 0;
-			if (inbytesConsumed)
-				*inbytesConsumed = 0;
-			return false;
+			// We delay the codecOpen until the first decode, so the setChannels call from MediaEngine::getAudioSamples
+			// can take effect. Note, we don't do this with Atrac3, just Atrac3+.
+			if (codecFailed_) {
+				return false;
+			}
+			if (audioType_ == PSP_CODEC_AT3PLUS) {
+				at3pCtx_ = atrac3p_alloc(channels_, &blockAlign_);
+				if (at3pCtx_) {
+					codecOpen_ = true;
+				} else {
+					ERROR_LOG(Log::ME, "Failed to open atrac3+ context! (channels=%d blockAlign=%d)", channels_, (int)blockAlign_);
+					codecFailed_ = true;
+				}
+			}
+			if (!codecOpen_) {
+				WARN_LOG_N_TIMES(codecNotOpen, 5, Log::ME, "Atrac3Audio:Decode: Codec not open, not decoding");
+				return false;
+			}
 		}
 		if (inbytes != blockAlign_ && blockAlign_ != 0) {
 			WARN_LOG(Log::ME, "Atrac3Audio::Decode: Unexpected block align %d (expected %d). %s", inbytes, blockAlign_, at3pCtx_ ? "Atrac3+" : "Atrac3");
 		}
+
 		blockAlign_ = inbytes;
 		// We just call the decode function directly without going through the whole packet machinery.
 		int result;
@@ -125,7 +136,16 @@ public:
 	}
 
 	void SetChannels(int channels) override {
-		// Hmm. ignore for now.
+		_dbg_assert_(audioType_ == PSPAudioType::PSP_CODEC_AT3PLUS);
+		if (audioType_ == PSPAudioType::PSP_CODEC_AT3PLUS) {
+			if (!at3pCtx_) {
+				// Codec shouldn't be open yet.
+				_dbg_assert_(!codecOpen_);
+				channels_ = 1;
+			} else {
+				// This will come every packet, just ignore it.
+			}
+		}
 	}
 
 	PSPAudioType GetAudioType() const override { return audioType_; }
@@ -141,6 +161,7 @@ private:
 	float *buffers_[2]{};
 
 	bool codecOpen_ = false;
+	bool codecFailed_ = false;
 
 	PSPAudioType audioType_;
 };
diff --git a/Core/HW/SimpleAudioDec.h b/Core/HW/SimpleAudioDec.h
index 40d724c565..9884821361 100644
--- a/Core/HW/SimpleAudioDec.h
+++ b/Core/HW/SimpleAudioDec.h
@@ -43,6 +43,7 @@ public:
 	virtual bool Decode(const uint8_t *inbuf, int inbytes, int *inbytesConsumed, int outputChannels, int16_t *outbuf, int *outSamples) = 0;
 	virtual bool IsOK() const = 0;
 
+	// NOTE: This can come late (MediaEngine::getAudioSample)! But it will come before the first Decode.
 	virtual void SetChannels(int channels) = 0;
 	virtual void FlushBuffers() {}