From 8d2fbf7079f37f1e3621cb5b658b551131d1fa0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 18 Oct 2026 11:19:11 +0200 Subject: [PATCH] sceVideocodec: Hand pictures back when the PSP's decoder does The PSP's AVC decoder follows the H.264 output process (Annex C), measured in the new pspautotests video/mp4/avchold on streams that differ only in their headers, and video/mpeg/decodeend. With HRD parameters and picture timing SEI, a picture comes out dpb_output_delay after its access unit; Sony's movies have a one frame delay. Without, it is bumped out of a buffer three frames deep, whatever the stream declares. sceMpegAvcDecodeStop hands back what's left, and streams with more than three reference frames are refused with 0x80628002. We handed each picture back as soon as it was decoded. MGS Peace Walker's movie player only checks for the end of a movie while its clock is short of the movie's length, so its last picture landed right on that limit and the game hung on a black screen after the cutscene. ffmpeg still decodes the pixels, in display order; each access unit is tagged with its index so frames can be matched back, and ffmpeg's H.264 parser supplies the picture timing. A call can now hand back several pictures, as consecutive 0x2C byte descriptors the way mpeg.prx's sceMpegAvcDecodeStop reads them, each in a frame buffer slot of its own (reallocated when a savestate brings a block too small for that). Not followed: with B-frames and no picture timing, the PSP can complete several pictures in one call; here they come out one a call. All of the game movies checked have picture timing. Co-Authored-By: Claude Opus 5.5 (1M context) --- Core/HLE/sceVideocodec.cpp | 239 ++++++++++++++++++----------- Core/HW/AvcDecoder.cpp | 300 ++++++++++++++++++++++++++++++++++--- Core/HW/AvcDecoder.h | 58 ++++++- pspautotests | 2 +- test.py | 2 + 5 files changed, 481 insertions(+), 120 deletions(-) diff --git a/Core/HLE/sceVideocodec.cpp b/Core/HLE/sceVideocodec.cpp index 0ea5caaaf3..2e8ab645cc 100644 --- a/Core/HLE/sceVideocodec.cpp +++ b/Core/HLE/sceVideocodec.cpp @@ -273,38 +273,47 @@ u32 VideocodecFrameBufferLayout(int width, int height, int sizes[8], u32 offsets return total; } +// How many pictures one call can hand back. The decoder holds three back at most, so a call can +// complete them and its own; mpeg.prx reads one descriptor per picture, 0x2C bytes apart. +static const int MAX_PICTURES = 4; +static const u32 PICTURE_DESCRIPTOR_SIZE = 0x2C; + // The descriptor mpeg.prx passes in is empty: on hardware the ME owns the frame buffers and -// reports where it put them. So allocate them here and fill in the eight buffer addresses. -static bool PublishFrameBuffers(VideocodecCtx &vctx, u32 structAddr, int width, int height, u32 buffers[8]) { +// reports where it put them. So allocate them here and fill in the eight buffer addresses, for +// the slot'th picture of the call. +static bool PublishFrameBuffers(VideocodecCtx &vctx, u32 structAddr, int width, int height, int slot, u32 buffers[8]) { u32 offsets[8]; const u32 total = VideocodecFrameBufferLayout(width, height, nullptr, offsets); - if (total == 0) { + if (total == 0 || slot < 0 || slot >= MAX_PICTURES) { return false; } - if (vctx.frameBuffers && (width != vctx.frameBufferWidth || height != vctx.frameBufferHeight)) { + // A savestate from before frame buffers had a slot per picture carries a block with room for one. + if (vctx.frameBuffers && (width != vctx.frameBufferWidth || height != vctx.frameBufferHeight || + vctx.frameBuffersSize < total * MAX_PICTURES)) { g_meAlloc.Free(vctx.frameBuffers); vctx.frameBuffers = 0; } if (!vctx.frameBuffers) { MEEnsureRam(); - u32 size = total; + u32 size = total * MAX_PICTURES; vctx.frameBuffers = g_meAlloc.Alloc(size, false, "VideocodecFrame"); if (vctx.frameBuffers == (u32)-1) { vctx.frameBuffers = 0; - ERROR_LOG(Log::ME, "sceVideocodec: no room in ME memory for %d bytes of frame buffers", total); + ERROR_LOG(Log::ME, "sceVideocodec: no room in ME memory for %d bytes of frame buffers", total * MAX_PICTURES); return false; } - vctx.frameBuffersSize = total; + vctx.frameBuffersSize = total * MAX_PICTURES; vctx.frameBufferWidth = width; vctx.frameBufferHeight = height; INFO_LOG(Log::ME, "sceVideocodec: %d bytes of frame buffers at %08x for %dx%d", - total, vctx.frameBuffers, width, height); + total * MAX_PICTURES, vctx.frameBuffers, width, height); } for (int i = 0; i < 8; i++) { - buffers[i] = vctx.frameBuffers + offsets[i]; + buffers[i] = vctx.frameBuffers + slot * total + offsets[i]; } + structAddr += slot * PICTURE_DESCRIPTOR_SIZE; // mpeg.prx reads eight buffer addresses off the front of this structure (`lw` at 0x00..0x1C, // verified in 1.3 at 08805698 and 1.8 at 08805898) and takes the frame dimensions from its own @@ -341,9 +350,16 @@ void VideocodecGetCtxInfo(std::vector *infos) { bool VideocodecGetFrameBuffers(u32 firstBuffer, u32 buffers[8], int *width, int *height) { // sceMpegbase only has the first of the eight addresses, so find whose allocation it is. const VideocodecCtx *found = nullptr; + u32 base = 0; for (const auto &[addr, ctx] : g_videocodecCtxs) { - if (ctx.frameBuffers && ctx.frameBuffers == firstBuffer) { + if (!ctx.frameBuffers || firstBuffer < ctx.frameBuffers || firstBuffer >= ctx.frameBuffers + ctx.frameBuffersSize) { + continue; + } + // One of the slots of the pictures a call handed back. + const u32 total = VideocodecFrameBufferLayout(ctx.frameBufferWidth, ctx.frameBufferHeight, nullptr, nullptr); + if (total != 0 && (firstBuffer - ctx.frameBuffers) % total == 0) { found = &ctx; + base = firstBuffer; break; } } @@ -359,7 +375,7 @@ bool VideocodecGetFrameBuffers(u32 firstBuffer, u32 buffers[8], int *width, int *height = found->frameBufferHeight; } for (int i = 0; i < 8; i++) { - buffers[i] = found->frameBuffers + offsets[i]; + buffers[i] = base + offsets[i]; } return true; } @@ -368,16 +384,18 @@ bool VideocodecGetFrameBuffers(u32 firstBuffer, u32 buffers[8], int *width, int // vertical bands split into two 16-pixel halves, and which buffer a row lands in depends on // whether it is even or odd. This is the exact inverse of ReadTiledYCbCr in sceMpeg.cpp, which // is what reads it back out - see the comment there for the full layout. -static void WriteTiledYCbCr(const u32 *buffers, const AvcDecoder &dec, int width, int height) { +static void WriteTiledYCbCr(const u32 *buffers, const AvcPicture &pic) { + const int width = pic.width; + const int height = pic.height; const int width2 = width >> 1; const int height2 = height >> 1; - const u8 *srcY = dec.Plane(0); - const u8 *srcCb = dec.Plane(1); - const u8 *srcCr = dec.Plane(2); - const int strideY = dec.Stride(0); - const int strideCb = dec.Stride(1); - const int strideCr = dec.Stride(2); + const u8 *srcY = pic.planes[0]; + const u8 *srcCb = pic.planes[1]; + const u8 *srcCr = pic.planes[2]; + const int strideY = pic.strides[0]; + const int strideCb = pic.strides[1]; + const int strideCr = pic.strides[2]; if (!srcY || !srcCb || !srcCr) { return; } @@ -536,6 +554,64 @@ static int sceVideocodecReleaseEDRAM(u32 ctxAddr) { return MECall(hleLogInfo(Log::ME, 0, "released %08x", token), "videocodec releaseedram", releaseEdramUs); } +// Type 0: copies the pictures the decoder handed back into the frame buffers the ME reports, +// and says how many in the output descriptor. mpeg.prx converts the first one in +// sceMpegAvcDecode, and all of them in sceMpegAvcDecodeStop (08805130 in 6.61's mpeg.prx reads a +// descriptor per picture, 0x2C bytes apart). Returns the number published. +static int PublishPictures(VideocodecCtx &vctx, u32 ctxAddr, u32 outAddr) { + const std::vector &pictures = vctx.decoder->Pictures(); + const u32 yuvStructAddr = Memory::ReadUnchecked_U32(ctxAddr + CTX_YUV_STRUCT); + int published = 0; + if (!pictures.empty() && !Memory::IsValidRange(yuvStructAddr, 8 * 4)) { + WARN_LOG(Log::ME, "sceVideocodec: type 0 without a usable buffer list"); + } else { + for (const AvcPicture &pic : pictures) { + u32 buffers[8]; + if (published >= MAX_PICTURES || !PublishFrameBuffers(vctx, yuvStructAddr, pic.width, pic.height, published, buffers)) { + break; + } + WriteTiledYCbCr(buffers, pic); + published++; + } + } + if (published > 0) { + Memory::WriteUnchecked_U32(pictures[0].width, outAddr + 8); + Memory::WriteUnchecked_U32(pictures[0].height, outAddr + 12); + Memory::WriteUnchecked_U32(1, outAddr + 28); + } + // The count of images - mpeg.prx won't convert without it. + Memory::WriteUnchecked_U32(published, outAddr + 32); + Memory::WriteUnchecked_U32(published ? 0 : 1, outAddr + 36); + vctx.frameCount += published; + return published; +} + +// The decode takes real time on the ME: about 3.4ms for a 480x272 frame on a PSP, measured as +// sceMpegAvcDecode (5.8ms) less sceMpegAvcCsc alone (2.4ms), in pspautotests +// video/mpeg/playertiming. Movie players that present every decoded frame after a single +// vblank wait rely on decode, colour conversion and blit adding up to more than a vblank. +// It takes as long when the decoder holds the picture back: the first sceMpegAvcDecode calls of +// a stream, which return none, took 4.2-5.4ms in video/mp4/mp4timing. Until a picture has told +// us the size, assume full screen. +static int VideocodecDecodeResult(const VideocodecCtx &vctx, int type, int auBytes, int pictures) { + int delayUs = 0; + if (auBytes > 0) { + int w = vctx.frameBufferWidth > 0 ? vctx.frameBufferWidth : vctx.decoder->Width(); + int h = vctx.frameBufferHeight > 0 ? vctx.frameBufferHeight : vctx.decoder->Height(); + if (w <= 0 || h <= 0) { + w = 480; + h = 272; + } + delayUs = MEScheduleJob(PowerScaleFromDefaultClock((int)(3400LL * w * h / (480 * 272)))); + } + + if (delayUs > 0) { + return hleDelayResult(hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %d picture(s)", + type, auBytes, pictures), "videocodec decode", delayUs); + } + return hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %d picture(s)", type, auBytes, pictures); +} + static int sceVideocodecDecode(u32 ctxAddr, int type) { if (!Memory::IsValidRange(ctxAddr, 96)) { return hleLogError(Log::ME, -1, "bad context pointer"); @@ -580,6 +656,33 @@ static int sceVideocodecDecode(u32 ctxAddr, int type) { auSize, auAddr); } } + auto out32 = [outAddr](int offset, u32 value) { + Memory::WriteUnchecked_U32(value, outAddr + offset); + }; + + if (type == 0) { + // Pictures come out when the PSP's decoder would hand them back - see AvcDecoder.cpp. + // Only the type 1 path fills the rest of the descriptor in: for type 0 the YCbCr + // descriptor sits just 0x40 bytes after this one - mpeg.prx allocates them adjacently - so + // writing the type 1 fields here scribbles over the buffer addresses the colour conversion + // is about to read. Nor is a picture announced until the buffers holding it have been + // published: saying "one image decoded" and then failing to allocate would have mpeg.prx + // convert from whatever the descriptor pointed at last. + int pictures = 0; + if (au && auBytes > 0) { + if (vctx.decoder->DecodePictures(au, auBytes) < 0) { + // The PSP refuses streams with more than three reference frames, and mpeg.prx + // passes this on from sceMpegAvcDecode (pspautotests video/mp4/avchold). + return hleLogWarning(Log::ME, SCE_MPEG_ERROR_AVC_DECODE_FATAL, "stream the PSP's decoder can't play"); + } + pictures = PublishPictures(vctx, ctxAddr, outAddr); + } else { + out32(32, 0); + out32(36, 1); + } + return VideocodecDecodeResult(vctx, type, auBytes, pictures); + } + if (au && auBytes > 0) { gotFrame = vctx.decoder->Decode(au, auBytes); } @@ -587,82 +690,27 @@ static int sceVideocodecDecode(u32 ctxAddr, int type) { const int width = gotFrame ? vctx.decoder->Width() : 0; const int height = gotFrame ? vctx.decoder->Height() : 0; - auto out32 = [outAddr](int offset, u32 value) { - Memory::WriteUnchecked_U32(value, outAddr + offset); - }; - - // Only the type 1 path fills the descriptor in. For type 0 the YCbCr descriptor sits just - // 0x40 bytes after this one - mpeg.prx allocates them adjacently - so writing the type 1 - // fields here scribbles over the buffer addresses the colour conversion is about to read. - // For type 0 the frame isn't announced until the buffers holding it have been published - - // see below. Saying "one image decoded" and then failing to allocate would have mpeg.prx - // convert from whatever the descriptor pointed at last. - bool published = false; - if (type == 0) { - out32(8, width); - out32(12, height); - out32(28, 1); - } else { - out32(OUT_DATA, auAddr); - out32(OUT_SIZE, auSize); - out32(OUT_UNK12, 0x40); - out32(OUT_CONSUMED, auSize); - out32(OUT_WIDTH, width); - out32(OUT_HEIGHT, height); - out32(OUT_FRAME_READY, gotFrame ? 2 : 1); - out32(OUT_UNK64, 1); - out32(OUT_UNK72, (u32)-1); - out32(OUT_TIMESTAMP, vctx.frameCount * 0x64); - out32(OUT_FPS, 2997); - } + out32(OUT_DATA, auAddr); + out32(OUT_SIZE, auSize); + out32(OUT_UNK12, 0x40); + out32(OUT_CONSUMED, auSize); + out32(OUT_WIDTH, width); + out32(OUT_HEIGHT, height); + out32(OUT_FRAME_READY, gotFrame ? 2 : 1); + out32(OUT_UNK64, 1); + out32(OUT_UNK72, (u32)-1); + out32(OUT_TIMESTAMP, vctx.frameCount * 0x64); + out32(OUT_FPS, 2997); if (gotFrame) { vctx.frameCount++; - if (type == 0) { - const u32 yuvStructAddr = Memory::ReadUnchecked_U32(ctxAddr + CTX_YUV_STRUCT); - if (Memory::IsValidRange(yuvStructAddr, 8 * 4)) { - u32 buffers[8]; - if (PublishFrameBuffers(vctx, yuvStructAddr, width, height, buffers)) { - WriteTiledYCbCr(buffers, *vctx.decoder, width, height); - published = true; - } - } else { - WARN_LOG(Log::ME, "sceVideocodecDecode: type 0 without a usable buffer list"); - } - } else { - // Type 1 hands back plain planar YUV, so just point at the decoder's own planes - - // nothing in the descriptor is read until the caller copies from them. - out32(OUT_WIDTH_Y, width); - out32(OUT_WIDTH_CR, width / 2); - out32(OUT_WIDTH_CB, width / 2); - } + // Type 1 hands back plain planar YUV, so just point at the decoder's own planes - nothing + // in the descriptor is read until the caller copies from them. + out32(OUT_WIDTH_Y, width); + out32(OUT_WIDTH_CR, width / 2); + out32(OUT_WIDTH_CB, width / 2); } - - if (type == 0) { - out32(32, published ? 1 : 0); // images decoded - mpeg.prx won't convert without this - out32(36, published ? 0 : 1); - } - - // The decode takes real time on the ME: about 3.4ms for a 480x272 frame on a PSP, measured as - // sceMpegAvcDecode (5.8ms) less sceMpegAvcCsc alone (2.4ms), in pspautotests - // video/mpeg/playertiming. Movie players that present every decoded frame after a single - // vblank wait rely on decode, colour conversion and blit adding up to more than a vblank. - // It takes as long when the decoder holds the picture back: the first sceMpegAvcDecode calls of - // a stream, which return none, took 4.2-5.4ms in video/mp4/mp4timing. Until a picture has told - // us the size, assume full screen. - int delayUs = 0; - if (auBytes > 0) { - const int w = vctx.decoder->Width() > 0 ? vctx.decoder->Width() : 480; - const int h = vctx.decoder->Height() > 0 ? vctx.decoder->Height() : 272; - delayUs = MEScheduleJob(PowerScaleFromDefaultClock((int)(3400LL * w * h / (480 * 272)))); - } - - if (delayUs > 0) { - return hleDelayResult(hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %s %dx%d", - type, auBytes, gotFrame ? "frame" : "no frame yet", width, height), "videocodec decode", delayUs); - } - return hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %s %dx%d", - type, auBytes, gotFrame ? "frame" : "no frame yet", width, height); + return VideocodecDecodeResult(vctx, type, auBytes, gotFrame ? 1 : 0); } // Returning immediately from Stop or Delete is not correct, because a game can be relying on a @@ -673,10 +721,19 @@ static int sceVideocodecDecode(u32 ctxAddr, int type) { // already freed. One audio mix block is 64 samples at 44100Hz, about 1.45ms, which Delete's // measured time is well above. +// Stopping hands back the pictures the decoder was holding, once: on a PSP the first +// sceMpegAvcDecodeStop after a stream reports them, all at once, and the next ones none +// (pspautotests video/mpeg/decodeend, video/mp4/avchold). static int sceVideocodecStop(u32 ctxAddr, int type) { auto it = g_videocodecCtxs.find(ctxAddr); if (it != g_videocodecCtxs.end() && it->second.decoder) { - it->second.decoder->Flush(); + VideocodecCtx &vctx = it->second; + const u32 outAddr = Memory::IsValidRange(ctxAddr, 96) ? Memory::ReadUnchecked_U32(ctxAddr + CTX_OUT_INFO) : 0; + if (type == 0 && Memory::IsValidRange(outAddr, 108)) { + vctx.decoder->StopPictures(); + PublishPictures(vctx, ctxAddr, outAddr); + } + vctx.decoder->Flush(); } return MECall(hleLogInfo(Log::ME, 0), "videocodec stop", stopUs); } diff --git a/Core/HW/AvcDecoder.cpp b/Core/HW/AvcDecoder.cpp index 4627112440..9b787ca40b 100644 --- a/Core/HW/AvcDecoder.cpp +++ b/Core/HW/AvcDecoder.cpp @@ -17,6 +17,9 @@ #include "ppsspp_config.h" +#include +#include + #include "Common/Log.h" #include "Core/HW/AvcDecoder.h" #include "Core/HW/MediaEngine.h" @@ -29,6 +32,28 @@ extern "C" { #include "Core/FFMPEGCompat.h" #endif // USE_FFMPEG +// When the PSP's decoder hands a picture back. +// +// Measured in pspautotests video/mp4/avchold and video/mpeg/decodeend, on streams that differ only +// in their headers. The decoder follows the output process of the H.264 standard (Annex C): +// +// - A stream with HRD parameters and picture timing SEI gets each picture output dpb_output_delay +// after its access unit is removed from the coded picture buffer - one decode call per access +// unit. Sony's encoder writes these: its 30fps movies, without B-frames, come out a call late. +// - Any other stream is "bumped" out of a picture buffer three frames deep, whatever the stream +// says it needs: max_num_ref_frames, max_dec_frame_buffering, max_num_reorder_frames, size and +// level make no difference. So a stream without B-frames comes out three calls late. +// +// sceMpegAvcDecodeStop then hands back what's left. Streams with more than three reference frames +// are refused (0x80628002). +// +// ffmpeg only provides the pixels, in display order: each access unit gets its index as the +// packet's timestamp, so every frame can be matched to its unit, and the picture timing comes from +// ffmpeg's H.264 parser. The one thing this doesn't follow is bumping with B-frames and no picture +// timing: there the PSP's buffer also holds reference pictures that were already output, so a +// call can complete two or three pictures at once (only the first lands in the frame buffer). +// Here those come out one a call. Sony's movies have picture timing, so that's other encoders'. + AvcDecoder::AvcDecoder() { #ifdef USE_FFMPEG AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); @@ -48,6 +73,11 @@ AvcDecoder::AvcDecoder() { // The PSP hands us whole access units, so no parser is needed and no truncation is expected. codecCtx_->flags = 0; codecCtx_->flags2 = 0; +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 48, 101) + // DecodePictures keeps frames past the next decode, which the old API only allows for + // reference counted ones. The new one always hands those out. + codecCtx_->refcounted_frames = 1; +#endif if (avcodec_open2(codecCtx_, codec, nullptr) < 0) { ERROR_LOG(Log::ME, "AvcDecoder: couldn't open the H.264 decoder"); avcodec_free_context(&codecCtx_); @@ -58,12 +88,33 @@ AvcDecoder::AvcDecoder() { if (!frame_) { avcodec_free_context(&codecCtx_); codecCtx_ = nullptr; + return; + } + + parserCtx_ = avcodec_alloc_context3(nullptr); + parser_ = av_parser_init(AV_CODEC_ID_H264); + if (parser_) { + // Each packet is a whole access unit, so there's no need to wait for the next to find + // where this one ends. + parser_->flags |= PARSER_FLAG_COMPLETE_FRAMES; + } else { + WARN_LOG(Log::ME, "AvcDecoder: no H.264 parser in this ffmpeg build, movies may come out early"); } #endif } AvcDecoder::~AvcDecoder() { + ClearPictures(); #ifdef USE_FFMPEG + for (Decoded &d : decoded_) { + av_frame_free(&d.frame); + } + if (parser_) { + av_parser_close(parser_); + } + if (parserCtx_) { + avcodec_free_context(&parserCtx_); + } if (frame_) { av_frame_free(&frame_); } @@ -81,8 +132,19 @@ bool AvcDecoder::IsAvailable() { #endif } +void AvcDecoder::ClearPictures() { + pictures_.clear(); +#ifdef USE_FFMPEG + for (AVFrame *&frame : outFrames_) { + av_frame_free(&frame); + } +#endif + outFrames_.clear(); +} + void AvcDecoder::Flush() { haveFrame_ = false; + ClearPictures(); #ifdef USE_FFMPEG if (codecCtx_) { avcodec_flush_buffers(codecCtx_); @@ -90,6 +152,55 @@ void AvcDecoder::Flush() { if (frame_) { av_frame_unref(frame_); } + for (Decoded &d : decoded_) { + av_frame_free(&d.frame); + } + // The parser keeps the parameter sets and timing of the stream it saw, and has no reset. + if (parser_) { + av_parser_close(parser_); + parser_ = av_parser_init(AV_CODEC_ID_H264); + if (parser_) { + parser_->flags |= PARSER_FLAG_COMPLETE_FRAMES; + } + } +#endif + decoded_.clear(); + outputTicks_.clear(); + bufferingTick_ = 0; + firstTiming_ = true; + nextIndex_ = 0; + released_ = 0; +} + +bool AvcDecoder::AcceptFrame(AVFrame *frame) { +#ifdef USE_FFMPEG + // Check for decode errors. Concealing means covering up missing data, and to us, that is a bad error + // as we're normally streaming from disk. + if (frame->decode_error_flags || (frame->flags & AV_FRAME_FLAG_CORRUPT)) { + concealedFrames_++; + if (!wasConcealing_) { + WARN_LOG(Log::ME, "AvcDecoder: frame %d decoded from a damaged bitstream (flags %08x%s) - " + "the stream is missing data, not the decoder", frameCount_, frame->decode_error_flags, + (frame->flags & AV_FRAME_FLAG_CORRUPT) ? ", corrupt" : ""); + wasConcealing_ = true; + } + } else if (wasConcealing_) { + WARN_LOG(Log::ME, "AvcDecoder: clean again at frame %d, %d frames were concealed", + frameCount_, concealedFrames_); + wasConcealing_ = false; + } + frameCount_++; + + // Everything downstream indexes the three planes as 8-bit 4:2:0, which is all the PSP's + // encoder produced. Anything else would be read as if it were, so say so and drop the frame + // rather than converting garbage. + if (frame->format != AV_PIX_FMT_YUV420P && frame->format != AV_PIX_FMT_YUVJ420P) { + WARN_LOG(Log::ME, "AvcDecoder: unexpected pixel format %d, ignoring the frame", frame->format); + return false; + } + return frame->width > 0 && frame->height > 0; +#else + return false; #endif } @@ -160,39 +271,180 @@ bool AvcDecoder::Decode(const u8 *data, int size) { } #endif - // Check for decode errors. Concealing means covering up missing data, and to us, that is a bad error - // as we're normally streaming from disk. - if (frame_->decode_error_flags || (frame_->flags & AV_FRAME_FLAG_CORRUPT)) { - concealedFrames_++; - if (!wasConcealing_) { - WARN_LOG(Log::ME, "AvcDecoder: frame %d decoded from a damaged bitstream (flags %08x%s) - " - "the stream is missing data, not the decoder", frameCount_, frame_->decode_error_flags, - (frame_->flags & AV_FRAME_FLAG_CORRUPT) ? ", corrupt" : ""); - wasConcealing_ = true; - } - } else if (wasConcealing_) { - WARN_LOG(Log::ME, "AvcDecoder: clean again at frame %d, %d frames were concealed", - frameCount_, concealedFrames_); - wasConcealing_ = false; - } - frameCount_++; - - // Everything downstream indexes the three planes as 8-bit 4:2:0, which is all the PSP's - // encoder produced. Anything else would be read as if it were, so say so and drop the frame - // rather than converting garbage. - if (frame_->format != AV_PIX_FMT_YUV420P && frame_->format != AV_PIX_FMT_YUVJ420P) { - WARN_LOG(Log::ME, "AvcDecoder: unexpected pixel format %d, ignoring the frame", frame_->format); + if (!AcceptFrame(frame_)) { return false; } width_ = frame_->width; height_ = frame_->height; - haveFrame_ = width_ > 0 && height_ > 0; - return haveFrame_; + haveFrame_ = true; + return true; #else return false; #endif } +void AvcDecoder::FeedFfmpeg(const u8 *data, int size, int index) { +#ifdef USE_FFMPEG + AVPacket packet; + av_init_packet(&packet); + if (data) { + packetBuf_.resize((size_t)size + AV_INPUT_BUFFER_PADDING_SIZE); + memcpy(packetBuf_.data(), data, size); + memset(packetBuf_.data() + size, 0, AV_INPUT_BUFFER_PADDING_SIZE); + packet.data = packetBuf_.data(); + packet.size = size; + // Only a pts: with no dts to prefer, ffmpeg's best effort timestamp is this, whatever + // order the frames come back in. + packet.pts = index; + } else { + packet.data = nullptr; + packet.size = 0; + } + packet.dts = AV_NOPTS_VALUE; + + auto queue = [&](AVFrame *frame) { +#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 58, 100) + const s64 pts = frame->best_effort_timestamp; +#else + const s64 pts = av_frame_get_best_effort_timestamp(frame); +#endif + if (pts == AV_NOPTS_VALUE || !AcceptFrame(frame)) { + av_frame_unref(frame); + return; + } + AVFrame *copy = av_frame_alloc(); + av_frame_move_ref(copy, frame); + decoded_.push_back({ (int)pts, copy }); + }; + +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101) + int err = avcodec_send_packet(codecCtx_, data ? &packet : nullptr); + while (avcodec_receive_frame(codecCtx_, frame_) >= 0) { + queue(frame_); + } + if (err == AVERROR(EAGAIN)) { + // Everything waiting was taken out just above, so there's room now. + err = avcodec_send_packet(codecCtx_, &packet); + while (avcodec_receive_frame(codecCtx_, frame_) >= 0) { + queue(frame_); + } + } + if (err < 0 && err != AVERROR_EOF && data) { + WARN_LOG(Log::ME, "AvcDecoder: send_packet failed (%d)", err); + } +#else + for (int n = 0; n < 32; n++) { + int gotFrame = 0; + const int err = avcodec_decode_video2(codecCtx_, frame_, &gotFrame, &packet); + if (err < 0 && data) { + WARN_LOG(Log::ME, "AvcDecoder: decode_video2 failed (%d)", err); + } + if (gotFrame) { + queue(frame_); + } + // Draining hands back one frame per call; decoding, one at most. + if (data || !gotFrame) { + break; + } + } +#endif + if (!data) { + avcodec_flush_buffers(codecCtx_); + } +#endif +} + +void AvcDecoder::Release() { +#ifdef USE_FFMPEG + const Decoded d = decoded_.front(); + decoded_.pop_front(); + outputTicks_.erase(d.index); + released_++; + outFrames_.push_back(d.frame); + AvcPicture pic; + for (int i = 0; i < 3; i++) { + pic.planes[i] = d.frame->data[i]; + pic.strides[i] = d.frame->linesize[i]; + } + pic.width = d.frame->width; + pic.height = d.frame->height; + pictures_.push_back(pic); +#endif +} + +int AvcDecoder::DecodePictures(const u8 *data, int size) { + ClearPictures(); +#ifdef USE_FFMPEG + if (!codecCtx_ || !data || size <= 0) { + return 0; + } + const int index = nextIndex_++; + + // The picture timing SEI, in clock ticks from the first buffering period. The parser reports + // cpb_removal_delay and dpb_output_delay here, and whether this unit starts a buffering period. + bool timed = false; + s64 removalTick = 0; + if (parser_) { + u8 *out = nullptr; + int outSize = 0; + av_parser_parse2(parser_, parserCtx_, &out, &outSize, data, size, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + if (parser_->dts_ref_dts_delta != INT_MIN && parser_->pts_dts_delta != INT_MIN) { + timed = true; + if (parser_->dts_sync_point > 0) { + bufferingTick_ = firstTiming_ ? 0 : bufferingTick_ + parser_->dts_ref_dts_delta; + removalTick = bufferingTick_; + } else { + removalTick = bufferingTick_ + parser_->dts_ref_dts_delta; + } + firstTiming_ = false; + outputTicks_[index] = removalTick + parser_->pts_dts_delta; + } + } + + FeedFfmpeg(data, size, index); + if (codecCtx_->refs > 3) { + // Known once ffmpeg has seen the parameter sets. + for (Decoded &d : decoded_) { + av_frame_free(&d.frame); + } + decoded_.clear(); + return -1; + } + + if (timed) { + // Everything whose time has come, in display order - including this picture, if its delay + // is zero. A picture ffmpeg is still holding comes out late. + while (!decoded_.empty()) { + auto it = outputTicks_.find(decoded_.front().index); + if (it != outputTicks_.end() && it->second > removalTick) { + break; + } + Release(); + } + } else { + // The buffer holds three pictures. + while (!decoded_.empty() && nextIndex_ - released_ > 3) { + Release(); + } + } + return (int)pictures_.size(); +#else + return 0; +#endif +} + +int AvcDecoder::StopPictures() { + ClearPictures(); + if (!codecCtx_) { + return 0; + } + FeedFfmpeg(nullptr, 0, -1); + while (!decoded_.empty()) { + Release(); + } + return (int)pictures_.size(); +} + const u8 *AvcDecoder::Plane(int index) const { #ifdef USE_FFMPEG if (!haveFrame_ || !frame_ || index < 0 || index >= 3) { diff --git a/Core/HW/AvcDecoder.h b/Core/HW/AvcDecoder.h index 5724f3d5a7..f2affa6e9d 100644 --- a/Core/HW/AvcDecoder.h +++ b/Core/HW/AvcDecoder.h @@ -17,21 +17,32 @@ #pragma once +#include +#include #include #include "Common/CommonTypes.h" struct AVCodecContext; +struct AVCodecParserContext; struct AVFrame; -// A plain "hand it one access unit, get a frame back" H.264 decoder. -// -// This exists for sceVideocodec, which is a low level interface to the PSP Media Engine. +// One decoded picture: three 8-bit 4:2:0 planes (Y, Cb, Cr). Strides are in bytes and are not +// necessarily the width. +struct AvcPicture { + const u8 *planes[3]{}; + int strides[3]{}; + int width = 0; + int height = 0; +}; + +// An H.264 decoder for sceVideocodec, which is a low level interface to the PSP Media Engine. // PPSSPP's MediaEngine can't serve that - it is built around sceMpeg's own model, with the // PSMF demuxer, the ringbuffer and its own frame pacing wrapped around the decoder, so this // is a second path in addition to that. Implementing the old path around this might happen later. // -// Output is whatever pixel format the decoder produced, normally YUV420P; the caller converts. +// There are two ways in. Decode is "hand it one access unit, get a frame back when ffmpeg has +// one". DecodePictures hands pictures back when the PSP's decoder would - see AvcDecoder.cpp. class AvcDecoder { public: AvcDecoder(); @@ -45,6 +56,17 @@ public: // is not produced for every unit fed in, and that is not an error. bool Decode(const u8 *data, int size); + // Decodes one access unit and returns how many pictures the PSP's decoder hands back for it, + // which Pictures() then holds - none, one, or several. -1 for a stream the PSP's decoder + // refuses. + int DecodePictures(const u8 *data, int size); + // The end of the stream: hands back everything the decoder still holds, as stopping the PSP's + // decoder does. + int StopPictures(); + // The pictures of the last DecodePictures or StopPictures call, in output order. Valid until the + // next call. + const std::vector &Pictures() const { return pictures_; } + // Throws away any decoder state, for a seek or a new stream. void Flush(); @@ -61,6 +83,15 @@ public: static bool IsAvailable(); private: + // Feeds one packet to ffmpeg (null to drain at the end), and queues every frame that comes out + // under its access unit's index. + void FeedFfmpeg(const u8 *data, int size, int index); + // Checks a frame ffmpeg returned, and says whether it can be used. + bool AcceptFrame(AVFrame *frame); + // Hands back the frame at the front of the queue. + void Release(); + void ClearPictures(); + AVCodecContext *codecCtx_ = nullptr; AVFrame *frame_ = nullptr; // The access unit, copied so we can pad it - ffmpeg reads a little past the end of a packet. @@ -73,4 +104,23 @@ private: int frameCount_ = 0; int width_ = 0; int height_ = 0; + + // DecodePictures. The parser reads the picture timing SEI; it gets a context of its own so it + // can't change the decoder's. + AVCodecParserContext *parser_ = nullptr; + AVCodecContext *parserCtx_ = nullptr; + struct Decoded { + int index; + AVFrame *frame; + }; + // What ffmpeg has decoded and not been handed back yet, in output order. + std::deque decoded_; + // When each access unit's picture is due, in clock ticks, for streams with picture timing. + std::map outputTicks_; + s64 bufferingTick_ = 0; + bool firstTiming_ = true; + int nextIndex_ = 0; + int released_ = 0; + std::vector outFrames_; + std::vector pictures_; }; diff --git a/pspautotests b/pspautotests index 071a2a6f14..2533d2d44c 160000 --- a/pspautotests +++ b/pspautotests @@ -1 +1 @@ -Subproject commit 071a2a6f14477b8b186bd53201e2f2c0adc80679 +Subproject commit 2533d2d44c94de8808024bf079111c44accc9965 diff --git a/test.py b/test.py index b82d652e35..12595683f2 100755 --- a/test.py +++ b/test.py @@ -659,7 +659,9 @@ tests_next = [ #"utility/savedata/saveemptyfilename", "utility/savedata/secureversion", "utility/savedata/sizes", + "video/mp4/avchold", "video/mpeg/basic", + "video/mpeg/decodeend", "video/mpeg/deletetiming", "video/pmf/pmf", "video/pmf_simple/pmf_simple",