diff --git a/Core/HLE/sceVideocodec.cpp b/Core/HLE/sceVideocodec.cpp index 0ea5caaaf3..2e8ab645cc 100644 --- a/Core/HLE/sceVideocodec.cpp +++ b/Core/HLE/sceVideocodec.cpp @@ -273,38 +273,47 @@ u32 VideocodecFrameBufferLayout(int width, int height, int sizes[8], u32 offsets return total; } +// How many pictures one call can hand back. The decoder holds three back at most, so a call can +// complete them and its own; mpeg.prx reads one descriptor per picture, 0x2C bytes apart. +static const int MAX_PICTURES = 4; +static const u32 PICTURE_DESCRIPTOR_SIZE = 0x2C; + // The descriptor mpeg.prx passes in is empty: on hardware the ME owns the frame buffers and -// reports where it put them. So allocate them here and fill in the eight buffer addresses. -static bool PublishFrameBuffers(VideocodecCtx &vctx, u32 structAddr, int width, int height, u32 buffers[8]) { +// reports where it put them. So allocate them here and fill in the eight buffer addresses, for +// the slot'th picture of the call. +static bool PublishFrameBuffers(VideocodecCtx &vctx, u32 structAddr, int width, int height, int slot, u32 buffers[8]) { u32 offsets[8]; const u32 total = VideocodecFrameBufferLayout(width, height, nullptr, offsets); - if (total == 0) { + if (total == 0 || slot < 0 || slot >= MAX_PICTURES) { return false; } - if (vctx.frameBuffers && (width != vctx.frameBufferWidth || height != vctx.frameBufferHeight)) { + // A savestate from before frame buffers had a slot per picture carries a block with room for one. + if (vctx.frameBuffers && (width != vctx.frameBufferWidth || height != vctx.frameBufferHeight || + vctx.frameBuffersSize < total * MAX_PICTURES)) { g_meAlloc.Free(vctx.frameBuffers); vctx.frameBuffers = 0; } if (!vctx.frameBuffers) { MEEnsureRam(); - u32 size = total; + u32 size = total * MAX_PICTURES; vctx.frameBuffers = g_meAlloc.Alloc(size, false, "VideocodecFrame"); if (vctx.frameBuffers == (u32)-1) { vctx.frameBuffers = 0; - ERROR_LOG(Log::ME, "sceVideocodec: no room in ME memory for %d bytes of frame buffers", total); + ERROR_LOG(Log::ME, "sceVideocodec: no room in ME memory for %d bytes of frame buffers", total * MAX_PICTURES); return false; } - vctx.frameBuffersSize = total; + vctx.frameBuffersSize = total * MAX_PICTURES; vctx.frameBufferWidth = width; vctx.frameBufferHeight = height; INFO_LOG(Log::ME, "sceVideocodec: %d bytes of frame buffers at %08x for %dx%d", - total, vctx.frameBuffers, width, height); + total * MAX_PICTURES, vctx.frameBuffers, width, height); } for (int i = 0; i < 8; i++) { - buffers[i] = vctx.frameBuffers + offsets[i]; + buffers[i] = vctx.frameBuffers + slot * total + offsets[i]; } + structAddr += slot * PICTURE_DESCRIPTOR_SIZE; // mpeg.prx reads eight buffer addresses off the front of this structure (`lw` at 0x00..0x1C, // verified in 1.3 at 08805698 and 1.8 at 08805898) and takes the frame dimensions from its own @@ -341,9 +350,16 @@ void VideocodecGetCtxInfo(std::vector *infos) { bool VideocodecGetFrameBuffers(u32 firstBuffer, u32 buffers[8], int *width, int *height) { // sceMpegbase only has the first of the eight addresses, so find whose allocation it is. const VideocodecCtx *found = nullptr; + u32 base = 0; for (const auto &[addr, ctx] : g_videocodecCtxs) { - if (ctx.frameBuffers && ctx.frameBuffers == firstBuffer) { + if (!ctx.frameBuffers || firstBuffer < ctx.frameBuffers || firstBuffer >= ctx.frameBuffers + ctx.frameBuffersSize) { + continue; + } + // One of the slots of the pictures a call handed back. + const u32 total = VideocodecFrameBufferLayout(ctx.frameBufferWidth, ctx.frameBufferHeight, nullptr, nullptr); + if (total != 0 && (firstBuffer - ctx.frameBuffers) % total == 0) { found = &ctx; + base = firstBuffer; break; } } @@ -359,7 +375,7 @@ bool VideocodecGetFrameBuffers(u32 firstBuffer, u32 buffers[8], int *width, int *height = found->frameBufferHeight; } for (int i = 0; i < 8; i++) { - buffers[i] = found->frameBuffers + offsets[i]; + buffers[i] = base + offsets[i]; } return true; } @@ -368,16 +384,18 @@ bool VideocodecGetFrameBuffers(u32 firstBuffer, u32 buffers[8], int *width, int // vertical bands split into two 16-pixel halves, and which buffer a row lands in depends on // whether it is even or odd. This is the exact inverse of ReadTiledYCbCr in sceMpeg.cpp, which // is what reads it back out - see the comment there for the full layout. -static void WriteTiledYCbCr(const u32 *buffers, const AvcDecoder &dec, int width, int height) { +static void WriteTiledYCbCr(const u32 *buffers, const AvcPicture &pic) { + const int width = pic.width; + const int height = pic.height; const int width2 = width >> 1; const int height2 = height >> 1; - const u8 *srcY = dec.Plane(0); - const u8 *srcCb = dec.Plane(1); - const u8 *srcCr = dec.Plane(2); - const int strideY = dec.Stride(0); - const int strideCb = dec.Stride(1); - const int strideCr = dec.Stride(2); + const u8 *srcY = pic.planes[0]; + const u8 *srcCb = pic.planes[1]; + const u8 *srcCr = pic.planes[2]; + const int strideY = pic.strides[0]; + const int strideCb = pic.strides[1]; + const int strideCr = pic.strides[2]; if (!srcY || !srcCb || !srcCr) { return; } @@ -536,6 +554,64 @@ static int sceVideocodecReleaseEDRAM(u32 ctxAddr) { return MECall(hleLogInfo(Log::ME, 0, "released %08x", token), "videocodec releaseedram", releaseEdramUs); } +// Type 0: copies the pictures the decoder handed back into the frame buffers the ME reports, +// and says how many in the output descriptor. mpeg.prx converts the first one in +// sceMpegAvcDecode, and all of them in sceMpegAvcDecodeStop (08805130 in 6.61's mpeg.prx reads a +// descriptor per picture, 0x2C bytes apart). Returns the number published. +static int PublishPictures(VideocodecCtx &vctx, u32 ctxAddr, u32 outAddr) { + const std::vector &pictures = vctx.decoder->Pictures(); + const u32 yuvStructAddr = Memory::ReadUnchecked_U32(ctxAddr + CTX_YUV_STRUCT); + int published = 0; + if (!pictures.empty() && !Memory::IsValidRange(yuvStructAddr, 8 * 4)) { + WARN_LOG(Log::ME, "sceVideocodec: type 0 without a usable buffer list"); + } else { + for (const AvcPicture &pic : pictures) { + u32 buffers[8]; + if (published >= MAX_PICTURES || !PublishFrameBuffers(vctx, yuvStructAddr, pic.width, pic.height, published, buffers)) { + break; + } + WriteTiledYCbCr(buffers, pic); + published++; + } + } + if (published > 0) { + Memory::WriteUnchecked_U32(pictures[0].width, outAddr + 8); + Memory::WriteUnchecked_U32(pictures[0].height, outAddr + 12); + Memory::WriteUnchecked_U32(1, outAddr + 28); + } + // The count of images - mpeg.prx won't convert without it. + Memory::WriteUnchecked_U32(published, outAddr + 32); + Memory::WriteUnchecked_U32(published ? 0 : 1, outAddr + 36); + vctx.frameCount += published; + return published; +} + +// The decode takes real time on the ME: about 3.4ms for a 480x272 frame on a PSP, measured as +// sceMpegAvcDecode (5.8ms) less sceMpegAvcCsc alone (2.4ms), in pspautotests +// video/mpeg/playertiming. Movie players that present every decoded frame after a single +// vblank wait rely on decode, colour conversion and blit adding up to more than a vblank. +// It takes as long when the decoder holds the picture back: the first sceMpegAvcDecode calls of +// a stream, which return none, took 4.2-5.4ms in video/mp4/mp4timing. Until a picture has told +// us the size, assume full screen. +static int VideocodecDecodeResult(const VideocodecCtx &vctx, int type, int auBytes, int pictures) { + int delayUs = 0; + if (auBytes > 0) { + int w = vctx.frameBufferWidth > 0 ? vctx.frameBufferWidth : vctx.decoder->Width(); + int h = vctx.frameBufferHeight > 0 ? vctx.frameBufferHeight : vctx.decoder->Height(); + if (w <= 0 || h <= 0) { + w = 480; + h = 272; + } + delayUs = MEScheduleJob(PowerScaleFromDefaultClock((int)(3400LL * w * h / (480 * 272)))); + } + + if (delayUs > 0) { + return hleDelayResult(hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %d picture(s)", + type, auBytes, pictures), "videocodec decode", delayUs); + } + return hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %d picture(s)", type, auBytes, pictures); +} + static int sceVideocodecDecode(u32 ctxAddr, int type) { if (!Memory::IsValidRange(ctxAddr, 96)) { return hleLogError(Log::ME, -1, "bad context pointer"); @@ -580,6 +656,33 @@ static int sceVideocodecDecode(u32 ctxAddr, int type) { auSize, auAddr); } } + auto out32 = [outAddr](int offset, u32 value) { + Memory::WriteUnchecked_U32(value, outAddr + offset); + }; + + if (type == 0) { + // Pictures come out when the PSP's decoder would hand them back - see AvcDecoder.cpp. + // Only the type 1 path fills the rest of the descriptor in: for type 0 the YCbCr + // descriptor sits just 0x40 bytes after this one - mpeg.prx allocates them adjacently - so + // writing the type 1 fields here scribbles over the buffer addresses the colour conversion + // is about to read. Nor is a picture announced until the buffers holding it have been + // published: saying "one image decoded" and then failing to allocate would have mpeg.prx + // convert from whatever the descriptor pointed at last. + int pictures = 0; + if (au && auBytes > 0) { + if (vctx.decoder->DecodePictures(au, auBytes) < 0) { + // The PSP refuses streams with more than three reference frames, and mpeg.prx + // passes this on from sceMpegAvcDecode (pspautotests video/mp4/avchold). + return hleLogWarning(Log::ME, SCE_MPEG_ERROR_AVC_DECODE_FATAL, "stream the PSP's decoder can't play"); + } + pictures = PublishPictures(vctx, ctxAddr, outAddr); + } else { + out32(32, 0); + out32(36, 1); + } + return VideocodecDecodeResult(vctx, type, auBytes, pictures); + } + if (au && auBytes > 0) { gotFrame = vctx.decoder->Decode(au, auBytes); } @@ -587,82 +690,27 @@ static int sceVideocodecDecode(u32 ctxAddr, int type) { const int width = gotFrame ? vctx.decoder->Width() : 0; const int height = gotFrame ? vctx.decoder->Height() : 0; - auto out32 = [outAddr](int offset, u32 value) { - Memory::WriteUnchecked_U32(value, outAddr + offset); - }; - - // Only the type 1 path fills the descriptor in. For type 0 the YCbCr descriptor sits just - // 0x40 bytes after this one - mpeg.prx allocates them adjacently - so writing the type 1 - // fields here scribbles over the buffer addresses the colour conversion is about to read. - // For type 0 the frame isn't announced until the buffers holding it have been published - - // see below. Saying "one image decoded" and then failing to allocate would have mpeg.prx - // convert from whatever the descriptor pointed at last. - bool published = false; - if (type == 0) { - out32(8, width); - out32(12, height); - out32(28, 1); - } else { - out32(OUT_DATA, auAddr); - out32(OUT_SIZE, auSize); - out32(OUT_UNK12, 0x40); - out32(OUT_CONSUMED, auSize); - out32(OUT_WIDTH, width); - out32(OUT_HEIGHT, height); - out32(OUT_FRAME_READY, gotFrame ? 2 : 1); - out32(OUT_UNK64, 1); - out32(OUT_UNK72, (u32)-1); - out32(OUT_TIMESTAMP, vctx.frameCount * 0x64); - out32(OUT_FPS, 2997); - } + out32(OUT_DATA, auAddr); + out32(OUT_SIZE, auSize); + out32(OUT_UNK12, 0x40); + out32(OUT_CONSUMED, auSize); + out32(OUT_WIDTH, width); + out32(OUT_HEIGHT, height); + out32(OUT_FRAME_READY, gotFrame ? 2 : 1); + out32(OUT_UNK64, 1); + out32(OUT_UNK72, (u32)-1); + out32(OUT_TIMESTAMP, vctx.frameCount * 0x64); + out32(OUT_FPS, 2997); if (gotFrame) { vctx.frameCount++; - if (type == 0) { - const u32 yuvStructAddr = Memory::ReadUnchecked_U32(ctxAddr + CTX_YUV_STRUCT); - if (Memory::IsValidRange(yuvStructAddr, 8 * 4)) { - u32 buffers[8]; - if (PublishFrameBuffers(vctx, yuvStructAddr, width, height, buffers)) { - WriteTiledYCbCr(buffers, *vctx.decoder, width, height); - published = true; - } - } else { - WARN_LOG(Log::ME, "sceVideocodecDecode: type 0 without a usable buffer list"); - } - } else { - // Type 1 hands back plain planar YUV, so just point at the decoder's own planes - - // nothing in the descriptor is read until the caller copies from them. - out32(OUT_WIDTH_Y, width); - out32(OUT_WIDTH_CR, width / 2); - out32(OUT_WIDTH_CB, width / 2); - } + // Type 1 hands back plain planar YUV, so just point at the decoder's own planes - nothing + // in the descriptor is read until the caller copies from them. + out32(OUT_WIDTH_Y, width); + out32(OUT_WIDTH_CR, width / 2); + out32(OUT_WIDTH_CB, width / 2); } - - if (type == 0) { - out32(32, published ? 1 : 0); // images decoded - mpeg.prx won't convert without this - out32(36, published ? 0 : 1); - } - - // The decode takes real time on the ME: about 3.4ms for a 480x272 frame on a PSP, measured as - // sceMpegAvcDecode (5.8ms) less sceMpegAvcCsc alone (2.4ms), in pspautotests - // video/mpeg/playertiming. Movie players that present every decoded frame after a single - // vblank wait rely on decode, colour conversion and blit adding up to more than a vblank. - // It takes as long when the decoder holds the picture back: the first sceMpegAvcDecode calls of - // a stream, which return none, took 4.2-5.4ms in video/mp4/mp4timing. Until a picture has told - // us the size, assume full screen. - int delayUs = 0; - if (auBytes > 0) { - const int w = vctx.decoder->Width() > 0 ? vctx.decoder->Width() : 480; - const int h = vctx.decoder->Height() > 0 ? vctx.decoder->Height() : 272; - delayUs = MEScheduleJob(PowerScaleFromDefaultClock((int)(3400LL * w * h / (480 * 272)))); - } - - if (delayUs > 0) { - return hleDelayResult(hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %s %dx%d", - type, auBytes, gotFrame ? "frame" : "no frame yet", width, height), "videocodec decode", delayUs); - } - return hleLogDebug(Log::ME, 0, "type %d, %d bytes -> %s %dx%d", - type, auBytes, gotFrame ? "frame" : "no frame yet", width, height); + return VideocodecDecodeResult(vctx, type, auBytes, gotFrame ? 1 : 0); } // Returning immediately from Stop or Delete is not correct, because a game can be relying on a @@ -673,10 +721,19 @@ static int sceVideocodecDecode(u32 ctxAddr, int type) { // already freed. One audio mix block is 64 samples at 44100Hz, about 1.45ms, which Delete's // measured time is well above. +// Stopping hands back the pictures the decoder was holding, once: on a PSP the first +// sceMpegAvcDecodeStop after a stream reports them, all at once, and the next ones none +// (pspautotests video/mpeg/decodeend, video/mp4/avchold). static int sceVideocodecStop(u32 ctxAddr, int type) { auto it = g_videocodecCtxs.find(ctxAddr); if (it != g_videocodecCtxs.end() && it->second.decoder) { - it->second.decoder->Flush(); + VideocodecCtx &vctx = it->second; + const u32 outAddr = Memory::IsValidRange(ctxAddr, 96) ? Memory::ReadUnchecked_U32(ctxAddr + CTX_OUT_INFO) : 0; + if (type == 0 && Memory::IsValidRange(outAddr, 108)) { + vctx.decoder->StopPictures(); + PublishPictures(vctx, ctxAddr, outAddr); + } + vctx.decoder->Flush(); } return MECall(hleLogInfo(Log::ME, 0), "videocodec stop", stopUs); } diff --git a/Core/HW/AvcDecoder.cpp b/Core/HW/AvcDecoder.cpp index 4627112440..9b787ca40b 100644 --- a/Core/HW/AvcDecoder.cpp +++ b/Core/HW/AvcDecoder.cpp @@ -17,6 +17,9 @@ #include "ppsspp_config.h" +#include +#include + #include "Common/Log.h" #include "Core/HW/AvcDecoder.h" #include "Core/HW/MediaEngine.h" @@ -29,6 +32,28 @@ extern "C" { #include "Core/FFMPEGCompat.h" #endif // USE_FFMPEG +// When the PSP's decoder hands a picture back. +// +// Measured in pspautotests video/mp4/avchold and video/mpeg/decodeend, on streams that differ only +// in their headers. The decoder follows the output process of the H.264 standard (Annex C): +// +// - A stream with HRD parameters and picture timing SEI gets each picture output dpb_output_delay +// after its access unit is removed from the coded picture buffer - one decode call per access +// unit. Sony's encoder writes these: its 30fps movies, without B-frames, come out a call late. +// - Any other stream is "bumped" out of a picture buffer three frames deep, whatever the stream +// says it needs: max_num_ref_frames, max_dec_frame_buffering, max_num_reorder_frames, size and +// level make no difference. So a stream without B-frames comes out three calls late. +// +// sceMpegAvcDecodeStop then hands back what's left. Streams with more than three reference frames +// are refused (0x80628002). +// +// ffmpeg only provides the pixels, in display order: each access unit gets its index as the +// packet's timestamp, so every frame can be matched to its unit, and the picture timing comes from +// ffmpeg's H.264 parser. The one thing this doesn't follow is bumping with B-frames and no picture +// timing: there the PSP's buffer also holds reference pictures that were already output, so a +// call can complete two or three pictures at once (only the first lands in the frame buffer). +// Here those come out one a call. Sony's movies have picture timing, so that's other encoders'. + AvcDecoder::AvcDecoder() { #ifdef USE_FFMPEG AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); @@ -48,6 +73,11 @@ AvcDecoder::AvcDecoder() { // The PSP hands us whole access units, so no parser is needed and no truncation is expected. codecCtx_->flags = 0; codecCtx_->flags2 = 0; +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 48, 101) + // DecodePictures keeps frames past the next decode, which the old API only allows for + // reference counted ones. The new one always hands those out. + codecCtx_->refcounted_frames = 1; +#endif if (avcodec_open2(codecCtx_, codec, nullptr) < 0) { ERROR_LOG(Log::ME, "AvcDecoder: couldn't open the H.264 decoder"); avcodec_free_context(&codecCtx_); @@ -58,12 +88,33 @@ AvcDecoder::AvcDecoder() { if (!frame_) { avcodec_free_context(&codecCtx_); codecCtx_ = nullptr; + return; + } + + parserCtx_ = avcodec_alloc_context3(nullptr); + parser_ = av_parser_init(AV_CODEC_ID_H264); + if (parser_) { + // Each packet is a whole access unit, so there's no need to wait for the next to find + // where this one ends. + parser_->flags |= PARSER_FLAG_COMPLETE_FRAMES; + } else { + WARN_LOG(Log::ME, "AvcDecoder: no H.264 parser in this ffmpeg build, movies may come out early"); } #endif } AvcDecoder::~AvcDecoder() { + ClearPictures(); #ifdef USE_FFMPEG + for (Decoded &d : decoded_) { + av_frame_free(&d.frame); + } + if (parser_) { + av_parser_close(parser_); + } + if (parserCtx_) { + avcodec_free_context(&parserCtx_); + } if (frame_) { av_frame_free(&frame_); } @@ -81,8 +132,19 @@ bool AvcDecoder::IsAvailable() { #endif } +void AvcDecoder::ClearPictures() { + pictures_.clear(); +#ifdef USE_FFMPEG + for (AVFrame *&frame : outFrames_) { + av_frame_free(&frame); + } +#endif + outFrames_.clear(); +} + void AvcDecoder::Flush() { haveFrame_ = false; + ClearPictures(); #ifdef USE_FFMPEG if (codecCtx_) { avcodec_flush_buffers(codecCtx_); @@ -90,6 +152,55 @@ void AvcDecoder::Flush() { if (frame_) { av_frame_unref(frame_); } + for (Decoded &d : decoded_) { + av_frame_free(&d.frame); + } + // The parser keeps the parameter sets and timing of the stream it saw, and has no reset. + if (parser_) { + av_parser_close(parser_); + parser_ = av_parser_init(AV_CODEC_ID_H264); + if (parser_) { + parser_->flags |= PARSER_FLAG_COMPLETE_FRAMES; + } + } +#endif + decoded_.clear(); + outputTicks_.clear(); + bufferingTick_ = 0; + firstTiming_ = true; + nextIndex_ = 0; + released_ = 0; +} + +bool AvcDecoder::AcceptFrame(AVFrame *frame) { +#ifdef USE_FFMPEG + // Check for decode errors. Concealing means covering up missing data, and to us, that is a bad error + // as we're normally streaming from disk. + if (frame->decode_error_flags || (frame->flags & AV_FRAME_FLAG_CORRUPT)) { + concealedFrames_++; + if (!wasConcealing_) { + WARN_LOG(Log::ME, "AvcDecoder: frame %d decoded from a damaged bitstream (flags %08x%s) - " + "the stream is missing data, not the decoder", frameCount_, frame->decode_error_flags, + (frame->flags & AV_FRAME_FLAG_CORRUPT) ? ", corrupt" : ""); + wasConcealing_ = true; + } + } else if (wasConcealing_) { + WARN_LOG(Log::ME, "AvcDecoder: clean again at frame %d, %d frames were concealed", + frameCount_, concealedFrames_); + wasConcealing_ = false; + } + frameCount_++; + + // Everything downstream indexes the three planes as 8-bit 4:2:0, which is all the PSP's + // encoder produced. Anything else would be read as if it were, so say so and drop the frame + // rather than converting garbage. + if (frame->format != AV_PIX_FMT_YUV420P && frame->format != AV_PIX_FMT_YUVJ420P) { + WARN_LOG(Log::ME, "AvcDecoder: unexpected pixel format %d, ignoring the frame", frame->format); + return false; + } + return frame->width > 0 && frame->height > 0; +#else + return false; #endif } @@ -160,39 +271,180 @@ bool AvcDecoder::Decode(const u8 *data, int size) { } #endif - // Check for decode errors. Concealing means covering up missing data, and to us, that is a bad error - // as we're normally streaming from disk. - if (frame_->decode_error_flags || (frame_->flags & AV_FRAME_FLAG_CORRUPT)) { - concealedFrames_++; - if (!wasConcealing_) { - WARN_LOG(Log::ME, "AvcDecoder: frame %d decoded from a damaged bitstream (flags %08x%s) - " - "the stream is missing data, not the decoder", frameCount_, frame_->decode_error_flags, - (frame_->flags & AV_FRAME_FLAG_CORRUPT) ? ", corrupt" : ""); - wasConcealing_ = true; - } - } else if (wasConcealing_) { - WARN_LOG(Log::ME, "AvcDecoder: clean again at frame %d, %d frames were concealed", - frameCount_, concealedFrames_); - wasConcealing_ = false; - } - frameCount_++; - - // Everything downstream indexes the three planes as 8-bit 4:2:0, which is all the PSP's - // encoder produced. Anything else would be read as if it were, so say so and drop the frame - // rather than converting garbage. - if (frame_->format != AV_PIX_FMT_YUV420P && frame_->format != AV_PIX_FMT_YUVJ420P) { - WARN_LOG(Log::ME, "AvcDecoder: unexpected pixel format %d, ignoring the frame", frame_->format); + if (!AcceptFrame(frame_)) { return false; } width_ = frame_->width; height_ = frame_->height; - haveFrame_ = width_ > 0 && height_ > 0; - return haveFrame_; + haveFrame_ = true; + return true; #else return false; #endif } +void AvcDecoder::FeedFfmpeg(const u8 *data, int size, int index) { +#ifdef USE_FFMPEG + AVPacket packet; + av_init_packet(&packet); + if (data) { + packetBuf_.resize((size_t)size + AV_INPUT_BUFFER_PADDING_SIZE); + memcpy(packetBuf_.data(), data, size); + memset(packetBuf_.data() + size, 0, AV_INPUT_BUFFER_PADDING_SIZE); + packet.data = packetBuf_.data(); + packet.size = size; + // Only a pts: with no dts to prefer, ffmpeg's best effort timestamp is this, whatever + // order the frames come back in. + packet.pts = index; + } else { + packet.data = nullptr; + packet.size = 0; + } + packet.dts = AV_NOPTS_VALUE; + + auto queue = [&](AVFrame *frame) { +#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 58, 100) + const s64 pts = frame->best_effort_timestamp; +#else + const s64 pts = av_frame_get_best_effort_timestamp(frame); +#endif + if (pts == AV_NOPTS_VALUE || !AcceptFrame(frame)) { + av_frame_unref(frame); + return; + } + AVFrame *copy = av_frame_alloc(); + av_frame_move_ref(copy, frame); + decoded_.push_back({ (int)pts, copy }); + }; + +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101) + int err = avcodec_send_packet(codecCtx_, data ? &packet : nullptr); + while (avcodec_receive_frame(codecCtx_, frame_) >= 0) { + queue(frame_); + } + if (err == AVERROR(EAGAIN)) { + // Everything waiting was taken out just above, so there's room now. + err = avcodec_send_packet(codecCtx_, &packet); + while (avcodec_receive_frame(codecCtx_, frame_) >= 0) { + queue(frame_); + } + } + if (err < 0 && err != AVERROR_EOF && data) { + WARN_LOG(Log::ME, "AvcDecoder: send_packet failed (%d)", err); + } +#else + for (int n = 0; n < 32; n++) { + int gotFrame = 0; + const int err = avcodec_decode_video2(codecCtx_, frame_, &gotFrame, &packet); + if (err < 0 && data) { + WARN_LOG(Log::ME, "AvcDecoder: decode_video2 failed (%d)", err); + } + if (gotFrame) { + queue(frame_); + } + // Draining hands back one frame per call; decoding, one at most. + if (data || !gotFrame) { + break; + } + } +#endif + if (!data) { + avcodec_flush_buffers(codecCtx_); + } +#endif +} + +void AvcDecoder::Release() { +#ifdef USE_FFMPEG + const Decoded d = decoded_.front(); + decoded_.pop_front(); + outputTicks_.erase(d.index); + released_++; + outFrames_.push_back(d.frame); + AvcPicture pic; + for (int i = 0; i < 3; i++) { + pic.planes[i] = d.frame->data[i]; + pic.strides[i] = d.frame->linesize[i]; + } + pic.width = d.frame->width; + pic.height = d.frame->height; + pictures_.push_back(pic); +#endif +} + +int AvcDecoder::DecodePictures(const u8 *data, int size) { + ClearPictures(); +#ifdef USE_FFMPEG + if (!codecCtx_ || !data || size <= 0) { + return 0; + } + const int index = nextIndex_++; + + // The picture timing SEI, in clock ticks from the first buffering period. The parser reports + // cpb_removal_delay and dpb_output_delay here, and whether this unit starts a buffering period. + bool timed = false; + s64 removalTick = 0; + if (parser_) { + u8 *out = nullptr; + int outSize = 0; + av_parser_parse2(parser_, parserCtx_, &out, &outSize, data, size, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + if (parser_->dts_ref_dts_delta != INT_MIN && parser_->pts_dts_delta != INT_MIN) { + timed = true; + if (parser_->dts_sync_point > 0) { + bufferingTick_ = firstTiming_ ? 0 : bufferingTick_ + parser_->dts_ref_dts_delta; + removalTick = bufferingTick_; + } else { + removalTick = bufferingTick_ + parser_->dts_ref_dts_delta; + } + firstTiming_ = false; + outputTicks_[index] = removalTick + parser_->pts_dts_delta; + } + } + + FeedFfmpeg(data, size, index); + if (codecCtx_->refs > 3) { + // Known once ffmpeg has seen the parameter sets. + for (Decoded &d : decoded_) { + av_frame_free(&d.frame); + } + decoded_.clear(); + return -1; + } + + if (timed) { + // Everything whose time has come, in display order - including this picture, if its delay + // is zero. A picture ffmpeg is still holding comes out late. + while (!decoded_.empty()) { + auto it = outputTicks_.find(decoded_.front().index); + if (it != outputTicks_.end() && it->second > removalTick) { + break; + } + Release(); + } + } else { + // The buffer holds three pictures. + while (!decoded_.empty() && nextIndex_ - released_ > 3) { + Release(); + } + } + return (int)pictures_.size(); +#else + return 0; +#endif +} + +int AvcDecoder::StopPictures() { + ClearPictures(); + if (!codecCtx_) { + return 0; + } + FeedFfmpeg(nullptr, 0, -1); + while (!decoded_.empty()) { + Release(); + } + return (int)pictures_.size(); +} + const u8 *AvcDecoder::Plane(int index) const { #ifdef USE_FFMPEG if (!haveFrame_ || !frame_ || index < 0 || index >= 3) { diff --git a/Core/HW/AvcDecoder.h b/Core/HW/AvcDecoder.h index 5724f3d5a7..f2affa6e9d 100644 --- a/Core/HW/AvcDecoder.h +++ b/Core/HW/AvcDecoder.h @@ -17,21 +17,32 @@ #pragma once +#include +#include #include #include "Common/CommonTypes.h" struct AVCodecContext; +struct AVCodecParserContext; struct AVFrame; -// A plain "hand it one access unit, get a frame back" H.264 decoder. -// -// This exists for sceVideocodec, which is a low level interface to the PSP Media Engine. +// One decoded picture: three 8-bit 4:2:0 planes (Y, Cb, Cr). Strides are in bytes and are not +// necessarily the width. +struct AvcPicture { + const u8 *planes[3]{}; + int strides[3]{}; + int width = 0; + int height = 0; +}; + +// An H.264 decoder for sceVideocodec, which is a low level interface to the PSP Media Engine. // PPSSPP's MediaEngine can't serve that - it is built around sceMpeg's own model, with the // PSMF demuxer, the ringbuffer and its own frame pacing wrapped around the decoder, so this // is a second path in addition to that. Implementing the old path around this might happen later. // -// Output is whatever pixel format the decoder produced, normally YUV420P; the caller converts. +// There are two ways in. Decode is "hand it one access unit, get a frame back when ffmpeg has +// one". DecodePictures hands pictures back when the PSP's decoder would - see AvcDecoder.cpp. class AvcDecoder { public: AvcDecoder(); @@ -45,6 +56,17 @@ public: // is not produced for every unit fed in, and that is not an error. bool Decode(const u8 *data, int size); + // Decodes one access unit and returns how many pictures the PSP's decoder hands back for it, + // which Pictures() then holds - none, one, or several. -1 for a stream the PSP's decoder + // refuses. + int DecodePictures(const u8 *data, int size); + // The end of the stream: hands back everything the decoder still holds, as stopping the PSP's + // decoder does. + int StopPictures(); + // The pictures of the last DecodePictures or StopPictures call, in output order. Valid until the + // next call. + const std::vector &Pictures() const { return pictures_; } + // Throws away any decoder state, for a seek or a new stream. void Flush(); @@ -61,6 +83,15 @@ public: static bool IsAvailable(); private: + // Feeds one packet to ffmpeg (null to drain at the end), and queues every frame that comes out + // under its access unit's index. + void FeedFfmpeg(const u8 *data, int size, int index); + // Checks a frame ffmpeg returned, and says whether it can be used. + bool AcceptFrame(AVFrame *frame); + // Hands back the frame at the front of the queue. + void Release(); + void ClearPictures(); + AVCodecContext *codecCtx_ = nullptr; AVFrame *frame_ = nullptr; // The access unit, copied so we can pad it - ffmpeg reads a little past the end of a packet. @@ -73,4 +104,23 @@ private: int frameCount_ = 0; int width_ = 0; int height_ = 0; + + // DecodePictures. The parser reads the picture timing SEI; it gets a context of its own so it + // can't change the decoder's. + AVCodecParserContext *parser_ = nullptr; + AVCodecContext *parserCtx_ = nullptr; + struct Decoded { + int index; + AVFrame *frame; + }; + // What ffmpeg has decoded and not been handed back yet, in output order. + std::deque decoded_; + // When each access unit's picture is due, in clock ticks, for streams with picture timing. + std::map outputTicks_; + s64 bufferingTick_ = 0; + bool firstTiming_ = true; + int nextIndex_ = 0; + int released_ = 0; + std::vector outFrames_; + std::vector pictures_; }; diff --git a/pspautotests b/pspautotests index 071a2a6f14..2533d2d44c 160000 --- a/pspautotests +++ b/pspautotests @@ -1 +1 @@ -Subproject commit 071a2a6f14477b8b186bd53201e2f2c0adc80679 +Subproject commit 2533d2d44c94de8808024bf079111c44accc9965 diff --git a/test.py b/test.py index b82d652e35..12595683f2 100755 --- a/test.py +++ b/test.py @@ -659,7 +659,9 @@ tests_next = [ #"utility/savedata/saveemptyfilename", "utility/savedata/secureversion", "utility/savedata/sizes", + "video/mp4/avchold", "video/mpeg/basic", + "video/mpeg/decodeend", "video/mpeg/deletetiming", "video/pmf/pmf", "video/pmf_simple/pmf_simple",