From b1cb8489dfcead66df2a50897862d6005f417cdb Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sun, 23 Feb 2020 21:20:33 +1100 Subject: [PATCH] memory: try caching frameptrs instead of avframes so we have more control --- app/codec/ffmpeg/ffmpegdecoder.cpp | 146 +++++++++++--------------- app/codec/ffmpeg/ffmpegdecoder.h | 2 +- app/codec/ffmpeg/ffmpegframecache.cpp | 76 +++++++++----- app/codec/ffmpeg/ffmpegframecache.h | 59 +++++++---- app/codec/frame.cpp | 4 +- app/codec/frame.h | 6 +- 6 files changed, 158 insertions(+), 135 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 095be431f..c4dcd3867 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -49,7 +49,7 @@ FFmpegDecoder::FFmpegDecoder() : cache_at_eof_(false), opts_(nullptr) { - clear_timer_.setInterval(5000); + clear_timer_.setInterval(2500); connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent); } @@ -223,37 +223,32 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) int64_t target_ts = Timecode::time_to_timestamp(timecode, avstream_->time_base) + avstream_->start_time; - bool got_frame = false; - - uint8_t* input_data[4]; - int input_linesize[4]; + Frame* return_frame = nullptr; // See if our RAM cache already has a frame that matches this timestamp if (!cached_frames_.isEmpty()) { - AVFrame* found_frame = nullptr; + if (cache_at_zero_ && target_ts < cached_frames_.first()->native_timestamp()) { - if (cache_at_zero_ && target_ts < cached_frames_.first()->pts) { - - found_frame = cached_frames_.first(); + return_frame = cached_frames_.first(); cached_frames_.accessedFirst(); - } else if (cache_at_eof_ && target_ts > cached_frames_.last()->pts) { + } else if (cache_at_eof_ && target_ts > cached_frames_.last()->native_timestamp()) { - found_frame = cached_frames_.last(); + return_frame = cached_frames_.last(); cached_frames_.accessedLast(); - } else if (target_ts >= cached_frames_.first()->pts - && target_ts <= cached_frames_.last()->pts) { + } else if (target_ts >= cached_frames_.first()->native_timestamp() + && target_ts <= cached_frames_.last()->native_timestamp()) { // We already have this frame in the cache, find it for (int i=0;ipts == target_ts // Test for an exact match - || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->pts > target_ts)) { // Or for this frame to be the "closest" + if (this_frame->native_timestamp() == target_ts // Test for an exact match + || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->native_timestamp() > target_ts)) { // Or for this frame to be the "closest" - found_frame = this_frame; + return_frame = this_frame; cached_frames_.accessed(i); break; @@ -261,16 +256,6 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) } } } - - if (found_frame) { - // This frame is appropriate, return it - for (int i=0;i<4;i++) { - input_data[i] = found_frame->data[i]; - input_linesize[i] = found_frame->linesize[i]; - } - - got_frame = true; - } } // See if we stored this frame in the disk cache @@ -300,15 +285,15 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) */ // If we have no disk cache, we'll need to find this frame ourselves - if (!got_frame) { + if (!return_frame) { int64_t seek_ts = target_ts; bool still_seeking = false; // If the frame wasn't in the frame cache, see if this frame cache is too old to use if (cached_frames_.isEmpty() - || target_ts < cached_frames_.first()->pts - || target_ts > cached_frames_.last()->pts + 2*second_ts_) { + || target_ts < cached_frames_.first()->native_timestamp() + || target_ts > cached_frames_.last()->native_timestamp() + 2*second_ts_) { ClearFrameCache(); Seek(seek_ts); @@ -322,8 +307,6 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) int ret; AVPacket* pkt = av_packet_alloc(); - AVFrame* found_frame = nullptr; - while (true) { // Allocate a new frame AVFrame* working_frame = av_frame_alloc(); @@ -365,71 +348,73 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) // Handle an "expected" EOF by using the last frame of our cache cache_at_eof_ = true; av_frame_free(&working_frame); - found_frame = cached_frames_.last(); + return_frame = cached_frames_.last(); } else { + bool working_frame_is_the_one = false; + // If this is a valid frame, see if this or the frame before it are the one we need - if (!found_frame) { - if (working_frame->pts == target_ts) { - found_frame = working_frame; - } else if (working_frame->pts > target_ts) { - if (cached_frames_.isEmpty() && cache_at_zero_) { - found_frame = working_frame; - } else { - found_frame = cached_frames_.last(); - } + if (working_frame->pts == target_ts) { + working_frame_is_the_one = true; + } else if (working_frame->pts > target_ts) { + if (cached_frames_.isEmpty() && cache_at_zero_) { + working_frame_is_the_one = true; + } else { + return_frame = cached_frames_.last(); } } // Whatever it is, keep this frame in memory for the time being just in case - cached_frames_.append(working_frame); - } + Frame* working_frame_converted = cached_frames_.append(VideoRenderingParams(avstream_->codecpar->width, + avstream_->codecpar->height, + avstream_->time_base, + native_pix_fmt_, + RenderMode::kOffline)); - if (found_frame) { - // We found the frame we want - got_frame = true; + working_frame_converted->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base)); + working_frame_converted->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr)); + working_frame_converted->set_native_timestamp(working_frame->pts); - // Set data arrays to the frame's data - for (int i=0;i<4;i++) { - input_data[i] = found_frame->data[i]; - input_linesize[i] = found_frame->linesize[i]; + // Convert frame to RGBA for the rest of the pipeline + uint8_t* output_data = reinterpret_cast(working_frame_converted->data()); + int output_linesize = working_frame_converted->width() * kRGBAChannels * PixelService::BytesPerChannel(native_pix_fmt_); + + sws_scale(scale_ctx_, + working_frame->data, + working_frame->linesize, + 0, + avstream_->codecpar->height, + &output_data, + &output_linesize); + + if (working_frame_is_the_one) { + // We found the frame we want + return_frame = working_frame_converted; } - //CacheFrameToDisk(found_frame); - break; + if (return_frame) { + break; + } } } av_packet_free(&pkt); } - // If we're here and got the frame, we'll convert it and return it - if (got_frame) { + // We found the frame, we'll return a copy + if (return_frame) { + FramePtr copy = Frame::Create(); + copy->set_width(return_frame->width()); + copy->set_height(return_frame->height()); + copy->set_format(return_frame->format()); + copy->set_timestamp(return_frame->timestamp()); + copy->set_sample_aspect_ratio(return_frame->sample_aspect_ratio()); + copy->allocate(); - // Allocate frame that we'll return - FramePtr frame_container = Frame::Create(); - frame_container->set_width(avstream_->codecpar->width); - frame_container->set_height(avstream_->codecpar->height); - frame_container->set_format(native_pix_fmt_); - frame_container->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base)); - frame_container->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr)); - frame_container->allocate(); - - // Convert frame to RGBA for the rest of the pipeline - uint8_t* output_data = reinterpret_cast(frame_container->data()); - int output_linesize = frame_container->width() * kRGBAChannels * PixelService::BytesPerChannel(native_pix_fmt_); - - sws_scale(scale_ctx_, - input_data, - input_linesize, - 0, - avstream_->codecpar->height, - &output_data, - &output_linesize); - - return frame_container; + memcpy(copy->data(), return_frame->data(), copy->allocated_size()); + return copy; } return nullptr; @@ -940,11 +925,6 @@ void FFmpegDecoder::RemoveLastFromFrameCache() void FFmpegDecoder::ClearFrameCache() { - for (int i=0;i #include -int FFmpegFrameCache::global_frame_count_ = 0; +QMutex FFmpegFrameCache::pool_lock_; +QList FFmpegFrameCache::frame_pool_; -FFmpegFrameCache::FFmpegFrameCache() +Frame *FFmpegFrameCache::Client::append(const VideoRenderingParams& params) { - -} - -void FFmpegFrameCache::append(AVFrame *f) -{ - global_frame_count_++; + Frame* f = FFmpegFrameCache::Get(params); frames_.append({f, QDateTime::currentMSecsSinceEpoch()}); + + return f; } -void FFmpegFrameCache::clear() +void FFmpegFrameCache::Client::clear() { - global_frame_count_ -= frames_.size(); - + foreach (const CachedFrame& cf, frames_) { + FFmpegFrameCache::Release(cf.frame); + } frames_.clear(); } -bool FFmpegFrameCache::isEmpty() const +bool FFmpegFrameCache::Client::isEmpty() const { return frames_.isEmpty(); } -AVFrame *FFmpegFrameCache::first() const +Frame *FFmpegFrameCache::Client::first() const { return frames_.first().frame; } -AVFrame *FFmpegFrameCache::at(int i) const +Frame *FFmpegFrameCache::Client::at(int i) const { return frames_.at(i).frame; } -AVFrame *FFmpegFrameCache::last() const +Frame *FFmpegFrameCache::Client::last() const { return frames_.last().frame; } -int FFmpegFrameCache::size() const +int FFmpegFrameCache::Client::size() const { return frames_.size(); } -void FFmpegFrameCache::accessedFirst() +void FFmpegFrameCache::Client::accessedFirst() { frames_.first().accessed = QDateTime::currentMSecsSinceEpoch(); } -void FFmpegFrameCache::accessedLast() +void FFmpegFrameCache::Client::accessedLast() { frames_.last().accessed = QDateTime::currentMSecsSinceEpoch(); } -void FFmpegFrameCache::accessed(int i) +void FFmpegFrameCache::Client::accessed(int i) { frames_[i].accessed = QDateTime::currentMSecsSinceEpoch(); } -void FFmpegFrameCache::remove_old_frames(qint64 older_than) +void FFmpegFrameCache::Client::remove_old_frames(qint64 older_than) { int counter = 0; while (!frames_.isEmpty() && frames_.first().accessed < older_than) { - CachedFrame cf = frames_.takeFirst(); - av_frame_free(&cf.frame); + FFmpegFrameCache::Release(frames_.takeFirst().frame); counter++; } qDebug() << " * Removed" << counter << "frames"; - global_frame_count_ -= counter; - qDebug() << " * Global frames:" << global_frame_count_; +} + +Frame* FFmpegFrameCache::Get(const VideoRenderingParams ¶ms) +{ + QMutexLocker locker(&pool_lock_); + + // See if we have a frame matching this description in the pool + for (int i=0;iwidth() == params.width() + && frame_pool_.at(i)->height() == params.height() + && frame_pool_.at(i)->format() == params.format()) { + return frame_pool_.takeAt(i); + } + } + + qDebug() << "Nothing in frame pool existed, have to create new..."; + + // Otherwise we'll need to create one + Frame* f = new Frame(); + f->set_width(params.width()); + f->set_height(params.height()); + f->set_format(params.format()); + f->allocate(); + + return f; +} + +void FFmpegFrameCache::Release(Frame *f) +{ + QMutexLocker locker(&pool_lock_); + + frame_pool_.append(f); } diff --git a/app/codec/ffmpeg/ffmpegframecache.h b/app/codec/ffmpeg/ffmpegframecache.h index 49bd4a512..dde66744c 100644 --- a/app/codec/ffmpeg/ffmpegframecache.h +++ b/app/codec/ffmpeg/ffmpegframecache.h @@ -1,42 +1,55 @@ #ifndef FFMPEGFRAMECACHE_H #define FFMPEGFRAMECACHE_H -extern "C" { -#include -} - #include -#include +#include + +#include "codec/frame.h" +#include "render/videoparams.h" class FFmpegFrameCache { public: - FFmpegFrameCache(); + FFmpegFrameCache() = default; - void append(AVFrame* f); - void clear(); + static Frame* Get(const VideoRenderingParams& params); - bool isEmpty() const; - AVFrame* first() const; - AVFrame* at(int i) const; - AVFrame* last() const; - int size() const; + static void Release(Frame* f); - void accessedFirst(); - void accessedLast(); - void accessed(int i); + class Client + { + public: + Client() = default; - void remove_old_frames(qint64 older_than); + Frame* append(const VideoRenderingParams ¶ms); + void clear(); + + bool isEmpty() const; + Frame* first() const; + Frame* at(int i) const; + Frame* last() const; + int size() const; + + void accessedFirst(); + void accessedLast(); + void accessed(int i); + + void remove_old_frames(qint64 older_than); + + private: + struct CachedFrame { + Frame* frame; + qint64 accessed; + }; + + QList frames_; -private: - struct CachedFrame { - AVFrame* frame; - qint64 accessed; }; - QList frames_; +private: + static QMutex pool_lock_; - static int global_frame_count_; + static QList frame_pool_; }; diff --git a/app/codec/frame.cpp b/app/codec/frame.cpp index ad1b3f7f0..0cdf5f144 100644 --- a/app/codec/frame.cpp +++ b/app/codec/frame.cpp @@ -90,7 +90,7 @@ void Frame::set_timestamp(const rational ×tamp) timestamp_ = timestamp; } -/*const int64_t &Frame::native_timestamp() +const int64_t &Frame::native_timestamp() { return native_timestamp_; } @@ -98,7 +98,7 @@ void Frame::set_timestamp(const rational ×tamp) void Frame::set_native_timestamp(const int64_t ×tamp) { native_timestamp_ = timestamp; -}*/ +} const PixelFormat::Format &Frame::format() const { diff --git a/app/codec/frame.h b/app/codec/frame.h index 135dd8f27..1311a80dd 100644 --- a/app/codec/frame.h +++ b/app/codec/frame.h @@ -70,8 +70,8 @@ public: const rational& timestamp() const; void set_timestamp(const rational& timestamp); - /*const int64_t& native_timestamp(); - void set_native_timestamp(const int64_t& timestamp);*/ + const int64_t& native_timestamp(); + void set_native_timestamp(const int64_t& timestamp); /** * @brief Get frame's format @@ -141,6 +141,8 @@ private: rational timestamp_; + int64_t native_timestamp_; + rational sample_aspect_ratio_; };