From f3f81eb2be7be5e979815856fe4347620233f294 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Tue, 18 Aug 2020 04:55:23 +1000 Subject: [PATCH] ffmpeg: store divided frames in memory cache Since users will most likely be working with a divider most of the time, there's no reason to waste memory space with undivided frames considering the dividing process is quite fast. --- app/codec/ffmpeg/ffmpegdecoder.cpp | 115 ++++++++++++++++++++++----- app/codec/ffmpeg/ffmpegdecoder.h | 10 ++- app/codec/ffmpeg/ffmpegframepool.cpp | 27 +++---- app/codec/ffmpeg/ffmpegframepool.h | 17 ++-- app/common/memorypool.h | 18 ++++- app/render/videoparams.cpp | 10 ++- app/render/videoparams.h | 2 + 7 files changed, 147 insertions(+), 52 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 297065a08..eb61ea274 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -106,10 +106,7 @@ bool FFmpegDecoder::Open() if (!frame_pool) { // FIXME: Hardcoded value. It seems to work fine, but is there a possibility we should make // this a dynamic value somehow or a configurable value? - frame_pool = new FFmpegFramePool(64, - our_instance->stream()->codecpar->width, - our_instance->stream()->codecpar->height, - static_cast(our_instance->stream()->codecpar->format)); + frame_pool = new FFmpegFramePool(64); frame_pool_map_.insert(stream().get(), frame_pool); } @@ -194,13 +191,28 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid FFmpegDecoderInstance* working_instance = nullptr; + int divided_width = VideoParams::GetScaledDimension(vs->width(), divider); + int divided_height = VideoParams::GetScaledDimension(vs->height(), divider); + // Find instance do { QMutexLocker list_locker(&instance_map_lock_); - QList non_ideal_contenders; + const QList& instances = instance_map_.value(stream().get()); - QList instances = instance_map_.value(stream().get()); + FFmpegFramePool* pool = frame_pool_map_.value(stream().get()); + + if (pool->width() != divided_width || pool->height() != divided_height) { + // Clear all instance queues + foreach (FFmpegDecoderInstance* i, instances) { + i->ClearFrameCache(); + } + + // Set new frame pool parameters + pool->SetParameters(divided_width, divided_height, src_pix_fmt_); + } + + QList non_ideal_contenders; foreach (FFmpegDecoderInstance* i, instances) { @@ -291,7 +303,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid working_instance->SetWorking(true); // Retrieve frame - return_frame = working_instance->RetrieveFrame(target_ts, true); + return_frame = working_instance->RetrieveFrame(target_ts, divider, true); // Set working to false and wake any threads waiting working_instance->cache_lock()->lock(); @@ -310,8 +322,8 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid input_linesize, reinterpret_cast(return_frame->data()), src_pix_fmt_, - vs->width(), - vs->height(), + divided_width, + divided_height, 1); return BuffersToNativeFrame(divider, @@ -750,11 +762,6 @@ bool FFmpegDecoder::ConformAudio(const QAtomicInt *cancelled, const AudioParams return success; } -int FFmpegDecoder::GetScaledDimension(int dim, int divider) -{ - return dim / divider; -} - PixelFormat::Format FFmpegDecoder::GetNativePixelFormat(AVPixelFormat pix_fmt) { switch (pix_fmt) { @@ -812,7 +819,7 @@ FramePtr FFmpegDecoder::BuffersToNativeFrame(int divider, int width, int height, input_data, input_linesize, 0, - height, + VideoParams::GetScaledDimension(height, divider), &output_data, &output_linesize); @@ -954,12 +961,17 @@ void FFmpegDecoderInstance::ClearFrameCache() cache_at_zero_ = false; } -FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) +FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, int divider, bool cache_is_locked) { if (!cache_is_locked) { cache_lock_.lock(); } + if (scale_divider_ != divider) { + FreeScaler(); + InitScaler(divider); + } + int64_t seek_ts = target_ts; bool still_seeking = false; @@ -1052,7 +1064,7 @@ FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& break; } - FFmpegFramePool::ElementPtr cached = frame_pool_->Get(working_frame.frame()); + FFmpegFramePool::ElementPtr cached = frame_pool_->Get(); if (!cached) { qCritical() << "Frame pool failed to return a valid frame - out of memory?"; @@ -1060,6 +1072,27 @@ FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& break; } + { + uint8_t* scale_data[4]; + int scale_linesize[4]; + + av_image_fill_arrays(scale_data, + scale_linesize, + cached->data(), + static_cast(working_frame.frame()->format), + VideoParams::GetScaledDimension(working_frame.frame()->width, divider), + VideoParams::GetScaledDimension(working_frame.frame()->height, divider), + 1); + + sws_scale(scale_ctx_, + working_frame.frame()->data, + working_frame.frame()->linesize, + 0, + working_frame.frame()->height, + scale_data, + scale_linesize); + } + // Set timestamp so this frame can be identified later cached->set_timestamp(working_frame.frame()->pts); @@ -1113,11 +1146,14 @@ void FFmpegDecoder::InitScaler(int divider) { VideoStream* vs = static_cast(stream().get()); - scale_ctx_ = sws_getContext(vs->width(), - vs->height(), + int scaled_width = VideoParams::GetScaledDimension(vs->width(), divider); + int scaled_height = VideoParams::GetScaledDimension(vs->height(), divider); + + scale_ctx_ = sws_getContext(scaled_width, + scaled_height, src_pix_fmt_, - GetScaledDimension(vs->width(), divider), - GetScaledDimension(vs->height(), divider), + scaled_width, + scaled_height, ideal_pix_fmt_, SWS_FAST_BILINEAR, nullptr, @@ -1141,6 +1177,39 @@ void FFmpegDecoder::FreeScaler() } } +void FFmpegDecoderInstance::InitScaler(int divider) +{ + int scaled_width = VideoParams::GetScaledDimension(avstream_->codecpar->width, divider); + int scaled_height = VideoParams::GetScaledDimension(avstream_->codecpar->height, divider); + + scale_ctx_ = sws_getContext(avstream_->codecpar->width, + avstream_->codecpar->height, + static_cast(avstream_->codecpar->format), + scaled_width, + scaled_height, + static_cast(avstream_->codecpar->format), + SWS_FAST_BILINEAR, + nullptr, + nullptr, + nullptr); + + if (scale_ctx_) { + scale_divider_ = divider; + } else { + scale_divider_ = 0; + } +} + +void FFmpegDecoderInstance::FreeScaler() +{ + if (scale_ctx_) { + sws_freeContext(scale_ctx_); + scale_ctx_ = nullptr; + + scale_divider_ = 0; + } +} + int64_t FFmpegDecoderInstance::RangeStart() const { if (cached_frames_.isEmpty()) { @@ -1236,6 +1305,8 @@ void FFmpegDecoderInstance::TruncateCacheRangeTo(const qint64 &t) FFmpegDecoderInstance::FFmpegDecoderInstance(const char *filename, int stream_index) : fmt_ctx_(nullptr), opts_(nullptr), + scale_ctx_(nullptr), + scale_divider_(0), frame_pool_(nullptr), is_working_(false), cache_at_zero_(false), @@ -1365,6 +1436,8 @@ void FFmpegDecoderInstance::ClearResources() avformat_close_input(&fmt_ctx_); fmt_ctx_ = nullptr; } + + FreeScaler(); } void FFmpegDecoderInstance::ClearTimerEvent() diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index b421e453d..e88892cb9 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -76,7 +76,7 @@ public: void ClearFrameCache(); - FFmpegFramePool::ElementPtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); + FFmpegFramePool::ElementPtr RetrieveFrame(const int64_t &target_ts, int divider, bool cache_is_locked); /** * @brief Uses the FFmpeg API to retrieve a packet (stored in pkt_) and decode it (stored in frame_) @@ -98,11 +98,17 @@ private: void Seek(int64_t timestamp); + void InitScaler(int divider); + void FreeScaler(); + AVFormatContext* fmt_ctx_; AVCodecContext* codec_ctx_; AVStream* avstream_; AVDictionary* opts_; + SwsContext* scale_ctx_; + int scale_divider_; + int64_t second_ts_; QWaitCondition cache_wait_cond_; @@ -179,8 +185,6 @@ private: FramePtr RetrieveStillImage(const rational& timecode, const int& divider); - static int GetScaledDimension(int dim, int divider); - static PixelFormat::Format GetNativePixelFormat(AVPixelFormat pix_fmt); static uint64_t ValidateChannelLayout(AVStream *stream); diff --git a/app/codec/ffmpeg/ffmpegframepool.cpp b/app/codec/ffmpeg/ffmpegframepool.cpp index ea7914101..cca05aed4 100644 --- a/app/codec/ffmpeg/ffmpegframepool.cpp +++ b/app/codec/ffmpeg/ffmpegframepool.cpp @@ -26,30 +26,21 @@ extern "C" { OLIVE_NAMESPACE_ENTER -FFmpegFramePool::FFmpegFramePool(int element_count, int width, int height, AVPixelFormat format) : +FFmpegFramePool::FFmpegFramePool(int element_count) : MemoryPool(element_count), - width_(width), - height_(height), - format_(format) + width_(0), + height_(0), + format_(AV_PIX_FMT_NONE) { } -FFmpegFramePool::ElementPtr FFmpegFramePool::Get(AVFrame *copy) +void FFmpegFramePool::SetParameters(int width, int height, AVPixelFormat format) { - ElementPtr ele = MemoryPool::Get(); + Clear(); - if (ele) { - av_image_copy_to_buffer(ele->data(), - GetElementSize(), - copy->data, - copy->linesize, - format_, - width_, - height_, - 1); - } - - return ele; + width_ = width; + height_ = height; + format_ = format; } size_t FFmpegFramePool::GetElementSize() diff --git a/app/codec/ffmpeg/ffmpegframepool.h b/app/codec/ffmpeg/ffmpegframepool.h index ec7a81c09..81a31312e 100644 --- a/app/codec/ffmpeg/ffmpegframepool.h +++ b/app/codec/ffmpeg/ffmpegframepool.h @@ -30,12 +30,19 @@ OLIVE_NAMESPACE_ENTER class FFmpegFramePool : public MemoryPool { public: - FFmpegFramePool(int element_count, - int width, - int height, - AVPixelFormat format); + FFmpegFramePool(int element_count); - ElementPtr Get(AVFrame* copy); + void SetParameters(int width, int height, AVPixelFormat format); + + const int& width() const + { + return width_; + } + + const int& height() const + { + return height_; + } protected: virtual size_t GetElementSize() override; diff --git a/app/common/memorypool.h b/app/common/memorypool.h index 28955989f..6566505eb 100644 --- a/app/common/memorypool.h +++ b/app/common/memorypool.h @@ -66,12 +66,25 @@ public: * Deletes all arenas. */ virtual ~MemoryPool() { - ignore_arena_empty_signal_ = true; - qDeleteAll(arenas_); + Clear(); } DISABLE_COPY_MOVE(MemoryPool) + /** + * @brief Clears all arenas, freeing all of their memory + * + * Note that this function is not safe, any elements that are still out there will be invalid + * and accessing them will cause a crash. You'll need to make sure all elements are already + * relinquished before then. + */ + void Clear() + { + ignore_arena_empty_signal_ = true; + qDeleteAll(arenas_); + ignore_arena_empty_signal_ = false; + } + /** * @brief Returns whether any arenas are successfully allocated */ @@ -212,6 +225,7 @@ public: ElementPtr e = std::make_shared(this, reinterpret_cast(data_ + i * element_sz_)); lent_elements_.push_back(e.get()); + return e; } } diff --git a/app/render/videoparams.cpp b/app/render/videoparams.cpp index 8e20a717e..00167c63a 100644 --- a/app/render/videoparams.cpp +++ b/app/render/videoparams.cpp @@ -145,9 +145,8 @@ bool VideoParams::operator!=(const VideoParams &rhs) const void VideoParams::calculate_effective_size() { - // Fast rounding up to an even number - effective_width_ = qCeil(width() / divider_ * 0.5) * 2; - effective_height_ = qCeil(height() / divider_ * 0.5) * 2; + effective_width_ = GetScaledDimension(width(), divider_); + effective_height_ = GetScaledDimension(height(), divider_); } void VideoParams::validate_pixel_aspect_ratio() @@ -195,4 +194,9 @@ QString VideoParams::FormatPixelAspectRatioString(const QString &format, const r return format.arg(QString::number(ratio.toDouble(), 'f', 4)); } +int VideoParams::GetScaledDimension(int dim, int divider) +{ + return qCeil(dim / divider * 0.5) * 2; +} + OLIVE_NAMESPACE_EXIT diff --git a/app/render/videoparams.h b/app/render/videoparams.h index 9eb331ea0..fa24e2027 100644 --- a/app/render/videoparams.h +++ b/app/render/videoparams.h @@ -114,6 +114,8 @@ public: static QStringList GetStandardPixelAspectRatioNames(); static QString FormatPixelAspectRatioString(const QString& format, const rational& ratio); + static int GetScaledDimension(int dim, int divider); + private: void calculate_effective_size();