From d87b4fde8c049c0b13f73162574a13b022541356 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Thu, 27 Feb 2020 15:47:08 +1100 Subject: [PATCH] renderer/decoder: use divider at the footage level Optimizes rendering if a divider is being used. --- app/codec/decoder.cpp | 2 +- app/codec/decoder.h | 2 +- app/codec/ffmpeg/ffmpegdecoder.cpp | 63 ++++++++++++++--------- app/codec/ffmpeg/ffmpegdecoder.h | 8 +-- app/codec/oiio/oiiodecoder.cpp | 30 ++++++----- app/codec/oiio/oiiodecoder.h | 5 +- app/render/backend/opengl/openglproxy.cpp | 12 +++-- app/render/backend/opengl/openglproxy.h | 1 + 8 files changed, 75 insertions(+), 48 deletions(-) diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index ed0dea143..224818912 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -61,7 +61,7 @@ void Decoder::set_stream(StreamPtr fs) stream_ = fs; } -FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/) +FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const int &/*divider*/) { return nullptr; } diff --git a/app/codec/decoder.h b/app/codec/decoder.h index 86ffc2710..af3507e33 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -142,7 +142,7 @@ public: * A FramePtr of valid data at this timecode or nullptr if there was nothing to retrieve at the provided timecode or * the media could not be opened. */ - virtual FramePtr RetrieveVideo(const rational& timecode); + virtual FramePtr RetrieveVideo(const rational& timecode, const int& divider); /** * @brief Retrieve video frame diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 149e353fe..8b8b1ee2b 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -46,6 +46,7 @@ FFmpegDecoder::FFmpegDecoder() : fmt_ctx_(nullptr), codec_ctx_(nullptr), scale_ctx_(nullptr), + scale_divider_(-1), cache_at_zero_(false), cache_at_eof_(false), opts_(nullptr) @@ -172,22 +173,6 @@ bool FFmpegDecoder::Open() qFatal("Invalid output format"); } - scale_ctx_ = sws_getContext(avstream_->codecpar->width, - avstream_->codecpar->height, - static_cast(avstream_->codecpar->format), - avstream_->codecpar->width, - avstream_->codecpar->height, - ideal_pix_fmt_, - 0, - nullptr, - nullptr, - nullptr); - - if (!scale_ctx_) { - Error(QStringLiteral("Failed to allocate SwsContext")); - return false; - } - second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base))); QMetaObject::invokeMethod(&clear_timer_, "start"); @@ -220,7 +205,7 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time) return kReady; } -FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) +FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int ÷r) { QMutexLocker locker(&mutex_); @@ -237,6 +222,12 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) Frame* return_frame = nullptr; + if (divider != scale_divider_) { + ClearFrameCache(); + FreeScaler(); + SetupScaler(divider); + } + // See if our RAM cache already has a frame that matches this timestamp if (!cached_frames_.isEmpty()) { @@ -375,8 +366,8 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) } // Whatever it is, keep this frame in memory for the time being just in case - Frame* working_frame_converted = cached_frames_.append(VideoRenderingParams(avstream_->codecpar->width, - avstream_->codecpar->height, + Frame* working_frame_converted = cached_frames_.append(VideoRenderingParams(avstream_->codecpar->width / divider, + avstream_->codecpar->height / divider, avstream_->time_base, native_pix_fmt_, RenderMode::kOffline)); @@ -949,10 +940,7 @@ void FFmpegDecoder::ClearResources() ClearFrameCache(); - if (scale_ctx_) { - sws_freeContext(scale_ctx_); - scale_ctx_ = nullptr; - } + FreeScaler(); if (codec_ctx_) { avcodec_free_context(&codec_ctx_); @@ -967,6 +955,35 @@ void FFmpegDecoder::ClearResources() open_ = false; } +void FFmpegDecoder::SetupScaler(const int ÷r) +{ + scale_ctx_ = sws_getContext(avstream_->codecpar->width, + avstream_->codecpar->height, + static_cast(avstream_->codecpar->format), + avstream_->codecpar->width / divider, + avstream_->codecpar->height / divider, + ideal_pix_fmt_, + SWS_FAST_BILINEAR, + nullptr, + nullptr, + nullptr); + + if (!scale_ctx_) { + Error(QStringLiteral("Failed to allocate SwsContext")); + } else { + scale_divider_ = divider; + } +} + +void FFmpegDecoder::FreeScaler() +{ + if (scale_ctx_) { + sws_freeContext(scale_ctx_); + scale_ctx_ = nullptr; + scale_divider_ = -1; + } +} + void FFmpegDecoder::ClearTimerEvent() { QMutexLocker locker(&mutex_); diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index e80c70593..4ceb50184 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -54,7 +54,7 @@ public: virtual bool Open() override; virtual RetrieveState GetRetrieveState(const rational &time) override; - virtual FramePtr RetrieveVideo(const rational &timecode) override; + virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override; virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override; virtual void Close() override; @@ -102,12 +102,13 @@ private: void CacheFrameToDisk(AVFrame* f); - //void RemoveFirstFromFrameCache(); - //void RemoveLastFromFrameCache(); void ClearFrameCache(); void ClearResources(); + void SetupScaler(const int& divider); + void FreeScaler(); + AVFormatContext* fmt_ctx_; AVCodecContext* codec_ctx_; AVStream* avstream_; @@ -116,6 +117,7 @@ private: PixelFormat::Format native_pix_fmt_; SwsContext* scale_ctx_; + int scale_divider_; FFmpegFrameCache::Client cached_frames_; bool cache_at_zero_; diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index 4aca39e6b..c2de2b690 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -20,6 +20,7 @@ #include "oiiodecoder.h" +#include #include #include @@ -29,7 +30,7 @@ QStringList OIIODecoder::supported_formats_; OIIODecoder::OIIODecoder() : image_(nullptr), - frame_(nullptr) + buffer_(nullptr) { } @@ -134,6 +135,9 @@ bool OIIODecoder::Open() // FIXME: Many OIIO pixel formats are not handled here type_ = PixelFormat::GetOIIOTypeDesc(pix_fmt_); + buffer_ = new OIIO::ImageBuf(OIIO::ImageSpec(spec.width, spec.height, spec.nchannels, type_), OIIO::InitializePixels::No); + image_->read_image(type_, buffer_->localpixels()); + open_ = true; return true; @@ -150,7 +154,7 @@ Decoder::RetrieveState OIIODecoder::GetRetrieveState(const rational &time) return kReady; } -FramePtr OIIODecoder::RetrieveVideo(const rational &timecode) +FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const int& divider) { QMutexLocker locker(&mutex_); @@ -158,21 +162,20 @@ FramePtr OIIODecoder::RetrieveVideo(const rational &timecode) return nullptr; } - Q_UNUSED(timecode) + FramePtr frame = Frame::Create(); - if (!frame_) { - frame_ = Frame::Create(); + frame->set_width(width_ / divider); + frame->set_height(height_ / divider); + frame->set_format(pix_fmt_); + frame->allocate(); - frame_->set_width(width_); - frame_->set_height(height_); - frame_->set_format(pix_fmt_); - frame_->allocate(); + OIIO::ImageBuf dst(OIIO::ImageSpec(frame->width(), frame->height(), buffer_->spec().nchannels, buffer_->spec().format), frame->data()); - // Use the native format to determine what format OIIO should return - image_->read_image(type_, frame_->data()); + if (!OIIO::ImageBufAlgo::resize(dst, *buffer_)) { + qWarning() << "OIIO resize failed"; } - return frame_; + return frame; } void OIIODecoder::Close() @@ -187,7 +190,8 @@ void OIIODecoder::Close() image_ = nullptr; } - frame_ = nullptr; + delete buffer_; + buffer_ = nullptr; } bool OIIODecoder::SupportsVideo() diff --git a/app/codec/oiio/oiiodecoder.h b/app/codec/oiio/oiiodecoder.h index 92f1ea4c4..be6e5f21e 100644 --- a/app/codec/oiio/oiiodecoder.h +++ b/app/codec/oiio/oiiodecoder.h @@ -22,6 +22,7 @@ #define OIIODECODER_H #include +#include #include "codec/decoder.h" #include "render/pixelformat.h" @@ -37,7 +38,7 @@ public: virtual bool Open() override; virtual RetrieveState GetRetrieveState(const rational &time) override; - virtual FramePtr RetrieveVideo(const rational &timecode) override; + virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override; virtual void Close() override; virtual bool SupportsVideo() override; @@ -61,7 +62,7 @@ private: bool is_rgba_; - FramePtr frame_; + OIIO::ImageBuf* buffer_; static QStringList supported_formats_; diff --git a/app/render/backend/opengl/openglproxy.cpp b/app/render/backend/opengl/openglproxy.cpp index 7aad9279f..76bf19cbb 100644 --- a/app/render/backend/opengl/openglproxy.cpp +++ b/app/render/backend/opengl/openglproxy.cpp @@ -60,7 +60,9 @@ void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR if (stream->type() == Stream::kImage && still_image_cache_.Has(stream.get())) { CachedStill cs = still_image_cache_.Get(stream.get()); - if (cs.colorspace == colorspace_match && cs.alpha_is_associated == video_stream->premultiplied_alpha()) { + if (cs.colorspace == colorspace_match + && cs.alpha_is_associated == video_stream->premultiplied_alpha() + && cs.divider == video_params_.divider()) { footage_tex_ref = cs.texture; } else { still_image_cache_.Remove(stream.get()); @@ -80,7 +82,7 @@ void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode()); - FramePtr frame = decoder->RetrieveVideo(range.in());; + FramePtr frame = decoder->RetrieveVideo(range.in(), video_params_.divider()); // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU if (ocio_method == ColorManager::kOCIOAccurate) { @@ -154,7 +156,7 @@ void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR } if (stream->type() == Stream::kImage) { - still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha()}); + still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha(), video_params_.divider()}); } } @@ -281,8 +283,8 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c int res_param_location = shader->uniformLocation(QStringLiteral("%1_resolution").arg(input->id())); if (res_param_location > -1) { shader->setUniformValue(res_param_location, - static_cast(texture->texture()->width()), - static_cast(texture->texture()->height())); + static_cast(texture->texture()->width() * video_params_.divider()), + static_cast(texture->texture()->height() * video_params_.divider())); } } diff --git a/app/render/backend/opengl/openglproxy.h b/app/render/backend/opengl/openglproxy.h index d1b397706..9f7f6e362 100644 --- a/app/render/backend/opengl/openglproxy.h +++ b/app/render/backend/opengl/openglproxy.h @@ -69,6 +69,7 @@ private: OpenGLTextureCache::ReferencePtr texture; QString colorspace; bool alpha_is_associated; + int divider; }; RenderCache still_image_cache_;