renderer/decoder: use divider at the footage level

Optimizes rendering if a divider is being used.

app/codec/decoder.cpp

@@ -61,7 +61,7 @@ void Decoder::set_stream(StreamPtr fs)
   stream_ = fs;
 }
 
-FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/)
+FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const int &/*divider*/)
 {
   return nullptr;
 }

app/codec/decoder.h

@@ -142,7 +142,7 @@ public:
    * A FramePtr of valid data at this timecode or nullptr if there was nothing to retrieve at the provided timecode or
    * the media could not be opened.
    */
-  virtual FramePtr RetrieveVideo(const rational& timecode);
+  virtual FramePtr RetrieveVideo(const rational& timecode, const int& divider);
 
   /**
    * @brief Retrieve video frame

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -46,6 +46,7 @@ FFmpegDecoder::FFmpegDecoder() :
   fmt_ctx_(nullptr),
   codec_ctx_(nullptr),
   scale_ctx_(nullptr),
+  scale_divider_(-1),
   cache_at_zero_(false),
   cache_at_eof_(false),
   opts_(nullptr)
@@ -172,22 +173,6 @@ bool FFmpegDecoder::Open()
       qFatal("Invalid output format");
     }
 
-    scale_ctx_ = sws_getContext(avstream_->codecpar->width,
-                                avstream_->codecpar->height,
-                                static_cast<AVPixelFormat>(avstream_->codecpar->format),
-                                avstream_->codecpar->width,
-                                avstream_->codecpar->height,
-                                ideal_pix_fmt_,
-                                0,
-                                nullptr,
-                                nullptr,
-                                nullptr);
-
-    if (!scale_ctx_) {
-      Error(QStringLiteral("Failed to allocate SwsContext"));
-      return false;
-    }
-
     second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base)));
 
     QMetaObject::invokeMethod(&clear_timer_, "start");
@@ -220,7 +205,7 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time)
   return kReady;
 }
 
-FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
+FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divider)
 {
   QMutexLocker locker(&mutex_);
 
@@ -237,6 +222,12 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 
   Frame* return_frame = nullptr;
 
+  if (divider != scale_divider_) {
+    ClearFrameCache();
+    FreeScaler();
+    SetupScaler(divider);
+  }
+
   // See if our RAM cache already has a frame that matches this timestamp
   if (!cached_frames_.isEmpty()) {
 
@@ -375,8 +366,8 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
         }
 
         // Whatever it is, keep this frame in memory for the time being just in case
-        Frame* working_frame_converted = cached_frames_.append(VideoRenderingParams(avstream_->codecpar->width,
-                                                                                    avstream_->codecpar->height,
+        Frame* working_frame_converted = cached_frames_.append(VideoRenderingParams(avstream_->codecpar->width / divider,
+                                                                                    avstream_->codecpar->height / divider,
                                                                                     avstream_->time_base,
                                                                                     native_pix_fmt_,
                                                                                     RenderMode::kOffline));
@@ -949,10 +940,7 @@ void FFmpegDecoder::ClearResources()
 
   ClearFrameCache();
 
-  if (scale_ctx_) {
-    sws_freeContext(scale_ctx_);
-    scale_ctx_ = nullptr;
-  }
+  FreeScaler();
 
   if (codec_ctx_) {
     avcodec_free_context(&codec_ctx_);
@@ -967,6 +955,35 @@ void FFmpegDecoder::ClearResources()
   open_ = false;
 }
 
+void FFmpegDecoder::SetupScaler(const int &divider)
+{
+  scale_ctx_ = sws_getContext(avstream_->codecpar->width,
+                              avstream_->codecpar->height,
+                              static_cast<AVPixelFormat>(avstream_->codecpar->format),
+                              avstream_->codecpar->width / divider,
+                              avstream_->codecpar->height / divider,
+                              ideal_pix_fmt_,
+                              SWS_FAST_BILINEAR,
+                              nullptr,
+                              nullptr,
+                              nullptr);
+
+  if (!scale_ctx_) {
+    Error(QStringLiteral("Failed to allocate SwsContext"));
+  } else {
+    scale_divider_ = divider;
+  }
+}
+
+void FFmpegDecoder::FreeScaler()
+{
+  if (scale_ctx_) {
+    sws_freeContext(scale_ctx_);
+    scale_ctx_ = nullptr;
+    scale_divider_ = -1;
+  }
+}
+
 void FFmpegDecoder::ClearTimerEvent()
 {
   QMutexLocker locker(&mutex_);

app/codec/ffmpeg/ffmpegdecoder.h

@@ -54,7 +54,7 @@ public:
 
   virtual bool Open() override;
   virtual RetrieveState GetRetrieveState(const rational &time) override;
-  virtual FramePtr RetrieveVideo(const rational &timecode) override;
+  virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override;
   virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override;
   virtual void Close() override;
 
@@ -102,12 +102,13 @@ private:
 
   void CacheFrameToDisk(AVFrame* f);
 
-  //void RemoveFirstFromFrameCache();
-  //void RemoveLastFromFrameCache();
   void ClearFrameCache();
 
   void ClearResources();
 
+  void SetupScaler(const int& divider);
+  void FreeScaler();
+
   AVFormatContext* fmt_ctx_;
   AVCodecContext* codec_ctx_;
   AVStream* avstream_;
@@ -116,6 +117,7 @@ private:
   PixelFormat::Format native_pix_fmt_;
 
   SwsContext* scale_ctx_;
+  int scale_divider_;
 
   FFmpegFrameCache::Client cached_frames_;
   bool cache_at_zero_;

app/codec/oiio/oiiodecoder.cpp

@@ -20,6 +20,7 @@
 
 #include "oiiodecoder.h"
 
+#include <OpenImageIO/imagebufalgo.h>
 #include <QDebug>
 #include <QFileInfo>
 
@@ -29,7 +30,7 @@ QStringList OIIODecoder::supported_formats_;
 
 OIIODecoder::OIIODecoder() :
   image_(nullptr),
-  frame_(nullptr)
+  buffer_(nullptr)
 {
 }
 
@@ -134,6 +135,9 @@ bool OIIODecoder::Open()
   // FIXME: Many OIIO pixel formats are not handled here
   type_ = PixelFormat::GetOIIOTypeDesc(pix_fmt_);
 
+  buffer_ = new OIIO::ImageBuf(OIIO::ImageSpec(spec.width, spec.height, spec.nchannels, type_), OIIO::InitializePixels::No);
+  image_->read_image(type_, buffer_->localpixels());
+
   open_ = true;
 
   return true;
@@ -150,7 +154,7 @@ Decoder::RetrieveState OIIODecoder::GetRetrieveState(const rational &time)
   return kReady;
 }
 
-FramePtr OIIODecoder::RetrieveVideo(const rational &timecode)
+FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const int& divider)
 {
   QMutexLocker locker(&mutex_);
 
@@ -158,21 +162,20 @@ FramePtr OIIODecoder::RetrieveVideo(const rational &timecode)
     return nullptr;
   }
 
-  Q_UNUSED(timecode)
+  FramePtr frame = Frame::Create();
 
-  if (!frame_) {
-    frame_ = Frame::Create();
+  frame->set_width(width_ / divider);
+  frame->set_height(height_ / divider);
+  frame->set_format(pix_fmt_);
+  frame->allocate();
 
-    frame_->set_width(width_);
-    frame_->set_height(height_);
-    frame_->set_format(pix_fmt_);
-    frame_->allocate();
+  OIIO::ImageBuf dst(OIIO::ImageSpec(frame->width(), frame->height(), buffer_->spec().nchannels, buffer_->spec().format), frame->data());
 
-    // Use the native format to determine what format OIIO should return
-    image_->read_image(type_, frame_->data());
+  if (!OIIO::ImageBufAlgo::resize(dst, *buffer_)) {
+    qWarning() << "OIIO resize failed";
   }
 
-  return frame_;
+  return frame;
 }
 
 void OIIODecoder::Close()
@@ -187,7 +190,8 @@ void OIIODecoder::Close()
     image_ = nullptr;
   }
 
-  frame_ = nullptr;
+  delete buffer_;
+  buffer_ = nullptr;
 }
 
 bool OIIODecoder::SupportsVideo()

app/codec/oiio/oiiodecoder.h

@@ -22,6 +22,7 @@
 #define OIIODECODER_H
 
 #include <OpenImageIO/imageio.h>
+#include <OpenImageIO/imagebuf.h>
 
 #include "codec/decoder.h"
 #include "render/pixelformat.h"
@@ -37,7 +38,7 @@ public:
 
   virtual bool Open() override;
   virtual RetrieveState GetRetrieveState(const rational &time) override;
-  virtual FramePtr RetrieveVideo(const rational &timecode) override;
+  virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override;
   virtual void Close() override;
 
   virtual bool SupportsVideo() override;
@@ -61,7 +62,7 @@ private:
 
   bool is_rgba_;
 
-  FramePtr frame_;
+  OIIO::ImageBuf* buffer_;
 
   static QStringList supported_formats_;
 

app/render/backend/opengl/openglproxy.cpp

@@ -60,7 +60,9 @@ void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR
   if (stream->type() == Stream::kImage && still_image_cache_.Has(stream.get())) {
     CachedStill cs = still_image_cache_.Get(stream.get());
 
-    if (cs.colorspace == colorspace_match && cs.alpha_is_associated == video_stream->premultiplied_alpha()) {
+    if (cs.colorspace == colorspace_match
+        && cs.alpha_is_associated == video_stream->premultiplied_alpha()
+        && cs.divider == video_params_.divider()) {
       footage_tex_ref = cs.texture;
     } else {
       still_image_cache_.Remove(stream.get());
@@ -80,7 +82,7 @@ void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR
 
     ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode());
 
-    FramePtr frame = decoder->RetrieveVideo(range.in());;
+    FramePtr frame = decoder->RetrieveVideo(range.in(), video_params_.divider());
 
     // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU
     if (ocio_method == ColorManager::kOCIOAccurate) {
@@ -154,7 +156,7 @@ void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR
     }
 
     if (stream->type() == Stream::kImage) {
-      still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha()});
+      still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha(), video_params_.divider()});
     }
   }
 
@@ -281,8 +283,8 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c
             int res_param_location = shader->uniformLocation(QStringLiteral("%1_resolution").arg(input->id()));
             if (res_param_location > -1) {
               shader->setUniformValue(res_param_location,
-                                      static_cast<GLfloat>(texture->texture()->width()),
-                                      static_cast<GLfloat>(texture->texture()->height()));
+                                      static_cast<GLfloat>(texture->texture()->width() * video_params_.divider()),
+                                      static_cast<GLfloat>(texture->texture()->height() * video_params_.divider()));
             }
           }
 

app/render/backend/opengl/openglproxy.h

@@ -69,6 +69,7 @@ private:
     OpenGLTextureCache::ReferencePtr texture;
     QString colorspace;
     bool alpha_is_associated;
+    int divider;
   };
 
   RenderCache<Stream*, CachedStill> still_image_cache_;