ffmpegdecoder: use avfilter system instead of raw swscale commands

Heh this was inevitable

app/codec/decoder.cpp

@@ -93,7 +93,7 @@ bool Decoder::Open(const CodecStream &stream)
   }
 }
 
-FramePtr Decoder::RetrieveVideo(const rational &timecode, const int &divider)
+FramePtr Decoder::RetrieveVideo(const rational &timecode, const RetrieveVideoParams &divider)
 {
   QMutexLocker locker(&mutex_);
 
@@ -302,7 +302,7 @@ int64_t Decoder::GetImageSequenceIndex(const QString &filename)
   return number_only.toLongLong();
 }
 
-FramePtr Decoder::RetrieveVideoInternal(const rational &timecode, const int &divider)
+FramePtr Decoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams &divider)
 {
   Q_UNUSED(timecode)
   Q_UNUSED(divider)

app/codec/decoder.h

@@ -143,6 +143,35 @@ public:
 
   static const rational kAnyTimecode;
 
+  struct RetrieveVideoParams
+  {
+    RetrieveVideoParams()
+    {
+      divider = 1;
+      src_interlacing = VideoParams::kInterlaceNone;
+      dst_interlacing = VideoParams::kInterlaceNone;
+    }
+
+    int divider;
+    VideoParams::Interlacing src_interlacing;
+    VideoParams::Interlacing dst_interlacing;
+
+    void reset()
+    {
+      *this = RetrieveVideoParams();
+    }
+
+    bool operator==(const RetrieveVideoParams& rhs) const
+    {
+      return divider == rhs.divider && src_interlacing == rhs.src_interlacing && dst_interlacing == rhs.dst_interlacing;
+    }
+
+    bool operator!=(const RetrieveVideoParams& rhs) const
+    {
+      return !(*this == rhs);
+    }
+  };
+
   /**
    * @brief Retrieves a video frame from footage
    *
@@ -153,7 +182,7 @@ public:
    *
    * This function is thread safe and can only run while the decoder is open. \see Open()
    */
-  FramePtr RetrieveVideo(const rational& timecode, const int& divider);
+  FramePtr RetrieveVideo(const rational& timecode, const RetrieveVideoParams& divider);
 
   /**
    * @brief Retrieve audio data from footage
@@ -236,7 +265,7 @@ protected:
    * Sub-classes must override this function IF they support video. Function is already mutexed
    * so sub-classes don't need to worry about thread safety.
    */
-  virtual FramePtr RetrieveVideoInternal(const rational& timecode, const int& divider);
+  virtual FramePtr RetrieveVideoInternal(const rational& timecode, const RetrieveVideoParams& divider);
 
   virtual bool ConformAudioInternal(const QString& filename, const AudioParams &params, const QAtomicInt* cancelled);
 

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -22,6 +22,8 @@
 
 extern "C" {
 #include <libavcodec/avcodec.h>
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
 #include <libavformat/avformat.h>
 #include <libavutil/imgutils.h>
 #include <libavutil/pixdesc.h>
@@ -48,8 +50,9 @@ extern "C" {
 namespace olive {
 
 FFmpegDecoder::FFmpegDecoder() :
-  scale_ctx_(nullptr),
-  scale_divider_(0),
+  filter_graph_(nullptr),
+  buffersrc_ctx_(nullptr),
+  buffersink_ctx_(nullptr),
   pool_(QThread::idealThreadCount()*2),
   is_working_(false),
   cache_at_zero_(false),
@@ -148,28 +151,12 @@ bool FFmpegDecoder::OpenInternal()
   return output_frame;
 }*/
 
-FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const int &divider)
+FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams &params)
 {
-  if (scale_divider_ != divider) {
-    FreeScaler();
-    InitScaler(divider);
-  }
-
   AVStream* s = instance_.avstream();
 
-  int divided_width = VideoParams::GetScaledDimension(s->codecpar->width, divider);
-  int divided_height = VideoParams::GetScaledDimension(s->codecpar->height, divider);
-
-  if (pool_.width() != divided_width || pool_.height() != divided_height) {
-    // Clear all instance queues
-    ClearFrameCache();
-
-    // Set new frame pool parameters
-    pool_.SetParameters(divided_width, divided_height, native_pix_fmt_, native_channel_count_);
-  }
-
   // Retrieve frame
-  FFmpegFramePool::ElementPtr return_frame = RetrieveFrame(timecode, divider);
+  FFmpegFramePool::ElementPtr return_frame = RetrieveFrame(timecode, params);
 
   // We found the frame, we'll return a copy
   if (return_frame) {
@@ -180,7 +167,7 @@ FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const in
                                        native_channel_count_,
                                        av_guess_sample_aspect_ratio(instance_.fmt_ctx(), s, nullptr), // May be incorrect,
                                        VideoParams::kInterlaceNone, // May be incorrect
-                                       divider));
+                                       filter_params_.divider));
     copy->set_timestamp(timecode);
     copy->allocate();
 
@@ -198,8 +185,42 @@ void FFmpegDecoder::CloseInternal()
   ClearFrameCache();
 
   instance_.Close();
+}
 
-  FreeScaler();
+int FFmpegDecoder::GetFilteredFrame(AVPacket* packet, AVFrame* output_frame, const RetrieveVideoParams& params)
+{
+  // Ensure scaler is correct for these parameters
+  if (!InitScaler(params)) {
+    return AVERROR(EINVAL);
+  }
+
+  int ret;
+
+  AVFrame* working_frame = av_frame_alloc();
+
+  // Try to pull frame from buffersink
+  while ((ret = av_buffersink_get_frame(buffersink_ctx_, output_frame)) == AVERROR(EAGAIN)) {
+    // If no frame is ready in the buffersink, pull from codec
+    ret = instance_.GetFrame(packet, working_frame);
+
+    if (ret >= 0) {
+      // If succeeded in pulling from codec, send to buffer source
+      ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, working_frame, AV_BUFFERSRC_FLAG_KEEP_REF);
+
+      if (ret < 0) {
+        // If failed to send to buffer source, return break and error code
+        qDebug() << "Failed to feed filter graph:" << FFmpegError(ret);
+        break;
+      }
+    } else {
+      // If failed to read from decoder, return break and error code
+      break;
+    }
+  }
+
+  av_frame_free(&working_frame);
+
+  return ret;
 }
 
 QString FFmpegDecoder::id() const
@@ -538,17 +559,6 @@ uint64_t FFmpegDecoder::ValidateChannelLayout(AVStream* stream)
   return av_get_default_channel_layout(stream->codecpar->channels);
 }
 
-void FFmpegDecoder::FFmpegBufferToNativeBuffer(uint8_t **input_data, int *input_linesize, uint8_t** output_buffer, int* output_linesize)
-{
-  sws_scale(scale_ctx_,
-            input_data,
-            input_linesize,
-            0,
-            instance_.avstream()->codecpar->height,
-            output_buffer,
-            output_linesize);
-}
-
 /* OLD UNUSED CODE: Keeping this around in case the code proves useful
 
 void FFmpegDecoder::CacheFrameToDisk(AVFrame *f)
@@ -611,9 +621,12 @@ void FFmpegDecoder::ClearFrameCache()
   cached_frames_.clear();
   cache_at_eof_ = false;
   cache_at_zero_ = false;
+
+  // Filter graph may rely on "continuous" video frames, so we free the scaler here
+  FreeScaler();
 }
 
-FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, int divider)
+FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, const RetrieveVideoParams &params)
 {
   int64_t target_ts = GetTimeInTimebaseUnits(time, instance_.avstream()->time_base, instance_.avstream()->start_time);
   int64_t seek_ts = target_ts;
@@ -650,7 +663,7 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, i
   while (true) {
 
     // Pull from the decoder
-    ret = instance_.GetFrame(pkt, working_frame);
+    ret = GetFilteredFrame(pkt, working_frame, params);
 
     // Handle any errors that aren't EOF (EOF is handled later on)
     if (ret < 0 && ret != AVERROR_EOF) {
@@ -707,8 +720,9 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, i
 
       // Store in queue, converting to native format
       uint8_t* destination_data = cached->data();
-      int destination_linesize = Frame::generate_linesize_bytes(VideoParams::GetScaledDimension(instance_.avstream()->codecpar->width, divider), native_pix_fmt_, native_channel_count_);
-      FFmpegBufferToNativeBuffer(working_frame->data, working_frame->linesize, &destination_data, &destination_linesize);
+      int destination_linesize = Frame::generate_linesize_bytes(working_frame->width, native_pix_fmt_, native_channel_count_);
+
+      av_image_copy(&destination_data, &destination_linesize, const_cast<const uint8_t**>(working_frame->data), working_frame->linesize, static_cast<AVPixelFormat>(working_frame->format), working_frame->width, working_frame->height);
 
       // Set timestamp so this frame can be identified later
       cached->set_timestamp(working_frame->pts);
@@ -746,81 +760,124 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, i
   return return_frame;
 }
 
-void FFmpegDecoder::InitScaler(int divider)
+bool FFmpegDecoder::InitScaler(const RetrieveVideoParams& params)
 {
-  int src_width = instance_.avstream()->codecpar->width;
-  int src_height = instance_.avstream()->codecpar->height;
-
-  int scaled_width = VideoParams::GetScaledDimension(src_width, divider);
-  int scaled_height = VideoParams::GetScaledDimension(src_height, divider);
-
-  scale_ctx_ = sws_getContext(src_width,
-                              src_height,
-                              static_cast<AVPixelFormat>(instance_.avstream()->codecpar->format),
-                              scaled_width,
-                              scaled_height,
-                              ideal_pix_fmt_,
-                              SWS_FAST_BILINEAR,
-                              nullptr,
-                              nullptr,
-                              nullptr);
-
-  if (scale_ctx_) {
-    scale_divider_ = divider;
-  } else {
-    scale_divider_ = 0;
+  if (params == filter_params_ && filter_graph_) {
+    // We have an appropriate filter for these parameters, just return true
+    return true;
   }
+
+  // We need to (re)create the filter, delete current if necessary
+  ClearFrameCache();
+
+  // Set our params to this
+  filter_params_ = params;
+
+  // Allocate filter graph
+  filter_graph_ = avfilter_graph_alloc();
+  if (!filter_graph_) {
+    qWarning() << "Failed to allocate filter graph";
+    return false;
+  }
+
+  AVStream* s = instance_.avstream();
+
+  int src_width = s->codecpar->width;
+  int src_height = s->codecpar->height;
+
+  // Define filter parameters
+  static const int kFilterArgSz = 1024;
+  char filter_args[kFilterArgSz];
+  snprintf(filter_args, kFilterArgSz, "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
+           src_width,
+           src_height,
+           s->codecpar->format,
+           s->time_base.num,
+           s->time_base.den,
+           s->codecpar->sample_aspect_ratio.num,
+           s->codecpar->sample_aspect_ratio.den);
+
+  // Create path in and out of the filter graph (the buffer in and the buffersink out)
+  avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("buffer"), "in", filter_args, nullptr, filter_graph_);
+  avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("buffersink"), "out", nullptr, nullptr, filter_graph_);
+
+  // Link filters as necessary
+  AVFilterContext *last_filter = buffersrc_ctx_;
+
+  // Add interlacing filter if necessary
+  if (filter_params_.src_interlacing != filter_params_.dst_interlacing) {
+    // Determine what kind of interlacing we'll be doing
+    if (filter_params_.dst_interlacing == VideoParams::kInterlaceNone) {
+      // Deinterlace source
+    } else if (filter_params_.src_interlacing == VideoParams::kInterlaceNone) {
+      // We will be interlacing a previous progressive source
+    } else {
+      // We will simply be flipping the fields
+    }
+  }
+
+  // Add scale filter if necessary
+  int dst_width, dst_height;
+  if (filter_params_.divider > 1) {
+    AVFilterContext* scale_filter;
+
+    dst_width = VideoParams::GetScaledDimension(src_width, filter_params_.divider);
+    dst_height = VideoParams::GetScaledDimension(src_height, filter_params_.divider);
+
+    snprintf(filter_args, kFilterArgSz, "w=%d:h=%d:flags=fast_bilinear:interl=%d",
+             dst_width,
+             dst_height,
+             params.dst_interlacing != VideoParams::kInterlaceNone);
+
+    avfilter_graph_create_filter(&scale_filter, avfilter_get_by_name("scale"), "scale", filter_args, nullptr, filter_graph_);
+
+    avfilter_link(last_filter, 0, scale_filter, 0);
+    last_filter = scale_filter;
+  } else {
+    dst_width = src_width;
+    dst_height = src_height;
+  }
+
+  // Add format filter if necessary
+  if (ideal_pix_fmt_ != s->codecpar->format) {
+    AVFilterContext* format_filter;
+
+    snprintf(filter_args, kFilterArgSz, "pix_fmts=%u", ideal_pix_fmt_);
+
+    avfilter_graph_create_filter(&format_filter, avfilter_get_by_name("format"), "format", filter_args, nullptr, filter_graph_);
+
+    avfilter_link(last_filter, 0, format_filter, 0);
+    last_filter = format_filter;
+  }
+
+  // Finally, link the last filter with the buffersink
+  avfilter_link(last_filter, 0, buffersink_ctx_, 0);
+
+  // Configure graph
+  if (int ret = avfilter_graph_config(filter_graph_, nullptr) < 0) {
+    qDebug() << "Failed to configure graph:" << FFmpegError(ret);
+    return false;
+  }
+
+  // Configure frame pool
+  if (pool_.width() != dst_width || pool_.height() != dst_height) {
+    // Set new frame pool parameters
+    pool_.SetParameters(dst_width, dst_height, native_pix_fmt_, native_channel_count_);
+  }
+
+  return true;
 }
 
 void FFmpegDecoder::FreeScaler()
 {
-  if (scale_ctx_) {
-    sws_freeContext(scale_ctx_);
-    scale_ctx_ = nullptr;
-
-    scale_divider_ = 0;
+  if (filter_graph_) {
+    avfilter_graph_free(&filter_graph_);
+    filter_graph_ = nullptr;
+    buffersrc_ctx_ = nullptr;
+    buffersink_ctx_ = nullptr;
   }
 }
 
-/*int64_t FFmpegDecoder::RangeStart() const
-{
-  if (cached_frames_.isEmpty()) {
-    return AV_NOPTS_VALUE;
-  }
-  return cached_frames_.first()->timestamp();
-}
-
-int64_t FFmpegDecoder::RangeEnd() const
-{
-  if (cached_frames_.isEmpty()) {
-    return AV_NOPTS_VALUE;
-  }
-  return cached_frames_.last()->timestamp();
-}
-
-bool FFmpegDecoder::CacheContainsTime(const int64_t &t) const
-{
-  return !cached_frames_.isEmpty()
-      && ((RangeStart() <= t && RangeEnd() >= t)
-          || (cache_at_zero_ && t < cached_frames_.first()->timestamp())
-          || (cache_at_eof_ && t > cached_frames_.last()->timestamp()));
-}
-
-bool FFmpegDecoder::CacheWillContainTime(const int64_t &t) const
-{
-  return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= cache_target_time_;
-}
-
-bool FFmpegDecoder::CacheCouldContainTime(const int64_t &t) const
-{
-  return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= (cache_target_time_ + 2*second_ts_);
-}
-
-bool FFmpegDecoder::CacheIsEmpty() const
-{
-  return cached_frames_.isEmpty();
-}*/
-
 FFmpegFramePool::ElementPtr FFmpegDecoder::GetFrameFromCache(const int64_t &t) const
 {
   if (t < cached_frames_.first()->timestamp()) {
@@ -856,39 +913,6 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::GetFrameFromCache(const int64_t &t) c
   return nullptr;
 }
 
-/*void FFmpegDecoder::RemoveFramesBefore(const qint64 &t)
-{
-  while (!cached_frames_.isEmpty() && cached_frames_.first()->last_accessed() < t) {
-    RemoveFirstFrame();
-  }
-}
-
-int FFmpegDecoder::TruncateCacheRangeToTime(const qint64 &t)
-{
-  int counter = 0;
-
-  // We keep one frame in memory as an identifier for what pts the decoder is up to
-  while (cached_frames_.size() > 1 && (RangeEnd() - RangeStart()) > t) {
-    RemoveFirstFrame();
-    counter++;
-  }
-
-  return counter;
-}
-
-int FFmpegDecoder::TruncateCacheRangeToFrames(int nb_frames)
-{
-  int counter = 0;
-
-  // We keep one frame in memory as an identifier for what pts the decoder is up to
-  while (cached_frames_.size() > nb_frames) {
-    RemoveFirstFrame();
-    counter++;
-  }
-
-  return counter;
-}*/
-
 void FFmpegDecoder::RemoveFirstFrame()
 {
   cached_frames_.removeFirst();

app/codec/ffmpeg/ffmpegdecoder.h

@@ -21,7 +21,11 @@
 #ifndef FFMPEGDECODER_H
 #define FFMPEGDECODER_H
 
+// Fixes weird define issue when including <avfilter.h>
+#include <inttypes.h>
+
 extern "C" {
+#include <libavfilter/avfilter.h>
 #include <libavformat/avformat.h>
 #include <libswscale/swscale.h>
 #include <libswresample/swresample.h>
@@ -60,7 +64,7 @@ public:
 
 protected:
   virtual bool OpenInternal() override;
-  virtual FramePtr RetrieveVideoInternal(const rational &timecode, const int& divider) override;
+  virtual FramePtr RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams& params) override;
   virtual bool ConformAudioInternal(const QString& filename, const AudioParams &params, const QAtomicInt* cancelled) override;
   virtual void CloseInternal() override;
 
@@ -108,6 +112,8 @@ private:
 
   };
 
+  int GetFilteredFrame(AVPacket *packet, AVFrame *frame, const RetrieveVideoParams &params);
+
   /**
    * @brief Handle an FFmpeg error code
    *
@@ -118,28 +124,26 @@ private:
    */
   static QString FFmpegError(int error_code);
 
-  void InitScaler(int divider);
+  bool InitScaler(const RetrieveVideoParams &params);
   void FreeScaler();
 
-  //FramePtr RetrieveStillImage(const rational& timecode, const int& divider);
-
   static VideoParams::Format GetNativePixelFormat(AVPixelFormat pix_fmt);
   static int GetNativeChannelCount(AVPixelFormat pix_fmt);
 
   static uint64_t ValidateChannelLayout(AVStream *stream);
 
-  void FFmpegBufferToNativeBuffer(uint8_t** input_data, int* input_linesize, uint8_t **output_buffer, int *output_linesize);
-
   FFmpegFramePool::ElementPtr GetFrameFromCache(const int64_t& t) const;
 
   void ClearFrameCache();
 
-  FFmpegFramePool::ElementPtr RetrieveFrame(const rational &time, int divider);
+  FFmpegFramePool::ElementPtr RetrieveFrame(const rational &time, const RetrieveVideoParams &params);
 
   void RemoveFirstFrame();
 
-  SwsContext* scale_ctx_;
-  int scale_divider_;
+  RetrieveVideoParams filter_params_;
+  AVFilterGraph* filter_graph_;
+  AVFilterContext* buffersrc_ctx_;
+  AVFilterContext* buffersink_ctx_;
   AVPixelFormat ideal_pix_fmt_;
   VideoParams::Format native_pix_fmt_;
   int native_channel_count_;

app/codec/oiio/oiiodecoder.cpp

@@ -106,7 +106,7 @@ bool OIIODecoder::OpenInternal()
   return OpenImageHandler(stream().filename());
 }
 
-FramePtr OIIODecoder::RetrieveVideoInternal(const rational &timecode, const int& divider)
+FramePtr OIIODecoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams &divider)
 {
   Q_UNUSED(timecode)
 
@@ -118,10 +118,10 @@ FramePtr OIIODecoder::RetrieveVideoInternal(const rational &timecode, const int&
                                       channel_count_,
                                       OIIOUtils::GetPixelAspectRatioFromOIIO(buffer_->spec()),
                                       VideoParams::kInterlaceNone, // FIXME: Does OIIO deinterlace for us?
-                                      divider));
+                                      divider.divider));
   frame->allocate();
 
-  if (divider == 1) {
+  if (divider.divider == 1) {
 
     OIIOUtils::BufferToFrame(buffer_, frame.get());
 

app/codec/oiio/oiiodecoder.h

@@ -44,7 +44,7 @@ public:
 
 protected:
   virtual bool OpenInternal() override;
-  virtual FramePtr RetrieveVideoInternal(const rational &timecode, const int& divider) override;
+  virtual FramePtr RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams& divider) override;
   virtual void CloseInternal() override;
 
 private:

app/render/renderprocessor.cpp

@@ -361,7 +361,12 @@ QVariant RenderProcessor::ProcessVideoFootage(const FootageJob &stream, const ra
     }
 
     if (decoder) {
-      FramePtr frame = decoder->RetrieveVideo((stream_data.video_type() == VideoParams::kVideoTypeVideo) ? input_time : Decoder::kAnyTimecode, footage_divider);
+      Decoder::RetrieveVideoParams p;
+      p.divider = footage_divider;
+      p.src_interlacing = stream_data.interlacing();
+      p.dst_interlacing = GetCacheVideoParams().interlacing();
+
+      FramePtr frame = decoder->RetrieveVideo((stream_data.video_type() == VideoParams::kVideoTypeVideo) ? input_time : Decoder::kAnyTimecode, p);
 
       if (frame) {
         // Return a texture from the derived class