decoder/renderer: use memory cache and single decoder to optimize frames for

cycles

app/codec/decoder.h

@@ -21,6 +21,7 @@
 #ifndef DECODER_H
 #define DECODER_H
 
+#include <QMutex>
 #include <QObject>
 #include <stdint.h>
 
@@ -265,6 +266,8 @@ protected:
 
   bool open_;
 
+  QMutex mutex_;
+
 private:
   void ConformInternal(SwrContext *resampler, WaveOutput *output, const char *in_data, int in_sample_count);
 

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -45,13 +45,10 @@ FFmpegDecoder::FFmpegDecoder() :
   fmt_ctx_(nullptr),
   codec_ctx_(nullptr),
   scale_ctx_(nullptr),
-  pkt_(nullptr),
-  last_retrieved_frame_(nullptr),
   opts_(nullptr),
-  multithreading_(false)
+//  multithreading_(false)
+  multithreading_(true)
 {
-  frames_[0] = nullptr;
-  frames_[1] = nullptr;
 }
 
 FFmpegDecoder::~FFmpegDecoder()
@@ -61,6 +58,8 @@ FFmpegDecoder::~FFmpegDecoder()
 
 bool FFmpegDecoder::Open()
 {
+  QMutexLocker locker(&mutex_);
+
   if (open_) {
     return true;
   }
@@ -174,17 +173,10 @@ bool FFmpegDecoder::Open()
       Error(QStringLiteral("Failed to allocate SwsContext"));
       return false;
     }
-  }
 
-  pkt_ = av_packet_alloc();
-  if (!pkt_) {
-    Error(QStringLiteral("Failed to allocate AVPacket"));
-    return false;
+    second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base)));
   }
 
-  frames_[0] = av_frame_alloc();
-  frames_[1] = av_frame_alloc();
-
   // All allocation succeeded so we set the state to open
   open_ = true;
 
@@ -193,7 +185,9 @@ bool FFmpegDecoder::Open()
 
 Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time)
 {
-  if (!open_ && !Open()) {
+  QMutexLocker locker(&mutex_);
+
+  if (!open_) {
     return kFailedToOpen;
   }
 
@@ -212,7 +206,10 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time)
 
 FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 {
-  if (!open_ && !Open()) {
+  QMutexLocker locker(&mutex_);
+
+  if (!open_) {
+    qWarning() << "Tried to retrieve video on a decoder that's still closed";
     return nullptr;
   }
 
@@ -222,41 +219,42 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 
   int64_t target_ts = Timecode::time_to_timestamp(timecode, avstream_->time_base);
 
-  // Allocate frame that we'll return
-  FramePtr frame_container = Frame::Create();
-  frame_container->set_width(avstream_->codecpar->width);
-  frame_container->set_height(avstream_->codecpar->height);
-  frame_container->set_format(native_pix_fmt_);
-  frame_container->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base));
-  frame_container->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr));
-  frame_container->allocate();
-
   bool got_frame = false;
 
-  QByteArray frame_loader;
   uint8_t* input_data[4];
   int input_linesize[4];
 
-  {
-    AVFrame* test_existing = nullptr;
+  // See if our RAM cache already has a frame that matches this timestamp
+  if (!cached_frames_.isEmpty()) {
 
-    if (frames_[0]->pts == target_ts) {
-      got_frame = frames_[0];
-    } else if (frames_[1]->pts == target_ts) {
-      got_frame = frames_[1];
-    }
+    if (target_ts >= cached_frames_.first()->pts
+        && target_ts <= cached_frames_.last()->pts) {
 
-    if (test_existing) {
-      for (int i=0;i<4;i++) {
-        input_data[i] = test_existing->data[i];
-        input_linesize[i] = test_existing->linesize[i];
+      // We already have this frame in the cache, find it
+      for (int i=0;i<cached_frames_.size();i++) {
+        AVFrame* this_frame = cached_frames_.at(i);
+
+        if (this_frame->pts == target_ts // Test for an exact match
+            || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->pts > target_ts)) { // Or for this frame to be the "closest"
+
+          // This frame is appropriate, return it
+          for (int i=0;i<4;i++) {
+            input_data[i] = this_frame->data[i];
+            input_linesize[i] = this_frame->linesize[i];
+          }
+
+          got_frame = true;
+
+          break;
+
+        }
       }
-
-      got_frame = true;
     }
   }
 
   // See if we stored this frame in the disk cache
+  /*
+  QByteArray frame_loader;
   if (!got_frame) {
     QFile compressed_frame(GetIndexFilename().append(QString::number(target_ts)));
     if (compressed_frame.exists()
@@ -266,7 +264,6 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 
       // Read data
       frame_loader = qUncompress(compressed_frame.readAll());
-      //frame_loader = compressed_frame.readAll();
 
       av_image_fill_arrays(input_data,
                            input_linesize,
@@ -279,40 +276,45 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
       got_frame = true;
     }
   }
+  */
 
   // If we have no disk cache, we'll need to find this frame ourselves
   if (!got_frame) {
-    int64_t second_ts = qRound64(av_q2d(av_inv_q(avstream_->time_base)));
 
-    if (!last_retrieved_frame_
-        || last_retrieved_frame_->pts == AV_NOPTS_VALUE
-        || last_retrieved_frame_->pts < target_ts - 2*second_ts
-        || last_retrieved_frame_->pts > target_ts) {
+    // If the frame wasn't in the frame cache, see if this frame cache is too old to use
+    if (cached_frames_.isEmpty()
+        || target_ts < cached_frames_.first()->pts
+        || target_ts > cached_frames_.last()->pts + 2*second_ts_) {
+      ClearFrameCache();
       Seek(target_ts);
     }
 
     int64_t seek_ts = target_ts;
 
     int ret;
+    AVPacket* pkt = av_packet_alloc();
 
-    bool frame_flipper = false;
     bool still_seeking = true;
 
     while (true) {
-      AVFrame* working_frame = frames_[frame_flipper];
+      AVFrame* working_frame = av_frame_alloc();
 
-      ret = GetFrame(pkt_, working_frame);
+      ret = GetFrame(pkt, working_frame);
 
       if (ret < 0) {
         FFmpegError(ret);
+
+        av_frame_free(&working_frame);
         break;
       }
 
       if (still_seeking) {
         if (working_frame->pts > target_ts || working_frame->pts == AV_NOPTS_VALUE) {
           // Seek failed, try again
-          seek_ts -= second_ts;
+          seek_ts -= second_ts_;
           Seek(seek_ts);
+
+          av_frame_free(&working_frame);
           continue;
         } else {
           still_seeking = false;
@@ -323,10 +325,12 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 
       if (working_frame->pts == target_ts) {
         found_frame = working_frame;
-      } else if (working_frame->pts > target_ts && frames_[!frame_flipper]->pts < target_ts) {
-        found_frame = frames_[!frame_flipper];
+      } else if (working_frame->pts > target_ts) {
+        found_frame = cached_frames_.last();
       }
 
+      cached_frames_.append(working_frame);
+
       if (found_frame) {
         // We found the frame we want
         got_frame = true;
@@ -340,13 +344,23 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
         //CacheFrameToDisk(found_frame);
         break;
       }
-
-      frame_flipper = !frame_flipper;
     }
+
+    av_packet_free(&pkt);
   }
 
   // If we're here and got the frame, we'll convert it and return it
   if (got_frame) {
+
+    // Allocate frame that we'll return
+    FramePtr frame_container = Frame::Create();
+    frame_container->set_width(avstream_->codecpar->width);
+    frame_container->set_height(avstream_->codecpar->height);
+    frame_container->set_format(native_pix_fmt_);
+    frame_container->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base));
+    frame_container->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr));
+    frame_container->allocate();
+
     // Convert frame to RGBA for the rest of the pipeline
     uint8_t* output_data = reinterpret_cast<uint8_t*>(frame_container->data());
     int output_linesize = frame_container->width() * kRGBAChannels * PixelService::BytesPerChannel(native_pix_fmt_);
@@ -360,6 +374,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
               &output_linesize);
 
     return frame_container;
+
   }
 
   return nullptr;
@@ -367,7 +382,10 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 
 FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params)
 {
-  if (!open_ && !Open()) {
+  QMutexLocker locker(&mutex_);
+
+  if (!open_) {
+    qWarning() << "Tried to retrieve audio on a decoder that's still closed";
     return nullptr;
   }
 
@@ -401,27 +419,14 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &
 
 void FFmpegDecoder::Close()
 {
+  QMutexLocker locker(&mutex_);
+
   if (opts_) {
     av_dict_free(&opts_);
     opts_ = nullptr;
   }
 
-  last_retrieved_frame_ = nullptr;
-
-  if (frames_[1]) {
-    av_frame_free(&frames_[1]);
-    frames_[1] = nullptr;
-  }
-
-  if (frames_[0]) {
-    av_frame_free(&frames_[0]);
-    frames_[0] = nullptr;
-  }
-
-  if (pkt_) {
-    av_packet_free(&pkt_);
-    pkt_ = nullptr;
-  }
+  ClearFrameCache();
 
   if (scale_ctx_) {
     sws_freeContext(scale_ctx_);
@@ -638,10 +643,10 @@ void FFmpegDecoder::Index(const QAtomicInt* cancelled)
 
         input.close();
       }
+    } else {
+      UnconditionalAudioIndex(cancelled);
     }
 
-    UnconditionalAudioIndex(pkt_, frames_[0], cancelled);
-
   }
 }
 
@@ -656,7 +661,7 @@ QString FFmpegDecoder::GetIndexFilename()
       .append(QString::number(avstream_->index));
 }
 
-void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const QAtomicInt* cancelled)
+void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled)
 {
   // Iterate through each audio frame and extract the PCM data
 
@@ -703,6 +708,8 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
                                    FFmpegCommon::GetNativeSampleFormat(dst_sample_fmt));
   WaveOutput wave_out(GetIndexFilename(), wave_params);
 
+  AVPacket* pkt = av_packet_alloc();
+  AVFrame* frame = av_frame_alloc();
   int ret;
 
   if (wave_out.open()) {
@@ -777,10 +784,13 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
     swr_free(&resampler);
   }
 
+  av_frame_free(&frame);
+  av_packet_free(&pkt);
+
   Seek(0);
 }
 
-void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled)
+void FFmpegDecoder::UnconditionalVideoIndex(const QAtomicInt* cancelled)
 {
   VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());
 
@@ -792,6 +802,8 @@ void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const
   // Iterate through every single frame and get each timestamp
   // NOTE: Expects no frames to have been read so far
 
+  AVPacket* pkt = av_packet_alloc();
+  AVFrame* frame = av_frame_alloc();
   int ret;
 
   while (true) {
@@ -826,6 +838,9 @@ void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const
     }
   }
 
+  av_frame_free(&frame);
+  av_packet_free(&pkt);
+
   Seek(0);
 }
 
@@ -886,7 +901,7 @@ void FFmpegDecoder::ValidateVideoIndex(const QAtomicInt* cancelled)
     // Reset state
     Seek(0);
 
-    UnconditionalVideoIndex(pkt_, frames_[0], cancelled);
+    UnconditionalVideoIndex(cancelled);
 
     Seek(0);
   }
@@ -926,3 +941,11 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f)
     DiskManager::instance()->CreatedFile(save_frame.fileName(), QByteArray());
   }
 }
+
+void FFmpegDecoder::ClearFrameCache()
+{
+  for (int i=0;i<cached_frames_.size();i++) {
+    av_frame_free(&cached_frames_[i]);
+  }
+  cached_frames_.clear();
+}

app/codec/ffmpeg/ffmpegdecoder.h

@@ -95,8 +95,8 @@ private:
 
   virtual QString GetIndexFilename() override;
 
-  void UnconditionalAudioIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled);
-  void UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled);
+  void UnconditionalAudioIndex(const QAtomicInt* cancelled);
+  void UnconditionalVideoIndex(const QAtomicInt* cancelled);
 
   void ValidateVideoIndex(const QAtomicInt* cancelled);
 
@@ -104,6 +104,8 @@ private:
 
   void CacheFrameToDisk(AVFrame* f);
 
+  void ClearFrameCache();
+
   AVFormatContext* fmt_ctx_;
   AVCodecContext* codec_ctx_;
   AVStream* avstream_;
@@ -113,10 +115,9 @@ private:
 
   SwsContext* scale_ctx_;
 
-  AVPacket* pkt_;
+  QList<AVFrame*> cached_frames_;
 
-  AVFrame* frames_[2];
-  AVFrame* last_retrieved_frame_;
+  int64_t second_ts_;
 
   AVDictionary* opts_;
 

app/codec/oiio/oiiodecoder.cpp

@@ -135,7 +135,7 @@ bool OIIODecoder::Open()
 
 Decoder::RetrieveState OIIODecoder::GetRetrieveState(const rational &time)
 {
-  if (!open_ && !Open()) {
+  if (!open_) {
     return kFailedToOpen;
   }
 

app/render/backend/audio/audiobackend.cpp

@@ -24,7 +24,7 @@ bool AudioBackend::InitInternal()
   // Initiate one thread per CPU core
   for (int i=0;i<threads().size();i++) {
     // Create one processor object for each thread
-    AudioWorker* processor = new AudioWorker();
+    AudioWorker* processor = new AudioWorker(decoder_cache());
     processor->SetParameters(params());
     processors_.append(processor);
   }

app/render/backend/audio/audioworker.cpp

@@ -1,7 +1,7 @@
 #include "audioworker.h"
 
-AudioWorker::AudioWorker(QObject *parent) :
-  AudioRenderWorker(parent)
+AudioWorker::AudioWorker(DecoderCache* decoder_cache, QObject *parent) :
+  AudioRenderWorker(decoder_cache, parent)
 {
 }
 

app/render/backend/audio/audioworker.h

@@ -6,7 +6,7 @@
 class AudioWorker : public AudioRenderWorker
 {
 public:
-  AudioWorker(QObject* parent = nullptr);
+  AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr);
 
 protected:
   virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override;

app/render/backend/audiorenderworker.cpp

@@ -2,8 +2,8 @@
 
 #include "audio/audiomanager.h"
 
-AudioRenderWorker::AudioRenderWorker(QObject *parent) :
-  RenderWorker(parent)
+AudioRenderWorker::AudioRenderWorker(DecoderCache* decoder_cache, QObject *parent) :
+  RenderWorker(decoder_cache, parent)
 {
 }
 

app/render/backend/audiorenderworker.h

@@ -7,7 +7,7 @@ class AudioRenderWorker : public RenderWorker
 {
   Q_OBJECT
 public:
-  AudioRenderWorker(QObject* parent = nullptr);
+  AudioRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr);
 
   void SetParameters(const AudioRenderingParams& audio_params);
 

app/render/backend/opengl/openglbackend.cpp

@@ -39,7 +39,7 @@ bool OpenGLBackend::InitInternal()
   // Initiate one thread per CPU core
   for (int i=0;i<threads().size();i++) {
     // Create one processor object for each thread
-    OpenGLWorker* processor = new OpenGLWorker(frame_cache());
+    OpenGLWorker* processor = new OpenGLWorker(frame_cache(), decoder_cache());
     processor->SetParameters(params());
     processors_.append(processor);
 

app/render/backend/opengl/openglworker.cpp

@@ -9,8 +9,8 @@
 #include "render/colormanager.h"
 #include "render/pixelservice.h"
 
-OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, QObject *parent) :
-  VideoRenderWorker(frame_cache, parent)
+OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, DecoderCache* decoder_cache, QObject *parent) :
+  VideoRenderWorker(frame_cache, decoder_cache, parent)
 {
 }
 

app/render/backend/opengl/openglworker.h

@@ -12,7 +12,7 @@
 class OpenGLWorker : public VideoRenderWorker {
   Q_OBJECT
 public:
-  OpenGLWorker(VideoRenderFrameCache* frame_cache,
+  OpenGLWorker(VideoRenderFrameCache* frame_cache, DecoderCache *decoder_cache,
                QObject* parent = nullptr);
 
 signals:

app/render/backend/renderbackend.cpp

@@ -83,6 +83,8 @@ void RenderBackend::Close()
 
   threads_.clear();
   processors_.clear();
+
+  decoder_cache_.Clear();
 }
 
 const QString &RenderBackend::GetError() const
@@ -375,6 +377,11 @@ void RenderBackend::SetWorkerBusyState(RenderWorker *worker, bool busy)
   processor_busy_state_.replace(processors_.indexOf(worker), busy);
 }
 
+DecoderCache *RenderBackend::decoder_cache()
+{
+  return &decoder_cache_;
+}
+
 bool RenderBackend::AllProcessorsAreAvailable() const
 {
   foreach (bool busy, processor_busy_state_) {

app/render/backend/renderbackend.h

@@ -96,10 +96,14 @@ protected:
   bool WorkerIsBusy(RenderWorker* worker) const;
   void SetWorkerBusyState(RenderWorker* worker, bool busy);
 
+  DecoderCache* decoder_cache();
+
   TimeRangeList cache_queue_;
 
   QVector<RenderWorker*> processors_;
 
+  DecoderCache decoder_cache_;
+
   bool compiled_;
 
   QHash<TimeRange, qint64> render_job_info_;

app/render/backend/rendercache.h

@@ -18,41 +18,11 @@ public:
 
   bool Has(K key) const {return values_.contains(key);}
 
-private:
-  QHash<K, V> values_;
-};
+  void Lock() { lock_.lock(); }
 
-template<class K, class V>
-class ThreadSafeRenderCache
-{
-public:
-  ThreadSafeRenderCache() = default;
+  void Unlock() { lock_.unlock(); }
 
-  void Clear() {
-    lock_.lock();
-    values_.clear();
-    lock_.unlock();
-  }
-
-  void Add(K key, V val) {
-    lock_.lock();
-    values_.insert(key, val);
-    lock_.unlock();
-  }
-
-  V Get(K key) {
-    lock_.lock();
-    V val = values_.value(key);
-    lock_.unlock();
-    return val;
-  }
-
-  bool Has(K key) {
-    lock_.lock();
-    bool has = values_.contains(key);
-    lock_.unlock();
-    return has;
-  }
+  QMutex* lock() { return &lock_; }
 
 private:
   QHash<K, V> values_;

app/render/backend/renderworker.cpp

@@ -4,9 +4,10 @@
 
 #include "node/block/block.h"
 
-RenderWorker::RenderWorker(QObject *parent) :
+RenderWorker::RenderWorker(DecoderCache *decoder_cache, QObject *parent) :
   QObject(parent),
-  started_(false)
+  started_(false),
+  decoder_cache_(decoder_cache)
 {
 }
 
@@ -27,8 +28,6 @@ void RenderWorker::Close()
 {
   CloseInternal();
 
-  decoder_cache_.Clear();
-
   started_ = false;
 }
 
@@ -61,14 +60,22 @@ StreamPtr RenderWorker::ResolveStreamFromInput(NodeInput *input)
 
 DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream)
 {
+  QMutexLocker locker(decoder_cache_->lock());
+
   // Access a map of Node inputs and decoder instances and retrieve a frame!
-  DecoderPtr decoder = decoder_cache_.Get(stream.get());
+  DecoderPtr decoder = decoder_cache_->Get(stream.get());
 
   if (decoder == nullptr && stream != nullptr) {
     // Create a new Decoder here
     decoder = Decoder::CreateFromID(stream->footage()->decoder());
     decoder->set_stream(stream);
-    decoder_cache_.Add(stream.get(), decoder);
+
+    if (decoder->Open()) {
+      decoder_cache_->Add(stream.get(), decoder);
+    } else {
+      decoder = nullptr;
+      qWarning() << "Failed to open decoder for" << stream->footage()->filename();
+    }
   }
 
   return decoder;

app/render/backend/renderworker.h

@@ -13,7 +13,7 @@ class RenderWorker : public QObject, public CancelableObject
 {
   Q_OBJECT
 public:
-  RenderWorker(QObject* parent = nullptr);
+  RenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr);
 
   DISABLE_COPY_MOVE(RenderWorker)
 
@@ -62,7 +62,7 @@ private:
 
   bool started_;
 
-  DecoderCache decoder_cache_;
+  DecoderCache* decoder_cache_;
 
   NodeDependency path_;
 

app/render/backend/videorenderworker.cpp

@@ -6,8 +6,8 @@
 #include "project/project.h"
 #include "render/pixelservice.h"
 
-VideoRenderWorker::VideoRenderWorker(VideoRenderFrameCache *frame_cache, QObject *parent) :
-  RenderWorker(parent),
+VideoRenderWorker::VideoRenderWorker(VideoRenderFrameCache *frame_cache, DecoderCache* decoder_cache, QObject *parent) :
+  RenderWorker(decoder_cache, parent),
   frame_cache_(frame_cache),
   operating_mode_(kHashRenderCache)
 {

app/render/backend/videorenderworker.h

@@ -41,7 +41,7 @@ public:
     kHashRenderCache = 0x7
   };
 
-  VideoRenderWorker(VideoRenderFrameCache* frame_cache, QObject* parent = nullptr);
+  VideoRenderWorker(VideoRenderFrameCache* frame_cache, DecoderCache *decoder_cache, QObject* parent = nullptr);
 
   void SetParameters(const VideoRenderingParams& video_params);