ffmpeg: allow decoders immediate access to frames even while indexing

Previously, FFmpeg decoders needed to wait for the initial index to finish
before they could retrieve frames. Now they can retrieve frames while the
index is occurring, provided the appropriate frame has already been indexed (if
not, the retrieve will need to wait still).

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -380,8 +380,6 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &
 
 void FFmpegDecoder::Close()
 {
-  frame_index_.clear();
-
   if (opts_) {
     av_dict_free(&opts_);
     opts_ = nullptr;
@@ -676,7 +674,7 @@ bool FFmpegDecoder::Probe(Footage *f)
 
     // Use last frame index as the duration
     // FIXME: Does this skip the last frame?
-    int64_t duration = frame_index_.last();
+    int64_t duration = std::static_pointer_cast<VideoStream>(stream())->last_frame_index_timestamp();
 
     f->stream(0)->set_duration(duration);
 
@@ -719,25 +717,17 @@ void FFmpegDecoder::Index()
     return;
   }
 
-  stream()->index_lock_.lock();
+  QMutexLocker locker(stream()->index_process_lock());
 
-  if (!LoadIndex()) {
+  if (stream()->type() == Stream::kVideo) {
 
-    // Reset state
-    Seek(0);
+    ValidateVideoIndex();
 
-    if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
-      IndexVideo(pkt_, frame_);
-    } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
-      IndexAudio(pkt_, frame_);
+  } else if (stream()->type() == Stream::kAudio) {
+    if (!QFileInfo::exists(GetIndexFilename())) {
+      UnconditionalAudioIndex(pkt_, frame_);
     }
-
-    // Reset state
-    Seek(0);
-
   }
-
-  stream()->index_lock_.unlock();
 }
 
 QString FFmpegDecoder::GetIndexFilename()
@@ -777,62 +767,12 @@ QString FFmpegDecoder::GetConformedFilename(const AudioRenderingParams &params)
   return index_fn;
 }
 
-bool FFmpegDecoder::LoadIndex()
-{
-  switch (avstream_->codecpar->codec_type) {
-  case AVMEDIA_TYPE_VIDEO:
-  {
-    // Load index from file
-    QFile index_file(GetIndexFilename());
-
-    if (!index_file.exists()) {
-      return false;
-    }
-
-    if (index_file.open(QFile::ReadOnly)) {
-      // Resize based on filesize
-      frame_index_.resize(static_cast<int>(static_cast<size_t>(index_file.size()) / sizeof(int64_t)));
-
-      // Read frame index into vector
-      index_file.read(reinterpret_cast<char*>(frame_index_.data()),
-                      index_file.size());
-
-      index_file.close();
-
-      return true;
-    }
-    break;
-  }
-  case AVMEDIA_TYPE_AUDIO:
-  {
-    return QFileInfo::exists(GetIndexFilename());
-  }
-  default:
-    break;
-  }
-
-  return false;
-}
-
-void FFmpegDecoder::SaveIndex()
-{
-  // Save index to file
-  QFile index_file(GetIndexFilename());
-  if (index_file.open(QFile::WriteOnly)) {
-    // Write index in binary
-    index_file.write(reinterpret_cast<const char*>(frame_index_.constData()),
-                     frame_index_.size() * static_cast<int>(sizeof(int64_t)));
-
-    index_file.close();
-  } else {
-    qWarning() << QStringLiteral("Failed to save index for %1").arg(stream()->footage()->filename());
-  }
-}
-
-void FFmpegDecoder::IndexAudio(AVPacket *pkt, AVFrame *frame)
+void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame)
 {
   // Iterate through each audio frame and extract the PCM data
 
+  Seek(0);
+
   uint64_t channel_layout = avstream_->codecpar->channel_layout;
   if (!channel_layout) {
     if (!avstream_->codecpar->channels) {
@@ -925,12 +865,18 @@ void FFmpegDecoder::IndexAudio(AVPacket *pkt, AVFrame *frame)
   if (resampler != nullptr) {
     swr_free(&resampler);
   }
+
+  Seek(0);
 }
 
-void FFmpegDecoder::IndexVideo(AVPacket* pkt, AVFrame* frame)
+void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame)
 {
+  VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());
+
+  Seek(0);
+
   // This should be unnecessary, but just in case...
-  frame_index_.clear();
+  video_stream->clear_frame_index();
 
   // Iterate through every single frame and get each timestamp
   // NOTE: Expects no frames to have been read so far
@@ -941,7 +887,7 @@ void FFmpegDecoder::IndexVideo(AVPacket* pkt, AVFrame* frame)
     ret = GetFrame(pkt, frame);
 
     if (ret >= 0) {
-      frame_index_.append(frame->pts);
+      video_stream->append_frame_index(frame->pts);
     } else {
       // Assume we've reached the end of the file
       break;
@@ -949,7 +895,11 @@ void FFmpegDecoder::IndexVideo(AVPacket* pkt, AVFrame* frame)
   }
 
   // Save index to file
-  SaveIndex();
+  if (!video_stream->save_frame_index(GetIndexFilename())) {
+    qWarning() << QStringLiteral("Failed to save index for %1").arg(stream()->footage()->filename());
+  }
+
+  Seek(0);
 }
 
 int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame)
@@ -999,31 +949,65 @@ int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame)
 
 int64_t FFmpegDecoder::GetClosestTimestampInIndex(const int64_t &ts)
 {
-  // Index now if we haven't already
-  if (frame_index_.isEmpty()) {
-    Index();
-  }
+  VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());
 
-  if (frame_index_.isEmpty()) {
-    return -1;
-  }
+  bool index_is_being_created = false;
 
-  if (ts <= 0) {
-    return frame_index_.first();
-  }
+  // Check if the frame index has been populated
+  if (video_stream->is_frame_index_empty()) {
 
-  // Use index to find closest frame in file
-  for (int i=1;i<frame_index_.size();i++) {
-    int64_t this_ts = frame_index_.at(i);
+    // If not, check if one is being created right now
+    if (video_stream->index_process_lock()->tryLock()) {
 
-    if (this_ts == ts) {
-      return ts;
-    } else if (this_ts > ts) {
-      return frame_index_.at(i - 1);
+      // If not, make an index
+      ValidateVideoIndex();
+
+      video_stream->index_process_lock()->unlock();
+
+      // If the index is still empty, the video must just be empty
+      if (video_stream->is_frame_index_empty()) {
+        return -1;
+      }
+    } else {
+      // The index is being created in another thread, wait until we have more information
+      index_is_being_created = true;
     }
   }
 
-  return frame_index_.last();
+  int64_t closest_ts = -1;
+
+  do {
+    if (index_is_being_created && video_stream->index_process_lock()->tryLock()) {
+      index_is_being_created = false;
+      video_stream->index_process_lock()->unlock();
+    }
+
+    // FIXME: Wait for update from index
+    //WaitForUpdate();
+
+    closest_ts = video_stream->get_closest_timestamp_in_frame_index(ts);
+
+  } while (index_is_being_created);
+
+  return closest_ts;
+}
+
+void FFmpegDecoder::ValidateVideoIndex()
+{
+  VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());
+
+  if (video_stream->is_frame_index_empty()) {
+    video_stream->load_frame_index(GetIndexFilename());
+  }
+
+  if (video_stream->is_frame_index_empty()) {
+    // Reset state
+    Seek(0);
+
+    UnconditionalVideoIndex(pkt_, frame_);
+
+    Seek(0);
+  }
 }
 
 void FFmpegDecoder::Seek(int64_t timestamp)

app/codec/ffmpeg/ffmpegdecoder.h

@@ -32,12 +32,14 @@ extern "C" {
 #include "audio/sampleformat.h"
 #include "codec/decoder.h"
 #include "codec/waveoutput.h"
+#include "project/item/footage/videostream.h"
 
 /**
  * @brief A Decoder derivative that wraps FFmpeg functions as on Olive decoder
  */
 class FFmpegDecoder : public Decoder
 {
+  Q_OBJECT
 public:
   // Constructor
   FFmpegDecoder();
@@ -63,6 +65,8 @@ public:
 
   void SetMultithreading(bool e);
 
+  virtual void Index() override;
+
 private:
   void ConformInternal(SwrContext *resampler, WaveOutput *output, const char *in_data, int in_sample_count);
 
@@ -94,8 +98,6 @@ private:
    */
   int GetFrame(AVPacket* pkt, AVFrame* frame);
 
-  virtual void Index() override;
-
   /**
    * @brief Returns the filename for the index
    *
@@ -108,26 +110,13 @@ private:
    */
   QString GetConformedFilename(const AudioRenderingParams &params);
 
-  /**
-   * @brief Used internally to load a frame index into frame_index_
-   *
-   * @return
-   *
-   * TRUE if a frame index was successfully loaded. FALSE usually means the file didn't exist and Index() should be
-   * run to create it.
-   */
-  bool LoadIndex();
-
-  /**
-   * @brief Used in Index() to save the just created frame index to a file that can be loaded later
-   */
-  void SaveIndex();
-
-  void IndexAudio(AVPacket* pkt, AVFrame* frame);
-  void IndexVideo(AVPacket* pkt, AVFrame* frame);
+  void UnconditionalAudioIndex(AVPacket* pkt, AVFrame* frame);
+  void UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame);
 
   int64_t GetClosestTimestampInIndex(const int64_t& ts);
 
+  void ValidateVideoIndex();
+
   void Seek(int64_t timestamp);
 
   AVFormatContext* fmt_ctx_;
@@ -145,8 +134,6 @@ private:
 
   AVDictionary* opts_;
 
-  QVector<int64_t> frame_index_;
-
   bool multithreading_;
 
 };

app/project/item/footage/stream.cpp

@@ -140,6 +140,11 @@ StreamID Stream::ToID() const
   return StreamID(footage_->filename(), index_);
 }
 
+QMutex* Stream::index_process_lock()
+{
+  return &index_process_lock_;
+}
+
 void Stream::FootageSetEvent(Footage*)
 {
 }

app/project/item/footage/stream.h

@@ -103,7 +103,7 @@ public:
 
   StreamID ToID() const;
 
-  QMutex index_lock_;
+  QMutex* index_process_lock();
 
 protected:
   virtual void FootageSetEvent(Footage*);
@@ -112,6 +112,9 @@ protected:
 
   virtual void SaveCustomParameters(QXmlStreamWriter* writer) const;
 
+signals:
+  void IndexChanged();
+
 private:
   Footage* footage_;
 
@@ -125,6 +128,8 @@ private:
 
   bool enabled_;
 
+  QMutex index_process_lock_;
+
 };
 
 using StreamPtr = std::shared_ptr<Stream>;

app/project/item/footage/videostream.cpp

@@ -20,6 +20,8 @@
 
 #include "videostream.h"
 
+#include <QFile>
+
 VideoStream::VideoStream()
 {
   set_type(kVideo);
@@ -41,3 +43,92 @@ void VideoStream::set_frame_rate(const rational &frame_rate)
 {
   frame_rate_ = frame_rate;
 }
+
+int64_t VideoStream::get_closest_timestamp_in_frame_index(const int64_t &ts)
+{
+  QMutexLocker locker(&index_access_lock_);
+
+  if (ts <= 0) {
+    return frame_index_.first();
+  }
+
+  // Use index to find closest frame in file
+  for (int i=0;i<frame_index_.size();i++) {
+    int64_t this_ts = frame_index_.at(i);
+
+    if (this_ts == ts) {
+      return ts;
+    } else if (this_ts > ts) {
+      return frame_index_.at(i - 1);
+    }
+  }
+
+  return frame_index_.last();
+}
+
+void VideoStream::clear_frame_index()
+{
+  QMutexLocker locker(&index_access_lock_);
+
+  frame_index_.clear();
+}
+
+void VideoStream::append_frame_index(const int64_t &ts)
+{
+  QMutexLocker locker(&index_access_lock_);
+
+  frame_index_.append(ts);
+}
+
+bool VideoStream::is_frame_index_empty()
+{
+  QMutexLocker locker(&index_access_lock_);
+
+  return frame_index_.isEmpty();
+}
+
+int64_t VideoStream::last_frame_index_timestamp()
+{
+  QMutexLocker locker(&index_access_lock_);
+
+  return frame_index_.last();
+}
+
+bool VideoStream::load_frame_index(const QString &s)
+{
+  // Load index from file
+  QFile index_file(s);
+
+  if (index_file.exists() && index_file.open(QFile::ReadOnly)) {
+    QMutexLocker locker(&index_access_lock_);
+
+    // Resize based on filesize
+    frame_index_.resize(static_cast<size_t>(index_file.size()) / sizeof(int64_t));
+
+    // Read frame index into vector
+    index_file.read(reinterpret_cast<char*>(frame_index_.data()),
+                    index_file.size());
+
+    index_file.close();
+  }
+
+  return false;
+}
+
+bool VideoStream::save_frame_index(const QString &s)
+{
+  QFile index_file(s);
+
+  if (index_file.open(QFile::WriteOnly)) {
+    // Write index in binary
+    QMutexLocker locker(&index_access_lock_);
+
+    index_file.write(reinterpret_cast<const char*>(frame_index_.constData()),
+                     frame_index_.size() * static_cast<int>(sizeof(int64_t)));
+
+    index_file.close();
+    return true;
+  }
+
+  return false;
+}

app/project/item/footage/videostream.h

@@ -38,9 +38,22 @@ public:
   const rational& frame_rate() const;
   void set_frame_rate(const rational& frame_rate);
 
+  int64_t get_closest_timestamp_in_frame_index(const int64_t& ts);
+  void clear_frame_index();
+  void append_frame_index(const int64_t& ts);
+  bool is_frame_index_empty();
+  int64_t last_frame_index_timestamp();
+
+  bool load_frame_index(const QString& s);
+  bool save_frame_index(const QString& s);
+
 private:
   rational frame_rate_;
 
+  QVector<int64_t> frame_index_;
+
+  QMutex index_access_lock_;
+
 };
 
 using VideoStreamPtr = std::shared_ptr<VideoStream>;