decoder/renderer: no longer index automatically as part of the retrieve

functions

Indexing is a lengthy process and had a high chance of getting RenderWorkers
stuck doing it rather than being responsive to cache requests. This commit
introduces a system where workers never index media, but instead signal that
media is not ready to their RenderBackends which ensure that the media gets
indexed and re-queues the affected frames when those indexes are ready.

app/codec/decoder.cpp

@@ -26,6 +26,7 @@
 
 #include "codec/ffmpeg/ffmpegdecoder.h"
 #include "codec/oiio/oiiodecoder.h"
+#include "render/indexmanager.h"
 #include "task/index/index.h"
 #include "task/taskmanager.h"
 
@@ -57,12 +58,12 @@ void Decoder::set_stream(StreamPtr fs)
   stream_ = fs;
 }
 
-FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const QAtomicInt* cancelled)
+FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/)
 {
   return nullptr;
 }
 
-FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/, const QAtomicInt* cancelled)
+FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/)
 {
   return nullptr;
 }
@@ -134,13 +135,10 @@ bool Decoder::ProbeMedia(Footage *f, const QAtomicInt* cancelled)
 
       // Start an index task
       foreach (StreamPtr stream, f->streams()) {
-        IndexTask* index_task = new IndexTask(stream);
-        index_task->moveToThread(TaskManager::instance()->thread());
-
-        QMetaObject::invokeMethod(TaskManager::instance(),
-                                  "AddTask",
+        QMetaObject::invokeMethod(IndexManager::instance(),
+                                  "StartIndexingStream",
                                   Qt::QueuedConnection,
-                                  Q_ARG(Task*, index_task));
+                                  Q_ARG(StreamPtr, stream));
       }
 
       return true;

app/codec/decoder.h

@@ -52,6 +52,12 @@ class Decoder : public QObject
 {
   Q_OBJECT
 public:
+  enum RetrieveState {
+    kReady,
+    kFailedToOpen,
+    kIndexUnavailable
+  };
+
   Decoder();
 
   Decoder(Stream* fs);
@@ -106,6 +112,11 @@ public:
    */
   virtual bool Open() = 0;
 
+  /**
+   * @brief Determine whether the Decoder is able to retrieve data
+   */
+  virtual RetrieveState GetRetrieveState(const rational& time) = 0;
+
   /**
    * @brief Retrieve video frame
    *
@@ -127,7 +138,7 @@ public:
    * A FramePtr of valid data at this timecode or nullptr if there was nothing to retrieve at the provided timecode or
    * the media could not be opened.
    */
-  virtual FramePtr RetrieveVideo(const rational& timecode, const QAtomicInt* cancelled);
+  virtual FramePtr RetrieveVideo(const rational& timecode);
 
   /**
    * @brief Retrieve video frame
@@ -153,7 +164,7 @@ public:
    * A FramePtr of valid data at this timecode of the requested length or nullptr if there was nothing to retrieve at
    * the provided timecode or the media could not be opened.
    */
-  virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params, const QAtomicInt* cancelled);
+  virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params);
 
   virtual bool SupportsVideo();
   virtual bool SupportsAudio();
@@ -169,13 +180,6 @@ public:
    */
   virtual void Close() = 0;
 
-  /**
-   * @brief Get a media file's internal timestamp
-   *
-   * Used to determine which frame will be served at a given time, useful for caching.
-   */
-  virtual int64_t GetTimestampFromTime(const rational& time, const QAtomicInt* cancelled) = 0;
-
   /**
    * @brief Try to probe a Footage file by passing it through all available Decoders
    *
@@ -237,4 +241,6 @@ private:
   StreamPtr stream_;
 };
 
+Q_DECLARE_METATYPE(Decoder::RetrieveState)
+
 #endif // DECODER_H

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -192,7 +192,31 @@ bool FFmpegDecoder::Open()
   return true;
 }
 
-FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const QAtomicInt* cancelled)
+Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time)
+{
+  if (!open_ && !Open()) {
+    return kFailedToOpen;
+  }
+
+  if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+    // Check index
+    int64_t ts = std::static_pointer_cast<VideoStream>(stream())->get_closest_timestamp_in_frame_index(time);
+
+    if (ts < 0) {
+      return kIndexUnavailable;
+    }
+  } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+    AudioStreamPtr audio_stream = std::static_pointer_cast<AudioStream>(stream());
+
+    if (time > audio_stream->index_length() && !audio_stream->index_done()) {
+      return kIndexUnavailable;
+    }
+  }
+
+  return kReady;
+}
+
+FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 {
   if (!open_ && !Open()) {
     return nullptr;
@@ -203,7 +227,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const QAtomicInt
   }
 
   // Convert timecode to AVStream timebase
-  int64_t target_ts = GetTimestampFromTime(timecode, cancelled);
+  int64_t target_ts = std::static_pointer_cast<VideoStream>(stream())->get_closest_timestamp_in_frame_index(timecode);
 
   if (target_ts < 0) {
     Error(QStringLiteral("Index failed to produce a valid timestamp"));
@@ -318,7 +342,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const QAtomicInt
   return nullptr;
 }
 
-FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params, const QAtomicInt* cancelled)
+FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params)
 {
   if (!open_ && !Open()) {
     return nullptr;
@@ -328,13 +352,10 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &
     return nullptr;
   }
 
-  Index(cancelled);
-
-  Conform(params, cancelled);
+  //Conform(params, cancelled);
 
   WaveInput input(GetConformedFilename(params));
 
-  // FIXME: No handling if input failed to open/is corrupt
   if (input.open()) {
     const AudioRenderingParams& input_params = input.params();
 
@@ -395,24 +416,6 @@ QString FFmpegDecoder::id()
   return "ffmpeg";
 }
 
-int64_t FFmpegDecoder::GetTimestampFromTime(const rational &time, const QAtomicInt* cancelled)
-{
-  if (!open_ && !Open()) {
-    return -1;
-  }
-
-  // Get rough approximation of what the timestamp would be in this timebase
-  int64_t target_ts = Timecode::time_to_timestamp(time, avstream_->time_base);
-
-  // Adjust target by stream's start time
-  target_ts += avstream_->start_time;
-
-  // Find closest actual timebase in the file
-  target_ts = GetClosestTimestampInIndex(target_ts, cancelled);
-
-  return target_ts;
-}
-
 void FFmpegDecoder::Conform(const AudioRenderingParams &params, const QAtomicInt* cancelled)
 {
   if (avstream_->codecpar->codec_type != AVMEDIA_TYPE_AUDIO) {
@@ -583,6 +586,7 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
         video_stream->set_width(avstream_->codecpar->width);
         video_stream->set_height(avstream_->codecpar->height);
         video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx_, avstream_, nullptr));
+        video_stream->set_start_time(avstream_->start_time);
 
         str = video_stream;
 
@@ -711,9 +715,19 @@ void FFmpegDecoder::Index(const QAtomicInt* cancelled)
     ValidateVideoIndex(cancelled);
 
   } else if (stream()->type() == Stream::kAudio) {
-    if (!QFileInfo::exists(GetIndexFilename())) {
-      UnconditionalAudioIndex(pkt_, frame_, cancelled);
+
+    if (QFileInfo::exists(GetIndexFilename())) {
+      WaveInput input(GetIndexFilename());
+      if (input.open()) {
+        std::static_pointer_cast<AudioStream>(stream())->set_index_done(true);
+        std::static_pointer_cast<AudioStream>(stream())->set_index_length(input.params().bytes_to_time(input.data_length()));
+
+        input.close();
+      }
     }
+
+    UnconditionalAudioIndex(pkt_, frame_, cancelled);
+
   }
 }
 
@@ -771,6 +785,11 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
     channel_layout = static_cast<uint64_t>(av_get_default_channel_layout(avstream_->codecpar->channels));
   }
 
+  AudioStreamPtr audio_stream = std::static_pointer_cast<AudioStream>(stream());
+
+  // This should be unnecessary, but just in case...
+  audio_stream->clear_index();
+
   SwrContext* resampler = nullptr;
   AVSampleFormat src_sample_fmt = static_cast<AVSampleFormat>(avstream_->codecpar->format);
   AVSampleFormat dst_sample_fmt;
@@ -792,10 +811,10 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
     dst_sample_fmt = src_sample_fmt;
   }
 
-  WaveOutput wave_out(GetIndexFilename(),
-                      AudioRenderingParams(avstream_->codecpar->sample_rate,
-                                           channel_layout,
-                                           FFmpegCommon::GetNativeSampleFormat(dst_sample_fmt)));
+  AudioRenderingParams wave_params(avstream_->codecpar->sample_rate,
+                                   channel_layout,
+                                   FFmpegCommon::GetNativeSampleFormat(dst_sample_fmt));
+  WaveOutput wave_out(GetIndexFilename(), wave_params);
 
   int ret;
 
@@ -843,6 +862,8 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
         // Write packed WAV data to the disk cache
         wave_out.write(reinterpret_cast<char*>(data_frame->data[0]), buffer_sz);
 
+        audio_stream->set_index_length(wave_params.bytes_to_time(wave_out.data_length()));
+
         // If we allocated an output for the resampler, delete it here
         if (data_frame != frame) {
           av_frame_free(&data_frame);
@@ -855,6 +876,9 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
     if (cancelled && *cancelled) {
       // Audio index didn't complete, delete it
       QFile(GetIndexFilename()).remove();
+      audio_stream->clear_index();
+    } else {
+      audio_stream->set_index_done(true);
     }
   } else {
     qWarning() << "Failed to open WAVE output for indexing";
@@ -959,49 +983,6 @@ int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame)
   return ret;
 }
 
-int64_t FFmpegDecoder::GetClosestTimestampInIndex(const int64_t &ts, const QAtomicInt* cancelled)
-{
-  VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());
-
-  bool index_is_being_created = false;
-
-  // Check if an index is being created right now
-  if (video_stream->index_process_lock()->tryLock()) {
-
-    // If not, check if the frame index has been populated
-    if (!video_stream->is_frame_index_ready()) {
-      // If not, make a frame index
-      ValidateVideoIndex(cancelled);
-    }
-
-    video_stream->index_process_lock()->unlock();
-
-  } else {
-    index_is_being_created = true;
-  }
-
-  int64_t closest_ts = -1;
-
-  do {
-    if (cancelled && *cancelled) {
-      return -1;
-    }
-
-    if (index_is_being_created && video_stream->index_process_lock()->tryLock()) {
-      index_is_being_created = false;
-      video_stream->index_process_lock()->unlock();
-    }
-
-    // FIXME: Wait for update from index
-    //WaitForUpdate();
-
-    closest_ts = video_stream->get_closest_timestamp_in_frame_index(ts);
-
-  } while (closest_ts < 0 && index_is_being_created);
-
-  return closest_ts;
-}
-
 void FFmpegDecoder::ValidateVideoIndex(const QAtomicInt* cancelled)
 {
   VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());

app/codec/ffmpeg/ffmpegdecoder.h

@@ -50,14 +50,13 @@ public:
   virtual bool Probe(Footage *f, const QAtomicInt *cancelled) override;
 
   virtual bool Open() override;
-  virtual FramePtr RetrieveVideo(const rational &timecode, const QAtomicInt *cancelled) override;
-  virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params, const QAtomicInt *cancelled) override;
+  virtual RetrieveState GetRetrieveState(const rational &time) override;
+  virtual FramePtr RetrieveVideo(const rational &timecode) override;
+  virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override;
   virtual void Close() override;
 
   virtual QString id() override;
 
-  virtual int64_t GetTimestampFromTime(const rational& time, const QAtomicInt *cancelled) override;
-
   virtual void Conform(const AudioRenderingParams& params, const QAtomicInt *cancelled) override;
 
   virtual bool SupportsVideo() override;
@@ -113,8 +112,6 @@ private:
   void UnconditionalAudioIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled);
   void UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled);
 
-  int64_t GetClosestTimestampInIndex(const int64_t& ts, const QAtomicInt *cancelled);
-
   void ValidateVideoIndex(const QAtomicInt* cancelled);
 
   void Seek(int64_t timestamp);

app/codec/oiio/oiiodecoder.cpp

@@ -133,7 +133,16 @@ bool OIIODecoder::Open()
   return true;
 }
 
-FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const QAtomicInt *cancelled)
+Decoder::RetrieveState OIIODecoder::GetRetrieveState(const rational &time)
+{
+  if (!open_ && !Open()) {
+    return kFailedToOpen;
+  }
+
+  return kReady;
+}
+
+FramePtr OIIODecoder::RetrieveVideo(const rational &timecode)
 {
   if (!open_ && !Open()) {
     return nullptr;
@@ -174,15 +183,6 @@ void OIIODecoder::Close()
   frame_ = nullptr;
 }
 
-int64_t OIIODecoder::GetTimestampFromTime(const rational &time, const QAtomicInt *cancelled)
-{
-  Q_UNUSED(time)
-
-  // A still image will always return the same frame
-
-  return 0;
-}
-
 bool OIIODecoder::SupportsVideo()
 {
   return true;

app/codec/oiio/oiiodecoder.h

@@ -37,12 +37,11 @@ public:
 
   virtual bool Open() override;
 
-  virtual FramePtr RetrieveVideo(const rational &timecode, const QAtomicInt* cancelled) override;
+  virtual RetrieveState GetRetrieveState(const rational &time) override;
+  virtual FramePtr RetrieveVideo(const rational &timecode) override;
 
   virtual void Close() override;
 
-  virtual int64_t GetTimestampFromTime(const rational &time, const QAtomicInt* cancelled) override;
-
   virtual bool SupportsVideo() override;
 
 private: