decoder/renderer: no longer index automatically as part of the retrieve

functions

Indexing is a lengthy process and had a high chance of getting RenderWorkers
stuck doing it rather than being responsive to cache requests. This commit
introduces a system where workers never index media, but instead signal that
media is not ready to their RenderBackends which ensure that the media gets
indexed and re-queues the affected frames when those indexes are ready.

app/codec/decoder.cpp

@@ -26,6 +26,7 @@
 
 #include "codec/ffmpeg/ffmpegdecoder.h"
 #include "codec/oiio/oiiodecoder.h"
+#include "render/indexmanager.h"
 #include "task/index/index.h"
 #include "task/taskmanager.h"
 
@@ -57,12 +58,12 @@ void Decoder::set_stream(StreamPtr fs)
   stream_ = fs;
 }
 
-FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const QAtomicInt* cancelled)
+FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/)
 {
   return nullptr;
 }
 
-FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/, const QAtomicInt* cancelled)
+FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/)
 {
   return nullptr;
 }
@@ -134,13 +135,10 @@ bool Decoder::ProbeMedia(Footage *f, const QAtomicInt* cancelled)
 
       // Start an index task
       foreach (StreamPtr stream, f->streams()) {
-        IndexTask* index_task = new IndexTask(stream);
-        index_task->moveToThread(TaskManager::instance()->thread());
-
-        QMetaObject::invokeMethod(TaskManager::instance(),
-                                  "AddTask",
+        QMetaObject::invokeMethod(IndexManager::instance(),
+                                  "StartIndexingStream",
                                   Qt::QueuedConnection,
-                                  Q_ARG(Task*, index_task));
+                                  Q_ARG(StreamPtr, stream));
       }
 
       return true;

app/codec/decoder.h

@@ -52,6 +52,12 @@ class Decoder : public QObject
 {
   Q_OBJECT
 public:
+  enum RetrieveState {
+    kReady,
+    kFailedToOpen,
+    kIndexUnavailable
+  };
+
   Decoder();
 
   Decoder(Stream* fs);
@@ -106,6 +112,11 @@ public:
    */
   virtual bool Open() = 0;
 
+  /**
+   * @brief Determine whether the Decoder is able to retrieve data
+   */
+  virtual RetrieveState GetRetrieveState(const rational& time) = 0;
+
   /**
    * @brief Retrieve video frame
    *
@@ -127,7 +138,7 @@ public:
    * A FramePtr of valid data at this timecode or nullptr if there was nothing to retrieve at the provided timecode or
    * the media could not be opened.
    */
-  virtual FramePtr RetrieveVideo(const rational& timecode, const QAtomicInt* cancelled);
+  virtual FramePtr RetrieveVideo(const rational& timecode);
 
   /**
    * @brief Retrieve video frame
@@ -153,7 +164,7 @@ public:
    * A FramePtr of valid data at this timecode of the requested length or nullptr if there was nothing to retrieve at
    * the provided timecode or the media could not be opened.
    */
-  virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params, const QAtomicInt* cancelled);
+  virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params);
 
   virtual bool SupportsVideo();
   virtual bool SupportsAudio();
@@ -169,13 +180,6 @@ public:
    */
   virtual void Close() = 0;
 
-  /**
-   * @brief Get a media file's internal timestamp
-   *
-   * Used to determine which frame will be served at a given time, useful for caching.
-   */
-  virtual int64_t GetTimestampFromTime(const rational& time, const QAtomicInt* cancelled) = 0;
-
   /**
    * @brief Try to probe a Footage file by passing it through all available Decoders
    *
@@ -237,4 +241,6 @@ private:
   StreamPtr stream_;
 };
 
+Q_DECLARE_METATYPE(Decoder::RetrieveState)
+
 #endif // DECODER_H

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -192,7 +192,31 @@ bool FFmpegDecoder::Open()
   return true;
 }
 
-FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const QAtomicInt* cancelled)
+Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time)
+{
+  if (!open_ && !Open()) {
+    return kFailedToOpen;
+  }
+
+  if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+    // Check index
+    int64_t ts = std::static_pointer_cast<VideoStream>(stream())->get_closest_timestamp_in_frame_index(time);
+
+    if (ts < 0) {
+      return kIndexUnavailable;
+    }
+  } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+    AudioStreamPtr audio_stream = std::static_pointer_cast<AudioStream>(stream());
+
+    if (time > audio_stream->index_length() && !audio_stream->index_done()) {
+      return kIndexUnavailable;
+    }
+  }
+
+  return kReady;
+}
+
+FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode)
 {
   if (!open_ && !Open()) {
     return nullptr;
@@ -203,7 +227,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const QAtomicInt
   }
 
   // Convert timecode to AVStream timebase
-  int64_t target_ts = GetTimestampFromTime(timecode, cancelled);
+  int64_t target_ts = std::static_pointer_cast<VideoStream>(stream())->get_closest_timestamp_in_frame_index(timecode);
 
   if (target_ts < 0) {
     Error(QStringLiteral("Index failed to produce a valid timestamp"));
@@ -318,7 +342,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const QAtomicInt
   return nullptr;
 }
 
-FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params, const QAtomicInt* cancelled)
+FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params)
 {
   if (!open_ && !Open()) {
     return nullptr;
@@ -328,13 +352,10 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &
     return nullptr;
   }
 
-  Index(cancelled);
-
-  Conform(params, cancelled);
+  //Conform(params, cancelled);
 
   WaveInput input(GetConformedFilename(params));
 
-  // FIXME: No handling if input failed to open/is corrupt
   if (input.open()) {
     const AudioRenderingParams& input_params = input.params();
 
@@ -395,24 +416,6 @@ QString FFmpegDecoder::id()
   return "ffmpeg";
 }
 
-int64_t FFmpegDecoder::GetTimestampFromTime(const rational &time, const QAtomicInt* cancelled)
-{
-  if (!open_ && !Open()) {
-    return -1;
-  }
-
-  // Get rough approximation of what the timestamp would be in this timebase
-  int64_t target_ts = Timecode::time_to_timestamp(time, avstream_->time_base);
-
-  // Adjust target by stream's start time
-  target_ts += avstream_->start_time;
-
-  // Find closest actual timebase in the file
-  target_ts = GetClosestTimestampInIndex(target_ts, cancelled);
-
-  return target_ts;
-}
-
 void FFmpegDecoder::Conform(const AudioRenderingParams &params, const QAtomicInt* cancelled)
 {
   if (avstream_->codecpar->codec_type != AVMEDIA_TYPE_AUDIO) {
@@ -583,6 +586,7 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
         video_stream->set_width(avstream_->codecpar->width);
         video_stream->set_height(avstream_->codecpar->height);
         video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx_, avstream_, nullptr));
+        video_stream->set_start_time(avstream_->start_time);
 
         str = video_stream;
 
@@ -711,9 +715,19 @@ void FFmpegDecoder::Index(const QAtomicInt* cancelled)
     ValidateVideoIndex(cancelled);
 
   } else if (stream()->type() == Stream::kAudio) {
-    if (!QFileInfo::exists(GetIndexFilename())) {
-      UnconditionalAudioIndex(pkt_, frame_, cancelled);
+
+    if (QFileInfo::exists(GetIndexFilename())) {
+      WaveInput input(GetIndexFilename());
+      if (input.open()) {
+        std::static_pointer_cast<AudioStream>(stream())->set_index_done(true);
+        std::static_pointer_cast<AudioStream>(stream())->set_index_length(input.params().bytes_to_time(input.data_length()));
+
+        input.close();
+      }
     }
+
+    UnconditionalAudioIndex(pkt_, frame_, cancelled);
+
   }
 }
 
@@ -771,6 +785,11 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
     channel_layout = static_cast<uint64_t>(av_get_default_channel_layout(avstream_->codecpar->channels));
   }
 
+  AudioStreamPtr audio_stream = std::static_pointer_cast<AudioStream>(stream());
+
+  // This should be unnecessary, but just in case...
+  audio_stream->clear_index();
+
   SwrContext* resampler = nullptr;
   AVSampleFormat src_sample_fmt = static_cast<AVSampleFormat>(avstream_->codecpar->format);
   AVSampleFormat dst_sample_fmt;
@@ -792,10 +811,10 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
     dst_sample_fmt = src_sample_fmt;
   }
 
-  WaveOutput wave_out(GetIndexFilename(),
-                      AudioRenderingParams(avstream_->codecpar->sample_rate,
-                                           channel_layout,
-                                           FFmpegCommon::GetNativeSampleFormat(dst_sample_fmt)));
+  AudioRenderingParams wave_params(avstream_->codecpar->sample_rate,
+                                   channel_layout,
+                                   FFmpegCommon::GetNativeSampleFormat(dst_sample_fmt));
+  WaveOutput wave_out(GetIndexFilename(), wave_params);
 
   int ret;
 
@@ -843,6 +862,8 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
         // Write packed WAV data to the disk cache
         wave_out.write(reinterpret_cast<char*>(data_frame->data[0]), buffer_sz);
 
+        audio_stream->set_index_length(wave_params.bytes_to_time(wave_out.data_length()));
+
         // If we allocated an output for the resampler, delete it here
         if (data_frame != frame) {
           av_frame_free(&data_frame);
@@ -855,6 +876,9 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const
     if (cancelled && *cancelled) {
       // Audio index didn't complete, delete it
       QFile(GetIndexFilename()).remove();
+      audio_stream->clear_index();
+    } else {
+      audio_stream->set_index_done(true);
     }
   } else {
     qWarning() << "Failed to open WAVE output for indexing";
@@ -959,49 +983,6 @@ int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame)
   return ret;
 }
 
-int64_t FFmpegDecoder::GetClosestTimestampInIndex(const int64_t &ts, const QAtomicInt* cancelled)
-{
-  VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());
-
-  bool index_is_being_created = false;
-
-  // Check if an index is being created right now
-  if (video_stream->index_process_lock()->tryLock()) {
-
-    // If not, check if the frame index has been populated
-    if (!video_stream->is_frame_index_ready()) {
-      // If not, make a frame index
-      ValidateVideoIndex(cancelled);
-    }
-
-    video_stream->index_process_lock()->unlock();
-
-  } else {
-    index_is_being_created = true;
-  }
-
-  int64_t closest_ts = -1;
-
-  do {
-    if (cancelled && *cancelled) {
-      return -1;
-    }
-
-    if (index_is_being_created && video_stream->index_process_lock()->tryLock()) {
-      index_is_being_created = false;
-      video_stream->index_process_lock()->unlock();
-    }
-
-    // FIXME: Wait for update from index
-    //WaitForUpdate();
-
-    closest_ts = video_stream->get_closest_timestamp_in_frame_index(ts);
-
-  } while (closest_ts < 0 && index_is_being_created);
-
-  return closest_ts;
-}
-
 void FFmpegDecoder::ValidateVideoIndex(const QAtomicInt* cancelled)
 {
   VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream());

app/codec/ffmpeg/ffmpegdecoder.h

@@ -50,14 +50,13 @@ public:
   virtual bool Probe(Footage *f, const QAtomicInt *cancelled) override;
 
   virtual bool Open() override;
-  virtual FramePtr RetrieveVideo(const rational &timecode, const QAtomicInt *cancelled) override;
-  virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params, const QAtomicInt *cancelled) override;
+  virtual RetrieveState GetRetrieveState(const rational &time) override;
+  virtual FramePtr RetrieveVideo(const rational &timecode) override;
+  virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override;
   virtual void Close() override;
 
   virtual QString id() override;
 
-  virtual int64_t GetTimestampFromTime(const rational& time, const QAtomicInt *cancelled) override;
-
   virtual void Conform(const AudioRenderingParams& params, const QAtomicInt *cancelled) override;
 
   virtual bool SupportsVideo() override;
@@ -113,8 +112,6 @@ private:
   void UnconditionalAudioIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled);
   void UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled);
 
-  int64_t GetClosestTimestampInIndex(const int64_t& ts, const QAtomicInt *cancelled);
-
   void ValidateVideoIndex(const QAtomicInt* cancelled);
 
   void Seek(int64_t timestamp);

app/codec/oiio/oiiodecoder.cpp

@@ -133,7 +133,16 @@ bool OIIODecoder::Open()
   return true;
 }
 
-FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const QAtomicInt *cancelled)
+Decoder::RetrieveState OIIODecoder::GetRetrieveState(const rational &time)
+{
+  if (!open_ && !Open()) {
+    return kFailedToOpen;
+  }
+
+  return kReady;
+}
+
+FramePtr OIIODecoder::RetrieveVideo(const rational &timecode)
 {
   if (!open_ && !Open()) {
     return nullptr;
@@ -174,15 +183,6 @@ void OIIODecoder::Close()
   frame_ = nullptr;
 }
 
-int64_t OIIODecoder::GetTimestampFromTime(const rational &time, const QAtomicInt *cancelled)
-{
-  Q_UNUSED(time)
-
-  // A still image will always return the same frame
-
-  return 0;
-}
-
 bool OIIODecoder::SupportsVideo()
 {
   return true;

app/codec/oiio/oiiodecoder.h

@@ -37,12 +37,11 @@ public:
 
   virtual bool Open() override;
 
-  virtual FramePtr RetrieveVideo(const rational &timecode, const QAtomicInt* cancelled) override;
+  virtual RetrieveState GetRetrieveState(const rational &time) override;
+  virtual FramePtr RetrieveVideo(const rational &timecode) override;
 
   virtual void Close() override;
 
-  virtual int64_t GetTimestampFromTime(const rational &time, const QAtomicInt* cancelled) override;
-
   virtual bool SupportsVideo() override;
 
 private:

app/common/timerange.h

@@ -33,6 +33,8 @@ private:
 
 };
 
+Q_DECLARE_METATYPE(TimeRange)
+
 uint qHash(const TimeRange& r, uint seed);
 
 class TimeRangeList : public QList<TimeRange> {

app/core.cpp

@@ -49,6 +49,7 @@
 #include "render/backend/opengl/opengltexturecache.h"
 #include "render/colormanager.h"
 #include "render/diskmanager.h"
+#include "render/indexmanager.h"
 #include "render/pixelservice.h"
 #include "task/taskmanager.h"
 #include "ui/style/style.h"
@@ -109,6 +110,9 @@ void Core::Start()
   // Set up node factory/library
   NodeFactory::Initialize();
 
+  // Set up the index manager for renderers
+  IndexManager::CreateInstance();
+
   // Load application config
   Config::Load();
 
@@ -147,6 +151,8 @@ void Core::Stop()
 
   NodeFactory::Destroy();
 
+  IndexManager::DestroyInstance();
+
   delete main_window_;
 }
 
@@ -370,6 +376,8 @@ void Core::DeclareTypesForQt()
   qRegisterMetaType<FramePtr>();
   qRegisterMetaType<AudioRenderingParams>();
   qRegisterMetaType<NodeKeyframe::Type>();
+  qRegisterMetaType<Decoder::RetrieveState>();
+  qRegisterMetaType<TimeRange>();
 }
 
 void Core::StartGUI(bool full_screen)

app/dialog/export/export.cpp

@@ -177,7 +177,7 @@ ExportDialog::ExportDialog(ViewerOutput *viewer_node, QWidget *parent) :
   // Update renderer
   // FIXME: This is going to be VERY slow since it will need to hash every single frame. It would be better to have a
   //        the renderer save the map as some sort of file that this can load.
-  preview_viewer_->video_renderer()->InvalidateCache(0, viewer_node_->Length());
+  preview_viewer_->video_renderer()->InvalidateCache(TimeRange(0, viewer_node_->Length()));
 }
 
 void ExportDialog::accept()

app/node/output/viewer/viewer.cpp

@@ -101,9 +101,9 @@ void ViewerOutput::InvalidateCache(const rational &start_range, const rational &
   Node::InvalidateCache(start_range, end_range, from);
 
   if (from == texture_input()) {
-    emit VideoChangedBetween(start_range, end_range);
+    emit VideoChangedBetween(TimeRange(start_range, end_range));
   } else if (from == samples_input()) {
-    emit AudioChangedBetween(start_range, end_range);
+    emit AudioChangedBetween(TimeRange(start_range, end_range));
   } else if (from == length_input()) {
     emit LengthChanged(Length());
   }

app/node/output/viewer/viewer.h

@@ -84,9 +84,9 @@ protected:
 signals:
   void TimebaseChanged(const rational&);
 
-  void VideoChangedBetween(const rational&, const rational&);
+  void VideoChangedBetween(const TimeRange& range);
 
-  void AudioChangedBetween(const rational&, const rational&);
+  void AudioChangedBetween(const TimeRange& range);
 
   void VisibleInvalidated();
 

app/render/CMakeLists.txt

@@ -26,6 +26,8 @@ set(OLIVE_SOURCES
   render/colorprocessor.cpp
   render/diskmanager.h
   render/diskmanager.cpp
+  render/indexmanager.h
+  render/indexmanager.cpp
   render/pixelformat.h
   render/pixelformat.cpp
   render/pixelservice.h

app/render/audioparams.cpp

@@ -83,6 +83,11 @@ int AudioRenderingParams::samples_to_bytes(const int &samples) const
   return samples * channel_count() * bytes_per_sample_per_channel();
 }
 
+rational AudioRenderingParams::samples_to_time(const int &samples) const
+{
+  return rational(samples, sample_rate());
+}
+
 int AudioRenderingParams::bytes_to_samples(const int &bytes) const
 {
   Q_ASSERT(is_valid());
@@ -90,6 +95,13 @@ int AudioRenderingParams::bytes_to_samples(const int &bytes) const
   return bytes / (channel_count() * bytes_per_sample_per_channel());
 }
 
+rational AudioRenderingParams::bytes_to_time(const int &bytes) const
+{
+  Q_ASSERT(is_valid());
+
+  return samples_to_time(bytes_to_samples(bytes));
+}
+
 int AudioRenderingParams::channel_count() const
 {
   return av_get_channel_layout_nb_channels(channel_layout());

app/render/audioparams.h

@@ -31,7 +31,9 @@ public:
   int time_to_bytes(const rational& time) const;
   int time_to_samples(const rational& time) const;
   int samples_to_bytes(const int& samples) const;
+  rational samples_to_time(const int& samples) const;
   int bytes_to_samples(const int &bytes) const;
+  rational bytes_to_time(const int &bytes) const;
   int channel_count() const;
   int bytes_per_sample_per_channel() const;
   int bits_per_sample() const;

app/render/backend/audio/audioworker.cpp

@@ -51,8 +51,7 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c
 
         // If the input isn't keyframing, we don't need to update it unless it's connected, in which case it may change
         if (input->IsConnected() || input->is_keyframing()) {
-          input_params.Insert(input, ProcessInput(input,
-                                                  TimeRange(this_sample_time, this_sample_time)));
+          input_params.Insert(input, ProcessInput(input, TimeRange(this_sample_time, this_sample_time)));
         }
       }
     }

app/render/backend/audiorenderworker.cpp

@@ -23,9 +23,9 @@ void AudioRenderWorker::CloseInternal()
   // Nothing to init yet
 }
 
-FramePtr AudioRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range, const QAtomicInt *cancelled)
+FramePtr AudioRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range)
 {
-  return decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params_, cancelled);
+  return decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params_);
 }
 
 NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const TimeRange &range)
@@ -42,12 +42,12 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
     TimeRange range_for_block(qMax(b->in(), range.in()),
                               qMin(b->out(), range.out()));
 
-    NodeValueTable table = ProcessNode(NodeDependency(b,
-                                                      range_for_block));
+    NodeValueTable table = ProcessNode(NodeDependency(b, range_for_block));
 
     QByteArray samples_from_this_block = table.Take(NodeParam::kSamples).toByteArray();
     int destination_offset = audio_params_.time_to_bytes(range_for_block.in() - range.in());
     int maximum_copy_size = audio_params_.time_to_bytes(range_for_block.length());
+
     int copied_size = 0;
 
     if (!samples_from_this_block.isEmpty()) {
@@ -81,13 +81,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
 
       memcpy(block_range_buffer.data()+destination_offset,
              samples_from_this_block.data(),
-             static_cast<size_t>(copied_size));
-    }
-
-    if (copied_size < maximum_copy_size) {
-      memset(block_range_buffer.data()+destination_offset+copied_size,
-             0,
-             static_cast<size_t>(maximum_copy_size - copied_size));
+             qMax(maximum_copy_size, copied_size));
     }
 
     NodeValueTable::Merge({merged_table, table});

app/render/backend/audiorenderworker.h

@@ -16,7 +16,7 @@ protected:
 
   virtual void CloseInternal() override;
 
-  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range, const QAtomicInt* cancelled) override;
+  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) override;
 
   virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) override;
 

app/render/backend/exporter.cpp

@@ -216,14 +216,14 @@ void Exporter::EncoderOpenedSuccessfully()
     video_backend_->SetOperatingMode(VideoRenderWorker::kHashOnly);
     connect(video_backend_, &VideoRenderBackend::QueueComplete, this, &Exporter::VideoHashesComplete);
 
-    video_backend_->InvalidateCache(0, viewer_node_->Length());
+    video_backend_->InvalidateCache(TimeRange(0, viewer_node_->Length()));
   }
 
   if (!audio_done_) {
     // We set the audio backend to render the full sequence to the disk
     connect(audio_backend_, &AudioRenderBackend::QueueComplete, this, &Exporter::AudioRendered);
 
-    audio_backend_->InvalidateCache(0, viewer_node_->Length());
+    audio_backend_->InvalidateCache(TimeRange(0, viewer_node_->Length()));
   }
 }
 
@@ -282,6 +282,6 @@ void Exporter::VideoHashesComplete()
   //connect(video_backend_, &VideoRenderBackend::CachedFrameReady, this, &Exporter::FrameRendered);
 
   foreach (const TimeRange& range, ranges) {
-    video_backend_->InvalidateCache(range.in(), range.out());
+    video_backend_->InvalidateCache(range);
   }
 }

app/render/backend/renderbackend.cpp

@@ -4,6 +4,7 @@
 #include <QThread>
 
 #include "core.h"
+#include "render/indexmanager.h"
 #include "window/mainwindow/mainwindow.h"
 
 RenderBackend::RenderBackend(QObject *parent) :
@@ -17,6 +18,8 @@ RenderBackend::RenderBackend(QObject *parent) :
 {
   // FIXME: Don't create in CLI mode
   cancel_dialog_ = new RenderCancelDialog(Core::instance()->main_window());
+
+  connect(IndexManager::instance(), &IndexManager::StreamIndexUpdated, this, &RenderBackend::IndexUpdated);
 }
 
 bool RenderBackend::Init()
@@ -111,27 +114,6 @@ bool RenderBackend::IsInitiated()
   return started_;
 }
 
-void RenderBackend::InvalidateCache(const rational &start_range, const rational &end_range)
-{
-  if (!CanRender()) {
-    return;
-  }
-
-  // Adjust range to min/max values
-  rational start_range_adj = qMax(rational(0), start_range);
-  rational end_range_adj = qMin(GetSequenceLength(), end_range);
-
-  qDebug() << "Cache invalidated between"
-           << start_range_adj.toDouble()
-           << "and"
-           << end_range_adj.toDouble();
-
-  // Queue value update
-  QueueValueUpdate();
-
-  InvalidateCacheInternal(start_range_adj, end_range_adj);
-}
-
 bool RenderBackend::Compile()
 {
   if (compiled_) {
@@ -347,6 +329,27 @@ void RenderBackend::CancelQueue()
   cancel_dialog_->RunIfWorkersAreBusy();
 }
 
+void RenderBackend::InvalidateCache(const TimeRange &range)
+{
+  if (!CanRender()) {
+    return;
+  }
+
+  // Adjust range to min/max values
+  rational start_range_adj = qMax(rational(0), range.in());
+  rational end_range_adj = qMin(GetSequenceLength(), range.out());
+
+  qDebug() << "Cache invalidated between"
+           << start_range_adj.toDouble()
+           << "and"
+           << end_range_adj.toDouble();
+
+  // Queue value update
+  QueueValueUpdate();
+
+  InvalidateCacheInternal(start_range_adj, end_range_adj);
+}
+
 bool RenderBackend::ViewerIsConnected() const
 {
   return viewer_node_ != nullptr;
@@ -412,6 +415,7 @@ void RenderBackend::InitWorkers()
 
     // Connect cancel dialog to it
     connect(processor, &RenderWorker::CompletedCache, cancel_dialog_, &RenderCancelDialog::WorkerDone, Qt::QueuedConnection);
+    connect(processor, &RenderWorker::FootageUnavailable, this, &RenderBackend::FootageUnavailable, Qt::QueuedConnection);
 
     // Finally, we can move it to its own thread
     processor->moveToThread(thread);
@@ -428,3 +432,84 @@ void RenderBackend::QueueRecompile()
 {
   recompile_queued_ = true;
 }
+
+void RenderBackend::FootageUnavailable(StreamPtr stream, Decoder::RetrieveState state, const TimeRange &range, const rational &stream_time)
+{
+  if (state == Decoder::kFailedToOpen){
+
+    qWarning() << "For range" << range.in() << "-" << range.out() << stream->footage()->filename() << "stream" << stream->index() << "failed to open";
+
+  } else if (state == Decoder::kIndexUnavailable) {
+
+    FootageWaitInfo info = {stream, range, stream_time};
+
+    foreach (const FootageWaitInfo& compare, footage_wait_info_) {
+      if (info.stream == compare.stream
+          && info.stream_time == compare.stream_time
+          && info.affected_range == compare.affected_range) {
+        return;
+      }
+    }
+
+    qDebug() << "Waiting for" << stream.get() << "time" << stream_time.toDouble() << "for frame" << range.in();
+
+    if (IndexManager::instance()->IsIndexing(stream)) {
+
+      footage_wait_info_.append(info);
+
+    } else if ((stream->type() == Stream::kVideo && std::static_pointer_cast<VideoStream>(stream)->is_frame_index_ready())
+               || (stream->type() == Stream::kAudio && std::static_pointer_cast<AudioStream>(stream)->index_done())) {
+
+      // Index JUST finished, requeue this time
+      InvalidateCache(range);
+
+    } else {
+
+      // Start indexing process
+      footage_wait_info_.append(info);
+      IndexManager::instance()->StartIndexingStream(stream);
+
+    }
+
+  }
+}
+
+void RenderBackend::IndexUpdated(Stream* stream)
+{
+  for (int i=0;i<footage_wait_info_.size();i++) {
+    const FootageWaitInfo& info = footage_wait_info_.at(i);
+
+    if (info.stream.get() == stream) {
+      bool footage_ready = false;
+
+      // See if this stream now has an index for the requested time
+      if (stream->type() == Stream::kVideo) {
+
+        VideoStream* video_stream = static_cast<VideoStream*>(stream);
+
+        if (video_stream->get_closest_timestamp_in_frame_index(info.stream_time) >= 0) {
+          // This index now has this frame, we can re-render it
+          qDebug() << "Re-ICing video" << info.affected_range.in().toDouble() << "to" << info.affected_range.out().toDouble();
+          footage_ready = true;
+        }
+
+      } else if (stream->type() == Stream::kAudio) {
+
+        AudioStream* audio_stream = static_cast<AudioStream*>(stream);
+
+        if (audio_stream->index_length() >= info.stream_time) {
+          // The index now has this audio, we can re-render it
+          qDebug() << "Re-ICing audio" << info.affected_range.in().toDouble() << "to" << info.affected_range.out().toDouble();
+          footage_ready = true;
+        }
+
+      }
+
+      if (footage_ready) {
+        InvalidateCache(info.affected_range);
+        footage_wait_info_.removeAt(i);
+        i--;
+      }
+    }
+  }
+}

app/render/backend/renderbackend.h

@@ -31,7 +31,7 @@ public:
   void CancelQueue();
 
 public slots:
-  void InvalidateCache(const rational &start_range, const rational &end_range);
+  void InvalidateCache(const TimeRange &range);
 
   bool Compile();
 
@@ -145,6 +145,19 @@ private:
 
   RenderCancelDialog* cancel_dialog_;
 
+  struct FootageWaitInfo {
+    StreamPtr stream;
+    TimeRange affected_range;
+    rational stream_time;
+  };
+
+  QList<FootageWaitInfo> footage_wait_info_;
+
+private slots:
+  void FootageUnavailable(StreamPtr stream, Decoder::RetrieveState state, const TimeRange& path, const rational& stream_time);
+
+  void IndexUpdated(Stream *stream);
+
 };
 
 #endif // RENDERBACKEND_H

app/render/backend/renderworker.cpp

@@ -34,6 +34,8 @@ void RenderWorker::Close()
 
 void RenderWorker::Render(NodeDependency path, qint64 job_time)
 {
+  path_ = path;
+
   emit CompletedCache(path, RenderInternal(path, job_time), job_time);
 }
 
@@ -104,8 +106,7 @@ NodeValueTable RenderWorker::ProcessInput(const NodeInput *input, const TimeRang
 {
   if (input->IsConnected()) {
     // Value will equal something from the connected node, follow it
-    return ProcessNode(NodeDependency(input->get_connected_node(),
-                                      range));
+    return ProcessNode(NodeDependency(input->get_connected_node(), range));
   } else {
     // Push onto the table the value at this time from the input
     QVariant input_value = input->get_value_at_time(range.in());
@@ -116,6 +117,16 @@ NodeValueTable RenderWorker::ProcessInput(const NodeInput *input, const TimeRang
   }
 }
 
+void RenderWorker::ReportUnavailableFootage(StreamPtr stream, Decoder::RetrieveState state, const rational& stream_time)
+{
+  emit FootageUnavailable(stream, state, path_.range(), stream_time);
+}
+
+const NodeDependency &RenderWorker::CurrentPath() const
+{
+  return path_;
+}
+
 NodeValueDatabase RenderWorker::GenerateDatabase(const Node* node, const TimeRange &range)
 {
   NodeValueDatabase database;
@@ -140,10 +151,17 @@ NodeValueDatabase RenderWorker::GenerateDatabase(const Node* node, const TimeRan
           DecoderPtr decoder = ResolveDecoderFromInput(stream);
 
           if (decoder) {
-            FramePtr frame = RetrieveFromDecoder(decoder, input_time, &IsCancelled());
 
-            if (frame) {
-              FrameToValue(stream, frame, &table);
+            Decoder::RetrieveState state = decoder->GetRetrieveState(input_time.out());
+
+            if (state == Decoder::kReady) {
+              FramePtr frame = RetrieveFromDecoder(decoder, input_time);
+
+              if (frame) {
+                FrameToValue(stream, frame, &table);
+              }
+            } else {
+              ReportUnavailableFootage(stream, state, input_time.out());
             }
           }
         }

app/render/backend/renderworker.h

@@ -24,24 +24,26 @@ public:
 public slots:
   void Close();
 
-  void Render(NodeDependency path, qint64 job_time);
+  void Render(NodeDependency CurrentPath, qint64 job_time);
 
 signals:
   void CompletedCache(NodeDependency dep, NodeValueTable data, qint64 job_time);
 
+  void FootageUnavailable(StreamPtr stream, Decoder::RetrieveState state, const TimeRange& range, const rational& stream_time);
+
 protected:
   virtual bool InitInternal() = 0;
 
   virtual void CloseInternal() = 0;
 
-  virtual NodeValueTable RenderInternal(const NodeDependency& path, const qint64& job_time);
+  virtual NodeValueTable RenderInternal(const NodeDependency& CurrentPath, const qint64& job_time);
 
   virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, const NodeValueDatabase &input_params, NodeValueTable* output_params);
 
   StreamPtr ResolveStreamFromInput(NodeInput* input);
   DecoderPtr ResolveDecoderFromInput(StreamPtr stream);
 
-  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range, const QAtomicInt* cancelled) = 0;
+  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) = 0;
 
   virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) = 0;
 
@@ -51,6 +53,10 @@ protected:
 
   NodeValueTable ProcessInput(const NodeInput* input, const TimeRange &range);
 
+  virtual void ReportUnavailableFootage(StreamPtr stream, Decoder::RetrieveState state, const rational& stream_time);
+
+  const NodeDependency& CurrentPath() const;
+
 private:
   NodeValueDatabase GenerateDatabase(const Node *node, const TimeRange &range);
 
@@ -58,6 +64,8 @@ private:
 
   DecoderCache decoder_cache_;
 
+  NodeDependency path_;
+
 };
 
 #endif // RENDERWORKER_H

app/render/backend/videorenderworker.cpp

@@ -38,7 +38,7 @@ NodeValueTable VideoRenderWorker::RenderInternal(const NodeDependency& path, con
     hasher.addData(reinterpret_cast<const char*>(&vfmt), sizeof(PixelFormat::Format));
     hasher.addData(reinterpret_cast<const char*>(&vmode), sizeof(RenderMode::Mode));
 
-    HashNodeRecursively(&hasher, path.node(), path.in(), &IsCancelled());
+    HashNodeRecursively(&hasher, path.node(), path.in());
     hash = hasher.result();
   }
 
@@ -87,12 +87,12 @@ NodeValueTable VideoRenderWorker::RenderInternal(const NodeDependency& path, con
   return value;
 }
 
-FramePtr VideoRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range, const QAtomicInt* cancelled)
+FramePtr VideoRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range)
 {
-  return decoder->RetrieveVideo(range.in(), cancelled);
+  return decoder->RetrieveVideo(range.in());
 }
 
-void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, const Node* n, const rational& time, const QAtomicInt* cancelled)
+void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, const Node* n, const rational& time)
 {
   // Resolve BlockList
   if (n->IsTrack()) {
@@ -140,7 +140,7 @@ void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, const Node
 
       if (input->IsConnected()) {
         // Traverse down this edge
-        HashNodeRecursively(hash, input->get_connected_node(), input_time, cancelled);
+        HashNodeRecursively(hash, input->get_connected_node(), input_time);
       } else {
         // Grab the value at this time
         QVariant value = input->get_value_at_time(input_time);
@@ -150,32 +150,48 @@ void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, const Node
       // We have one exception for FOOTAGE types, since we resolve the footage into a frame in the renderer
       if (input->data_type() == NodeParam::kFootage) {
         StreamPtr stream = ResolveStreamFromInput(input);
-        DecoderPtr decoder = ResolveDecoderFromInput(stream);
 
-        if (decoder != nullptr) {
-          // Add footage details to hash
+        if (stream) {
+          DecoderPtr decoder = ResolveDecoderFromInput(stream);
 
-          // Footage filename
-          hash->addData(stream->footage()->filename().toUtf8());
+          if (decoder) {
 
-          // Footage last modified date
-          hash->addData(stream->footage()->timestamp().toString().toUtf8());
+            // Add footage details to hash
 
-          // Footage stream
-          hash->addData(QString::number(stream->index()).toUtf8());
+            // Footage filename
+            hash->addData(stream->footage()->filename().toUtf8());
 
-          if (stream->type() == Stream::kImage || stream->type() == Stream::kVideo) {
-            ImageStreamPtr video_stream = std::static_pointer_cast<ImageStream>(stream);
+            // Footage last modified date
+            hash->addData(stream->footage()->timestamp().toString().toUtf8());
+
+            // Footage stream
+            hash->addData(QString::number(stream->index()).toUtf8());
+
+            if (stream->type() == Stream::kImage || stream->type() == Stream::kVideo) {
+              ImageStreamPtr image_stream = std::static_pointer_cast<ImageStream>(stream);
+
+              // Current color config and space
+              hash->addData(image_stream->footage()->project()->ocio_config().toUtf8());
+              hash->addData(image_stream->colorspace().toUtf8());
+
+              // Alpha associated setting
+              hash->addData(QString::number(image_stream->premultiplied_alpha()).toUtf8());
+            }
 
             // Footage timestamp
-            hash->addData(QString::number(decoder->GetTimestampFromTime(input_time, cancelled)).toUtf8());
+            if (stream->type() == Stream::kVideo) {
+              Decoder::RetrieveState state = decoder->GetRetrieveState(input_time);
 
-            // Current color config and space
-            hash->addData(video_stream->footage()->project()->ocio_config().toUtf8());
-            hash->addData(video_stream->colorspace().toUtf8());
+              if (state == Decoder::kReady) {
+                VideoStreamPtr video_stream = std::static_pointer_cast<VideoStream>(stream);
 
-            // Alpha associated setting
-            hash->addData(QString::number(video_stream->premultiplied_alpha()).toUtf8());
+                int64_t timestamp_here = video_stream->get_closest_timestamp_in_frame_index(input_time);
+
+                hash->addData(QString::number(timestamp_here).toUtf8());
+              } else {
+                ReportUnavailableFootage(stream, state, input_time);
+              }
+            }
           }
         }
       }
@@ -258,13 +274,20 @@ NodeValueTable VideoRenderWorker::RenderBlock(const TrackOutput *track, const Ti
   NodeValueTable table;
 
   if (active_block) {
-    table = ProcessNode(NodeDependency(active_block,
-                                       range));
+    table = ProcessNode(NodeDependency(active_block, range));
   }
 
   return table;
 }
 
+void VideoRenderWorker::ReportUnavailableFootage(StreamPtr stream, Decoder::RetrieveState state, const rational &stream_time)
+{
+  emit FootageUnavailable(stream,
+                          state,
+                          TimeRange(CurrentPath().in(), CurrentPath().in() + video_params().time_base()),
+                          stream_time);
+}
+
 ColorProcessorCache *VideoRenderWorker::color_cache()
 {
   return &color_cache_;

app/render/backend/videorenderworker.h

@@ -48,13 +48,13 @@ public:
   void SetOperatingMode(const OperatingMode& mode);
 
 signals:
-  void CompletedFrame(NodeDependency path, qint64 job_time, QByteArray hash, QVariant value);
+  void CompletedFrame(NodeDependency CurrentPath, qint64 job_time, QByteArray hash, QVariant value);
 
-  void CompletedDownload(NodeDependency path, qint64 job_time, QByteArray hash, bool texture_existed);
+  void CompletedDownload(NodeDependency CurrentPath, qint64 job_time, QByteArray hash, bool texture_existed);
 
-  void HashAlreadyBeingCached(NodeDependency path, qint64 job_time, QByteArray hash);
+  void HashAlreadyBeingCached(NodeDependency CurrentPath, qint64 job_time, QByteArray hash);
 
-  void HashAlreadyExists(NodeDependency path, qint64 job_time, QByteArray hash);
+  void HashAlreadyExists(NodeDependency CurrentPath, qint64 job_time, QByteArray hash);
 
   void Aborted();
 
@@ -69,16 +69,18 @@ protected:
 
   virtual void TextureToBuffer(const QVariant& texture, QByteArray& buffer) = 0;
 
-  virtual NodeValueTable RenderInternal(const NodeDependency& path, const qint64& job_time) override;
+  virtual NodeValueTable RenderInternal(const NodeDependency& CurrentPath, const qint64& job_time) override;
 
-  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range, const QAtomicInt *cancelled) override;
+  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) override;
 
   virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) override;
 
+  virtual void ReportUnavailableFootage(StreamPtr stream, Decoder::RetrieveState state, const rational& stream_time) override;
+
   ColorProcessorCache* color_cache();
 
 private:
-  void HashNodeRecursively(QCryptographicHash* hash, const Node *n, const rational &time, const QAtomicInt *cancelled);
+  void HashNodeRecursively(QCryptographicHash* hash, const Node *n, const rational &time);
 
   void Download(QVariant texture, QString filename);
 

app/render/indexmanager.cpp

@@ -0,0 +1,69 @@
+#include "indexmanager.h"
+
+#include "task/taskmanager.h"
+
+IndexManager* IndexManager::instance_ = nullptr;
+
+IndexManager::IndexManager()
+{
+}
+
+void IndexManager::CreateInstance()
+{
+  instance_ = new IndexManager();
+}
+
+IndexManager *IndexManager::instance()
+{
+  return instance_;
+}
+
+void IndexManager::DestroyInstance()
+{
+  delete instance_;
+  instance_ = nullptr;
+}
+
+void IndexManager::StartIndexingStream(StreamPtr stream)
+{
+  if (IsIndexing(stream)) {
+    return;
+  }
+
+  IndexTask* index_task = new IndexTask(stream);
+  threads_.append({stream, index_task});
+
+  connect(stream.get(), &Stream::IndexChanged, this, &IndexManager::StreamIndexUpdatedEvent, Qt::QueuedConnection);
+  connect(index_task, &IndexTask::Succeeded, this, &IndexManager::IndexTaskFinished, Qt::QueuedConnection);
+
+  TaskManager::instance()->AddTask(index_task);
+}
+
+bool IndexManager::IsIndexing(StreamPtr stream)
+{
+  foreach (const StreamThreadPair& stp, threads_) {
+    if (stp.stream == stream) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void IndexManager::IndexTaskFinished()
+{
+  for (int i=0;i<threads_.size();i++) {
+    const StreamThreadPair& stp = threads_.at(i);
+
+    if (stp.task == sender()) {
+      //emit StreamIndexUpdated(stp.stream.get());
+      threads_.removeAt(i);
+      return;
+    }
+  }
+}
+
+void IndexManager::StreamIndexUpdatedEvent()
+{
+  emit StreamIndexUpdated(static_cast<Stream*>(sender()));
+}

app/render/indexmanager.h

@@ -0,0 +1,44 @@
+#ifndef INDEXMANAGER_H
+#define INDEXMANAGER_H
+
+#include <QObject>
+
+#include "project/item/footage/stream.h"
+#include "task/index/index.h"
+
+class IndexManager : public QObject
+{
+  Q_OBJECT
+public:
+  IndexManager();
+
+  static void CreateInstance();
+  static IndexManager* instance();
+  static void DestroyInstance();
+
+  bool IsIndexing(StreamPtr stream);
+
+public slots:
+  void StartIndexingStream(StreamPtr stream);
+
+signals:
+  void StreamIndexUpdated(Stream* stream);
+
+private:
+  static IndexManager* instance_;
+
+  struct StreamThreadPair {
+    StreamPtr stream;
+    IndexTask* task;
+  };
+
+  QList<StreamThreadPair> threads_;
+
+private slots:
+  void IndexTaskFinished();
+
+  void StreamIndexUpdatedEvent();
+
+};
+
+#endif // INDEXMANAGER_H

app/widget/viewer/footageviewer.cpp

@@ -58,7 +58,7 @@ void FootageViewerWidget::SetFootage(Footage *footage)
     }
 
     ConnectViewerNode(viewer_node_, footage->project()->color_manager());
-    video_renderer_->InvalidateCache(0, viewer_node_->Length());
-    audio_renderer_->InvalidateCache(0, viewer_node_->Length());
+    video_renderer_->InvalidateCache(TimeRange(0, viewer_node_->Length()));
+    audio_renderer_->InvalidateCache(TimeRange(0, viewer_node_->Length()));
   }
 }

app/widget/viewer/viewer.cpp

@@ -140,6 +140,8 @@ void ViewerWidget::ConnectNodeInternal(ViewerOutput *n)
 
 void ViewerWidget::DisconnectNodeInternal(ViewerOutput *n)
 {
+  Pause();
+
   SetTimebase(0);
 
   disconnect(n, &ViewerOutput::TimebaseChanged, this, &ViewerWidget::SetTimebase);
@@ -303,7 +305,7 @@ void ViewerWidget::UpdateRendererParameters()
   audio_renderer_->SetParameters(AudioRenderingParams(GetConnectedNode()->audio_params(),
                                                       SampleFormat::GetConfiguredFormatForMode(render_mode)));
 
-  video_renderer_->InvalidateCache(0, GetConnectedNode()->Length());
+  video_renderer_->InvalidateCache(TimeRange(0, GetConnectedNode()->Length()));
 }
 
 void ViewerWidget::ShowContextMenu(const QPoint &pos)
@@ -517,5 +519,5 @@ void ViewerWidget::SetDividerFromMenu(QAction *action)
 
 void ViewerWidget::InvalidateVisible()
 {
-  video_renderer_->InvalidateCache(GetTime(), GetTime());
+  video_renderer_->InvalidateCache(TimeRange(GetTime(), GetTime()));
 }