renderer: updated audio waveform generated for new cache overhaul

app/core.cpp

@@ -173,6 +173,7 @@ void Core::DeclareTypesForQt()
   qRegisterMetaType<OLIVE_NAMESPACE::GenerateJob>();
   qRegisterMetaType<OLIVE_NAMESPACE::VideoParams>();
   qRegisterMetaType<OLIVE_NAMESPACE::MainWindowLayoutInfo>();
+  qRegisterMetaType<OLIVE_NAMESPACE::RenderTicketPtr>();
 }
 
 void Core::Start()

app/render/backend/renderbackend.cpp

@@ -41,7 +41,7 @@ RenderBackend::RenderBackend(QObject *parent) :
   viewer_node_(nullptr),
   autocache_enabled_(false),
   autocache_paused_(false),
-  preview_job_time_(0),
+  generate_audio_previews_(false),
   render_mode_(RenderMode::kOnline),
   autocache_has_changed_(false),
   use_custom_autocache_range_(false)
@@ -374,6 +374,7 @@ void RenderBackend::RunNextJob()
     for (int i=0;i<workers_.size();i++) {
       RenderWorker* worker = CreateNewWorker();
 
+      connect(worker, &RenderWorker::WaveformGenerated, this, &RenderBackend::WorkerGeneratedWaveform);
       connect(worker, &RenderWorker::FinishedJob, this, &RenderBackend::WorkerFinished);
 
       workers_.replace(i, {worker, false});
@@ -393,19 +394,22 @@ void RenderBackend::RunNextJob()
       worker->SetAudioParams(audio_params_);
       worker->SetVideoDownloadMatrix(video_download_matrix_);
       worker->SetRenderMode(render_mode_);
-      if (preview_job_time_) {
-        worker->EnablePreviewGeneration(viewer_node_->audio_playback_cache(), preview_job_time_);
-      }
+      worker->SetPreviewGenerationEnabled(generate_audio_previews_);
       worker->SetCopyMap(&copy_map_);
 
+      // Move ticket from queue to running list
       RenderTicketPtr ticket = render_queue_.front();
       render_queue_.pop_front();
-
       running_tickets_.push_back(ticket);
+
+      // Create watcher to remove from running list
       RenderTicketWatcher* watcher = new RenderTicketWatcher();
       connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::TicketFinished);
       watcher->SetTicket(ticket);
 
+      // Set job time to now
+      ticket->SetJobTime();
+
       switch (ticket->GetType()) {
       case RenderTicket::kTypeHash:
         QtConcurrent::run(&thread_pool_,
@@ -458,6 +462,28 @@ void RenderBackend::TicketFinished()
   running_tickets_.remove(ticket);
 }
 
+void RenderBackend::WorkerGeneratedWaveform(RenderTicketPtr ticket, TrackOutput *track, AudioVisualWaveform samples, TimeRange range)
+{
+  qDebug() << "Hello?";
+
+  QList<TimeRange> valid_ranges = viewer_node_->audio_playback_cache()->GetValidRanges(range,
+                                                                                       ticket->GetJobTime());
+  if (!valid_ranges.isEmpty()) {
+    // Generate visual waveform in this background thread
+    track->waveform_lock()->lock();
+
+    track->waveform().set_channel_count(audio_params_.channel_count());
+
+    foreach (const TimeRange& r, valid_ranges) {
+      track->waveform().OverwriteSums(samples, r.in(), r.in() - range.in(), r.length());
+    }
+
+    track->waveform_lock()->unlock();
+
+    emit track->PreviewChanged();
+  }
+}
+
 void RenderBackend::AutoCacheVideoInvalidated(const TimeRange &range)
 {
   ClearVideoQueue();
@@ -465,7 +491,7 @@ void RenderBackend::AutoCacheVideoInvalidated(const TimeRange &range)
   // Hash these frames since that should be relatively quick.
   RenderTicketWatcher* watcher = new RenderTicketWatcher();
   QVector<rational> frames = viewer_node_->video_frame_cache()->GetFrameListFromTimeRange({range});
-  autocache_hash_tasks_.insert(watcher, {frames, QDateTime::currentMSecsSinceEpoch()});
+  autocache_hash_tasks_.insert(watcher, frames);
   connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheHashesGenerated);
   watcher->SetTicket(Hash(frames));
 }
@@ -474,7 +500,7 @@ void RenderBackend::AutoCacheAudioInvalidated(const TimeRange &range)
 {
   // Start a task to re-render the audio at this range
   RenderTicketWatcher* watcher = new RenderTicketWatcher();
-  autocache_audio_tasks_.insert(watcher, {range, QDateTime::currentMSecsSinceEpoch()});
+  autocache_audio_tasks_.insert(watcher, range);
   connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheAudioRendered);
   watcher->SetTicket(RenderAudio(range, true));
 }
@@ -509,17 +535,15 @@ void RenderBackend::AutoCacheHashesGenerated()
 
   if (autocache_hash_tasks_.contains(watcher)) {
     if (!watcher->WasCancelled()) {
-      const HashJobInfo& info = autocache_hash_tasks_.value(watcher);
-
       QFutureWatcher<void>* hw = new QFutureWatcher<void>();
       connect(hw, &QFutureWatcher<void>::finished, this, &RenderBackend::AutoCacheHashesProcessed);
       autocache_hash_process_tasks_.append(hw);
       hw->setFuture(QtConcurrent::run(this,
                                       &RenderBackend::SetHashes,
                                       viewer_node_->video_frame_cache(),
-                                      info.times,
+                                      autocache_hash_tasks_.value(watcher),
                                       watcher->Get().value<QVector<QByteArray> >(),
-                                      info.job_time));
+                                      watcher->GetTicket()->GetJobTime()));
     }
 
     autocache_hash_tasks_.remove(watcher);
@@ -547,11 +571,9 @@ void RenderBackend::AutoCacheAudioRendered()
 
   if (autocache_audio_tasks_.contains(watcher)) {
     if (!watcher->WasCancelled()) {
-      const AudioJobInfo& job_info = autocache_audio_tasks_.value(watcher);
-
-      viewer_node_->audio_playback_cache()->WritePCM(job_info.range,
+      viewer_node_->audio_playback_cache()->WritePCM(autocache_audio_tasks_.value(watcher),
                                                      watcher->Get().value<SampleBufferPtr>(),
-                                                     job_info.job_time);
+                                                     watcher->GetTicket()->GetJobTime());
     }
 
     autocache_audio_tasks_.remove(watcher);
@@ -566,14 +588,14 @@ void RenderBackend::AutoCacheVideoRendered()
 
   if (autocache_video_tasks_.contains(watcher)) {
     if (!watcher->WasCancelled()) {
-      const VideoJobInfo& info = autocache_video_tasks_.value(watcher);
+      const QByteArray& hash = autocache_video_tasks_.value(watcher);
 
       // Download frame in another thread
       QFutureWatcher<bool>* w = new QFutureWatcher<bool>();
-      autocache_video_download_tasks_.insert(w, info);
+      autocache_video_download_tasks_.insert(w, hash);
       connect(w, &QFutureWatcher<bool>::finished, this, &RenderBackend::AutoCacheVideoDownloaded);
       w->setFuture(QtConcurrent::run(FrameHashCache::SaveCacheFrame,
-                                     info.hash,
+                                     hash,
                                      watcher->Get().value<FramePtr>()));
     }
 
@@ -590,11 +612,11 @@ void RenderBackend::AutoCacheVideoDownloaded()
   if (autocache_video_download_tasks_.contains(watcher)) {
     if (!watcher->isCanceled()) {
       if (watcher->result()) {
-        const VideoJobInfo& info = autocache_video_download_tasks_.value(watcher);
+        const QByteArray& hash = autocache_video_download_tasks_.value(watcher);
 
-        currently_caching_hashes_.removeOne(info.hash);
+        currently_caching_hashes_.removeOne(hash);
 
-        viewer_node_->video_frame_cache()->ValidateFramesWithHash(info.hash);
+        viewer_node_->video_frame_cache()->ValidateFramesWithHash(hash);
       } else {
         qCritical() << "Failed to download video frame";
       }
@@ -792,7 +814,7 @@ void RenderBackend::AutoCacheRequeueFrames()
 
         RenderTicketWatcher* watcher = new RenderTicketWatcher();
         connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheVideoRendered);
-        autocache_video_tasks_.insert(watcher, {hash, QDateTime::currentMSecsSinceEpoch()});
+        autocache_video_tasks_.insert(watcher, hash);
 
         watcher->SetTicket(RenderFrame(t, false, hash));
       }

app/render/backend/renderbackend.h

@@ -95,9 +95,9 @@ public:
     render_mode_ = e;
   }
 
-  void EnablePreviewGeneration(qint64 job_time)
+  void SetPreviewGenerationEnabled(bool e)
   {
-    preview_job_time_ = job_time;
+    generate_audio_previews_ = e;
   }
 
   void ProcessUpdateQueue();
@@ -185,7 +185,7 @@ private:
   bool autocache_enabled_;
   bool autocache_paused_;
 
-  qint64 preview_job_time_;
+  bool generate_audio_previews_;
 
   RenderMode::Mode render_mode_;
 
@@ -202,30 +202,15 @@ private:
   static QThreadPool thread_pool_;
   void SetActiveInstance();
 
-  struct HashJobInfo {
-    QVector<rational> times;
-    qint64 job_time;
-  };
-
-  struct AudioJobInfo {
-    TimeRange range;
-    qint64 job_time;
-  };
-
-  struct VideoJobInfo {
-    QByteArray hash;
-    qint64 job_time;
-  };
-
-  QMap<RenderTicketWatcher*, HashJobInfo> autocache_hash_tasks_;
+  QMap<RenderTicketWatcher*, QVector<rational> > autocache_hash_tasks_;
 
   QList<QFutureWatcher<void>*> autocache_hash_process_tasks_;
 
-  QMap<RenderTicketWatcher*, AudioJobInfo> autocache_audio_tasks_;
+  QMap<RenderTicketWatcher*, TimeRange> autocache_audio_tasks_;
 
-  QMap<RenderTicketWatcher*, VideoJobInfo> autocache_video_tasks_;
+  QMap<RenderTicketWatcher*, QByteArray> autocache_video_tasks_;
 
-  QMap<QFutureWatcher<bool>*, VideoJobInfo> autocache_video_download_tasks_;
+  QMap<QFutureWatcher<bool>*, QByteArray> autocache_video_download_tasks_;
 
   QVector<QByteArray> currently_caching_hashes_;
 
@@ -236,6 +221,8 @@ private slots:
 
   void TicketFinished();
 
+  void WorkerGeneratedWaveform(OLIVE_NAMESPACE::RenderTicketPtr ticket, OLIVE_NAMESPACE::TrackOutput* track, OLIVE_NAMESPACE::AudioVisualWaveform samples, OLIVE_NAMESPACE::TimeRange range);
+
   void AutoCacheVideoInvalidated(const OLIVE_NAMESPACE::TimeRange &range);
 
   void AutoCacheAudioInvalidated(const OLIVE_NAMESPACE::TimeRange &range);

app/render/backend/renderticket.cpp

@@ -26,7 +26,8 @@ RenderTicket::RenderTicket(Type type, const QVariant &time) :
   finished_(false),
   cancelled_(false),
   time_(time),
-  type_(type)
+  type_(type),
+  job_time_(0)
 {
 }
 

app/render/backend/renderticket.h

@@ -21,6 +21,7 @@
 #ifndef RENDERTICKET_H
 #define RENDERTICKET_H
 
+#include <QDateTime>
 #include <QMutex>
 #include <QWaitCondition>
 
@@ -42,6 +43,16 @@ public:
 
   RenderTicket(Type type, const QVariant& time);
 
+  qint64 GetJobTime() const
+  {
+    return job_time_;
+  }
+
+  void SetJobTime()
+  {
+    job_time_ = QDateTime::currentMSecsSinceEpoch();
+  }
+
   const QVariant& GetTime() const
   {
     return time_;
@@ -87,10 +98,14 @@ private:
 
   Type type_;
 
+  qint64 job_time_;
+
 };
 
 using RenderTicketPtr = std::shared_ptr<RenderTicket>;
 
 OLIVE_NAMESPACE_EXIT
 
+Q_DECLARE_METATYPE(OLIVE_NAMESPACE::RenderTicketPtr)
+
 #endif // RENDERTICKET_H

app/render/backend/renderworker.cpp

@@ -34,14 +34,15 @@ OLIVE_NAMESPACE_ENTER
 RenderWorker::RenderWorker(RenderBackend* parent) :
   parent_(parent),
   available_(true),
-  audio_mode_is_preview_(false),
-  preview_cache_(nullptr),
+  generate_audio_previews_(false),
   render_mode_(RenderMode::kOnline)
 {
 }
 
 void RenderWorker::Hash(RenderTicketPtr ticket, ViewerOutput *viewer, const QVector<rational> &times)
 {
+  ticket_ = ticket;
+
   QVector<QByteArray> hashes(times.size());
 
   for (int i=0;i<hashes.size();i++) {
@@ -73,6 +74,8 @@ QByteArray RenderWorker::HashNode(const Node *n, const VideoParams &params, cons
 
 void RenderWorker::RenderFrame(RenderTicketPtr ticket, ViewerOutput* viewer, const rational &time)
 {
+  ticket_ = ticket;
+
   NodeValueTable table = ProcessInput(viewer->texture_input(),
                                       TimeRange(time, time + video_params_.time_base()));
 
@@ -109,6 +112,8 @@ void RenderWorker::RenderFrame(RenderTicketPtr ticket, ViewerOutput* viewer, con
 
 void RenderWorker::RenderAudio(RenderTicketPtr ticket, ViewerOutput* viewer, const TimeRange &range)
 {
+  ticket_ = ticket;
+
   NodeValueTable table = ProcessInput(viewer->samples_input(), range);
 
   QVariant samples = table.Get(NodeParam::kSamples);
@@ -168,38 +173,26 @@ NodeValueTable RenderWorker::GenerateBlockTable(const TrackOutput *track, const
       NodeValueTable::Merge({merged_table, table});
     }
 
-    if (preview_cache_) {
+    if (generate_audio_previews_) {
       // Find original track object
       TrackOutput* original_track = nullptr;
 
-      QList<TimeRange> valid_ranges = preview_cache_->GetValidRanges(range, preview_job_time_);
-      if (!valid_ranges.isEmpty()) {
-        QHash<Node*, Node*>::const_iterator i;
-        for (i=copy_map_->constBegin(); i!=copy_map_->constEnd(); i++) {
-          if (i.value() == track) {
-            original_track = static_cast<TrackOutput*>(i.key());
-            break;
-          }
+      // Have to do a manual loop since our track is const and QHash won't take it
+      QHash<Node*, Node*>::const_iterator i;
+      for (i=copy_map_->constBegin(); i!=copy_map_->constEnd(); i++) {
+        if (i.value() == track) {
+          original_track = static_cast<TrackOutput*>(i.key());
+          break;
         }
+      }
 
-        // Generate visual waveform in this background thread
-        if (original_track) {
-          AudioVisualWaveform visual_waveform;
-          visual_waveform.set_channel_count(audio_params_.channel_count());
-          visual_waveform.OverwriteSamples(block_range_buffer, audio_params_.sample_rate());
+      if (original_track) {
+        // Generate a visual waveform and send it back to the main thread
+        AudioVisualWaveform visual_waveform;
+        visual_waveform.set_channel_count(audio_params_.channel_count());
+        visual_waveform.OverwriteSamples(block_range_buffer, audio_params_.sample_rate());
 
-          original_track->waveform_lock()->lock();
-
-          original_track->waveform().set_channel_count(audio_params_.channel_count());
-
-          foreach (const TimeRange& r, valid_ranges) {
-            original_track->waveform().OverwriteSums(visual_waveform, r.in(), r.in() - range.in(), r.length());
-          }
-
-          original_track->waveform_lock()->unlock();
-
-          emit original_track->PreviewChanged();
-        }
+        emit WaveformGenerated(ticket_, original_track, visual_waveform, range);
       }
     }
 
@@ -386,37 +379,24 @@ QVariant RenderWorker::ProcessAudioFootage(StreamPtr stream, const TimeRange &in
     // See if we have a conformed version of this audio
     if (!decoder->HasConformedVersion(audio_params())) {
 
-      // If not, check what audio mode we're in
-      if (audio_mode_is_preview_) {
+      // If not, the audio needs to be conformed
+      // For online rendering/export, it's a waste of time to render the audio until we have
+      // all we need, so we try to handle the conform ourselves
+      AudioStreamPtr as = std::static_pointer_cast<AudioStream>(stream);
 
-        // For preview, we report the conform is missing and finish the render without it
-        // temporarily. The backend that picks up this signal will recache this section once the
-        // conform is available.
-        emit AudioConformUnavailable(decoder->stream(),
-                                     audio_render_time_,
-                                     input_time.out(),
-                                     audio_params());
+      // Check if any other threads are conforming this audio
+      if (as->try_start_conforming(audio_params())) {
+
+        // If not, conform it ourselves
+        decoder->ConformAudio(&IsCancelled(), audio_params());
 
       } else {
 
-        // For online rendering/export, it's a waste of time to render the audio until we have
-        // all we need, so we try to handle the conform ourselves
-        AudioStreamPtr as = std::static_pointer_cast<AudioStream>(stream);
+        // If another thread is conforming already, hackily try to wait until it's done.
+        do {
+          QThread::msleep(1000);
+        } while (!as->has_conformed_version(audio_params()) && !IsCancelled());
 
-        // Check if any other threads are conforming this audio
-        if (as->try_start_conforming(audio_params())) {
-
-          // If not, conform it ourselves
-          decoder->ConformAudio(&IsCancelled(), audio_params());
-
-        } else {
-
-          // If another thread is conforming already, hackily try to wait until it's done.
-          do {
-            QThread::msleep(1000);
-          } while (!as->has_conformed_version(audio_params()) && !IsCancelled());
-
-        }
       }
 
     }

app/render/backend/renderworker.h

@@ -63,11 +63,6 @@ public:
     video_download_matrix_ = mat;
   }
 
-  void SetAudioModeIsPreview(bool audio_mode_is_preview)
-  {
-    audio_mode_is_preview_ = audio_mode_is_preview;
-  }
-
   void SetCopyMap(QHash<Node*, Node*>* copy_map)
   {
     copy_map_ = copy_map;
@@ -78,10 +73,9 @@ public:
     render_mode_ = mode;
   }
 
-  void EnablePreviewGeneration(AudioPlaybackCache* cache, qint64 job_time)
+  void SetPreviewGenerationEnabled(bool e)
   {
-    preview_cache_ = cache;
-    preview_job_time_ = job_time;
+    generate_audio_previews_ = e;
   }
 
   void Hash(RenderTicketPtr ticket, ViewerOutput* viewer, const QVector<rational>& times);
@@ -143,7 +137,7 @@ signals:
 
   void FinishedJob();
 
-  void WaveformGenerated(OLIVE_NAMESPACE::TrackOutput* track, OLIVE_NAMESPACE::AudioVisualWaveform samples, OLIVE_NAMESPACE::TimeRange start);
+  void WaveformGenerated(OLIVE_NAMESPACE::RenderTicketPtr ticket, OLIVE_NAMESPACE::TrackOutput* track, OLIVE_NAMESPACE::AudioVisualWaveform samples, OLIVE_NAMESPACE::TimeRange range);
 
 private:
   DecoderPtr ResolveDecoderFromInput(StreamPtr stream);
@@ -152,6 +146,8 @@ private:
 
   RenderBackend* parent_;
 
+  RenderTicketPtr ticket_;
+
   VideoParams video_params_;
 
   AudioParams audio_params_;
@@ -173,10 +169,7 @@ private:
   TimeRange audio_render_time_;
   bool available_;
 
-  bool audio_mode_is_preview_;
-
-  AudioPlaybackCache* preview_cache_;
-  qint64 preview_job_time_;
+  bool generate_audio_previews_;
 
   QHash<Node*, Node*>* copy_map_;
 

app/widget/viewer/viewer.cpp

@@ -110,6 +110,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) :
   renderer_ = new OpenGLBackend(this);
   renderer_->SetAutoCacheEnabled(true);
   renderer_->SetRenderMode(RenderMode::kOffline);
+  renderer_->SetPreviewGenerationEnabled(true);
 
   // Ensures that seeking on the waveform view updates the time as expected
   connect(waveform_view_, &AudioWaveformView::TimeChanged, this, &ViewerWidget::SetTimeAndSignal);