renderer: move conform to task

Improved rendering system and user interface

Fixes #1375

app/codec/CMakeLists.txt

@@ -19,21 +19,23 @@ add_subdirectory(oiio)
 
 set(OLIVE_SOURCES
   ${OLIVE_SOURCES}
-  codec/decoder.h
+  codec/conformmanager.cpp
+  codec/conformmanager.h
   codec/decoder.cpp
-  codec/encoder.h
+  codec/decoder.h
   codec/encoder.cpp
-  codec/exportcodec.h
+  codec/encoder.h
   codec/exportcodec.cpp
-  codec/exportformat.h
+  codec/exportcodec.h
   codec/exportformat.cpp
-  codec/frame.h
+  codec/exportformat.h
   codec/frame.cpp
-  codec/samplebuffer.h
+  codec/frame.h
   codec/samplebuffer.cpp
-  codec/waveinput.h
+  codec/samplebuffer.h
   codec/waveinput.cpp
-  codec/waveoutput.h
+  codec/waveinput.h
   codec/waveoutput.cpp
+  codec/waveoutput.h
   PARENT_SCOPE
 )

app/codec/decoder.cpp

@@ -30,18 +30,12 @@
 #include "common/ffmpegutils.h"
 #include "common/filefunctions.h"
 #include "common/timecodefunctions.h"
+#include "conformmanager.h"
 #include "node/project/project.h"
-#ifdef USE_OTIO
-#include "task/project/loadotio/loadotio.h"
-#endif
 #include "task/taskmanager.h"
 
 namespace olive {
 
-QMutex Decoder::currently_conforming_mutex_;
-QWaitCondition Decoder::currently_conforming_wait_cond_;
-QVector<Decoder::CurrentlyConforming> Decoder::currently_conforming_;
-
 const rational Decoder::kAnyTimecode = RATIONAL_MIN;
 
 Decoder::Decoder()
@@ -112,7 +106,7 @@ FramePtr Decoder::RetrieveVideo(const rational &timecode, const RetrieveVideoPar
   return RetrieveVideoInternal(timecode, divider);
 }
 
-SampleBufferPtr Decoder::RetrieveAudio(const TimeRange &range, const AudioParams &params, const QString& cache_path, Footage::LoopMode loop_mode, const QAtomicInt *cancelled)
+Decoder::RetrieveAudioData Decoder::RetrieveAudio(const TimeRange &range, const AudioParams &params, const QString& cache_path, Footage::LoopMode loop_mode, RenderMode::Mode mode)
 {
   QMutexLocker locker(&mutex_);
 
@@ -120,58 +114,24 @@ SampleBufferPtr Decoder::RetrieveAudio(const TimeRange &range, const AudioParams
 
   if (!stream_.IsValid()) {
     qCritical() << "Can't retrieve audio on a closed decoder";
-    return nullptr;
+    return {kInvalid, nullptr, nullptr};
   }
 
   if (!SupportsAudio()) {
     qCritical() << "Decoder doesn't support audio";
-    return nullptr;
+    return {kInvalid, nullptr, nullptr};
   }
 
-  // Determine if we already have a conformed version
-  QString conform_filename = GetConformedFilename(cache_path, params);
-  CurrentlyConforming want_conform = {stream_, params};
-
-  currently_conforming_mutex_.lock();
-
-  // Wait for conform to complete
-  while (currently_conforming_.contains(want_conform)) {
-    currently_conforming_wait_cond_.wait(&currently_conforming_mutex_);
+  // Get conform state from ConformManager
+  ConformManager::Conform conform = ConformManager::instance()->GetConformState(id(), cache_path, stream_, params, (mode == RenderMode::kOnline));
+  if (conform.state == ConformManager::kConformGenerating) {
+    return {kWaitingForConform, nullptr, conform.task};
   }
 
   // See if we got the conform
-  SampleBufferPtr buffer = RetrieveAudioFromConform(conform_filename, range, loop_mode);
+  SampleBufferPtr out_buffer = RetrieveAudioFromConform(conform.filename, range, loop_mode);
 
-  if (!buffer) {
-    // We'll need to conform this ourselves
-    currently_conforming_.append(want_conform);
-    currently_conforming_mutex_.unlock();
-
-    // We conform to a different filename until it's done to make it clear even across sessions
-    // whether this conform is ready or not
-    QString working_fn = conform_filename;
-    working_fn.append(QStringLiteral(".working"));
-
-    if (ConformAudioInternal(working_fn, params, cancelled)) {
-      // Move file to standard conform name, making it clear this conform is ready for use
-      QFile::remove(conform_filename);
-      QFile::rename(working_fn, conform_filename);
-
-      // Return audio as planned
-      buffer = RetrieveAudioFromConform(conform_filename, range, loop_mode);
-    } else {
-      // Failed
-      qCritical() << "Failed to conform audio";
-    }
-
-    currently_conforming_mutex_.lock();
-    currently_conforming_.removeOne(want_conform);
-    currently_conforming_wait_cond_.wakeAll();
-  }
-
-  currently_conforming_mutex_.unlock();
-
-  return buffer;
+  return {kOK, out_buffer, nullptr};
 }
 
 qint64 Decoder::GetLastAccessedTime()
@@ -194,6 +154,11 @@ void Decoder::Close()
   }
 }
 
+bool Decoder::ConformAudio(const QString &output_filename, const AudioParams &params, const QAtomicInt *cancelled)
+{
+  return ConformAudioInternal(output_filename, params, cancelled);
+}
+
 /*
  * DECODER STATIC PUBLIC MEMBERS
  */
@@ -228,23 +193,6 @@ DecoderPtr Decoder::CreateFromID(const QString &id)
   return nullptr;
 }
 
-QString Decoder::GetConformedFilename(const QString& cache_path, const AudioParams &params)
-{
-  QString index_fn = QStringLiteral("%1.%2:%3").arg(FileFunctions::GetUniqueFileIdentifier(stream_.filename()),
-                                                    QString::number(stream_.stream()));
-
-  index_fn = QDir(cache_path).filePath(index_fn);
-
-  index_fn.append('.');
-  index_fn.append(QString::number(params.sample_rate()));
-  index_fn.append('.');
-  index_fn.append(QString::number(params.format()));
-  index_fn.append('.');
-  index_fn.append(QString::number(params.channel_layout()));
-
-  return index_fn;
-}
-
 int64_t Decoder::GetTimeInTimebaseUnits(const rational &time, const rational &timebase, int64_t start_time)
 {
   return Timecode::time_to_timestamp(time, timebase) + start_time;

app/codec/decoder.h

@@ -37,6 +37,7 @@ extern "C" {
 #include "common/rational.h"
 #include "node/project/footage/footage.h"
 #include "node/project/footage/footagedescription.h"
+#include "task/task.h"
 
 namespace olive {
 
@@ -184,6 +185,18 @@ public:
    */
   FramePtr RetrieveVideo(const rational& timecode, const RetrieveVideoParams& divider);
 
+  enum RetrieveAudioStatus {
+    kInvalid = -1,
+    kOK,
+    kWaitingForConform
+  };
+
+  struct RetrieveAudioData {
+    RetrieveAudioStatus status;
+    SampleBufferPtr samples;
+    Task *task;
+  };
+
   /**
    * @brief Retrieve audio data from footage
    *
@@ -192,7 +205,7 @@ public:
    *
    * This function is thread safe and can only run while the decoder is open. \see Open()
    */
-  SampleBufferPtr RetrieveAudio(const TimeRange& range, const AudioParams& params, const QString &cache_path, Footage::LoopMode loop_mode, const QAtomicInt *cancelled);
+  RetrieveAudioData RetrieveAudio(const TimeRange& range, const AudioParams& params, const QString &cache_path, Footage::LoopMode loop_mode, RenderMode::Mode mode);
 
   /**
    * @brief Determine the last time this decoder instance was used in any way
@@ -219,6 +232,11 @@ public:
    */
   void Close();
 
+  /**
+   * @brief Conform audio stream
+   */
+  bool ConformAudio(const QString &output_filename, const AudioParams &params, const QAtomicInt *cancelled = nullptr);
+
   /**
    * @brief Create a Decoder instance using a Decoder ID
    *
@@ -271,21 +289,6 @@ protected:
 
   void SignalProcessingProgress(int64_t ts, int64_t duration);
 
-  /**
-   * @brief Get the destination filename of an audio stream conformed to a set of parameters
-   */
-  QString GetConformedFilename(const QString &cache_path, const AudioParams &params);
-
-  struct CurrentlyConforming {
-    CodecStream stream;
-    AudioParams params;
-
-    bool operator==(const CurrentlyConforming& rhs) const
-    {
-      return this->stream == rhs.stream && this->params == rhs.params;
-    }
-  };
-
   /**
    * @brief Return currently open stream
    *
@@ -298,10 +301,6 @@ protected:
 
   static int64_t GetTimeInTimebaseUnits(const rational& time, const rational& timebase, int64_t start_time);
 
-  static QMutex currently_conforming_mutex_;
-  static QWaitCondition currently_conforming_wait_cond_;
-  static QVector<CurrentlyConforming> currently_conforming_;
-
 signals:
   /**
    * @brief While indexing, this signal will provide progress as a percentage (0-100 inclusive) if

app/core.cpp

@@ -35,6 +35,7 @@
 
 #include "audio/audiomanager.h"
 #include "cli/clitask/clitaskdialog.h"
+#include "codec/conformmanager.h"
 #include "common/filefunctions.h"
 #include "common/xmlutils.h"
 #include "config/config.h"
@@ -144,6 +145,9 @@ void Core::Start()
   // Initialize FrameManager
   FrameManager::CreateInstance();
 
+  // Initialize ConformManager
+  ConformManager::CreateInstance();
+
   //
   // Start application
   //
@@ -188,6 +192,8 @@ void Core::Stop()
     }
   }
 
+  ConformManager::DestroyInstance();
+
   FrameManager::DestroyInstance();
 
   RenderManager::DestroyInstance();

app/render/previewautocacher.cpp

@@ -3,6 +3,7 @@
 #include <QApplication>
 #include <QtConcurrent/QtConcurrent>
 
+#include "codec/conformmanager.h"
 #include "node/project/project.h"
 #include "render/rendermanager.h"
 #include "render/renderprocessor.h"
@@ -15,7 +16,8 @@ PreviewAutoCacher::PreviewAutoCacher() :
   use_custom_range_(false),
   single_frame_render_(nullptr),
   last_update_time_(0),
-  ignore_next_mouse_button_(false)
+  ignore_next_mouse_button_(false),
+  last_conform_task_(0)
 {
   paused_ = !Config::Current()[QStringLiteral("AutoCacheEnabled")].toBool(),
 
@@ -24,6 +26,8 @@ PreviewAutoCacher::PreviewAutoCacher() :
   delayed_requeue_timer_.setInterval(Config::Current()[QStringLiteral("AutoCacheDelay")].toInt());
   delayed_requeue_timer_.setSingleShot(true);
   connect(&delayed_requeue_timer_, &QTimer::timeout, this, &PreviewAutoCacher::RequeueFrames);
+
+  connect(ConformManager::instance(), &ConformManager::ConformReady, this, &PreviewAutoCacher::ConformFinished);
 }
 
 PreviewAutoCacher::~PreviewAutoCacher()
@@ -131,7 +135,7 @@ void PreviewAutoCacher::VideoInvalidated(const TimeRange &range)
 
 void PreviewAutoCacher::AudioInvalidated(const TimeRange &range)
 {
-  ClearAudioQueue();
+//  ClearAudioQueue();
 
   // Start jobs to re-render the audio at this range, split into 2 second chunks
   invalidated_audio_.insert(range);
@@ -165,35 +169,52 @@ void PreviewAutoCacher::AudioRendered()
 
   if (audio_tasks_.contains(watcher)) {
     if (watcher->HasResult()) {
-      viewer_node_->audio_playback_cache()->WritePCM(audio_tasks_.value(watcher),
+      const TimeRange &range = audio_tasks_.value(watcher);
+
+      viewer_node_->audio_playback_cache()->WritePCM(range,
                                                      watcher->Get().value<SampleBufferPtr>(),
                                                      watcher->GetTicket()->GetJobTime());
 
-      // Retrieve visual waveforms
-      QVector<RenderProcessor::RenderedWaveform> waveform_list = watcher->GetTicket()->property("waveforms").value< QVector<RenderProcessor::RenderedWaveform> >();
-      foreach (const RenderProcessor::RenderedWaveform& waveform_info, waveform_list) {
-        // Find original track
-        Track* track = nullptr;
+      bool pcm_is_usable = true;
 
-        for (auto it=copy_map_.cbegin(); it!=copy_map_.cend(); it++) {
-          if (it.value() == waveform_info.track) {
-            track = static_cast<Track*>(it.key());
-            break;
-          }
+      if (watcher->GetTicket()->property("incomplete").toBool()) {
+        if (last_conform_task_ > watcher->GetTicket()->GetJobTime()) {
+          // Requeue now
+          viewer_node_->audio_playback_cache()->Invalidate(range, QDateTime::currentMSecsSinceEpoch());
+          pcm_is_usable = false;
+        } else {
+          // Wait for conform
+          audio_needing_conform_.insert(range);
         }
+      }
 
-        if (track) {
-          QList<TimeRange> valid_ranges = viewer_node_->audio_playback_cache()->GetValidRanges(waveform_info.range,
-                                                                                               watcher->GetTicket()->GetJobTime());
-          if (!valid_ranges.isEmpty()) {
-            // Generate visual waveform in this background thread
-            track->waveform().set_channel_count(viewer_node_->GetAudioParams().channel_count());
+      if (pcm_is_usable) {
+        // Retrieve visual waveforms
+        QVector<RenderProcessor::RenderedWaveform> waveform_list = watcher->GetTicket()->property("waveforms").value< QVector<RenderProcessor::RenderedWaveform> >();
+        foreach (const RenderProcessor::RenderedWaveform& waveform_info, waveform_list) {
+          // Find original track
+          Track* track = nullptr;
 
-            foreach (const TimeRange& r, valid_ranges) {
-              track->waveform().OverwriteSums(waveform_info.waveform, r.in(), r.in() - waveform_info.range.in(), r.length());
+          for (auto it=copy_map_.cbegin(); it!=copy_map_.cend(); it++) {
+            if (it.value() == waveform_info.track) {
+              track = static_cast<Track*>(it.key());
+              break;
             }
+          }
 
-            emit track->PreviewChanged();
+          if (track) {
+            QList<TimeRange> valid_ranges = viewer_node_->audio_playback_cache()->GetValidRanges(waveform_info.range,
+                                                                                                 watcher->GetTicket()->GetJobTime());
+            if (!valid_ranges.isEmpty()) {
+              // Generate visual waveform in this background thread
+              track->waveform().set_channel_count(viewer_node_->GetAudioParams().channel_count());
+
+              foreach (const TimeRange& r, valid_ranges) {
+                track->waveform().OverwriteSums(waveform_info.waveform, r.in(), r.in() - waveform_info.range.in(), r.length());
+              }
+
+              emit track->PreviewChanged();
+            }
           }
         }
       }
@@ -491,7 +512,7 @@ void PreviewAutoCacher::TryRender()
         RenderTicketWatcher* watcher = new RenderTicketWatcher();
         connect(watcher, &RenderTicketWatcher::Finished, this, &PreviewAutoCacher::AudioRendered);
         audio_tasks_.insert(watcher, r);
-        watcher->SetTicket(RenderManager::instance()->RenderAudio(copied_viewer_node_, r, true));
+        watcher->SetTicket(RenderManager::instance()->RenderAudio(copied_viewer_node_, r, RenderMode::kOffline, true));
       }
     }
 
@@ -581,6 +602,18 @@ void PreviewAutoCacher::RequeueFrames()
   }
 }
 
+void PreviewAutoCacher::ConformFinished()
+{
+  last_conform_task_ = QDateTime::currentMSecsSinceEpoch();
+
+  if (viewer_node_) {
+    foreach (const TimeRange &range, audio_needing_conform_) {
+      viewer_node_->audio_playback_cache()->Invalidate(range, QDateTime::currentMSecsSinceEpoch());
+    }
+    audio_needing_conform_.clear();
+  }
+}
+
 void PreviewAutoCacher::IgnoreNextMouseButton()
 {
   ignore_next_mouse_button_ = true;
@@ -624,6 +657,9 @@ void PreviewAutoCacher::SetViewerNode(ViewerOutput *viewer_node)
     // No more immediate passthroughts
     video_immediate_passthroughs_.clear();
 
+    // No more audio conforms
+    audio_needing_conform_.clear();
+
     // Delete all of our copied nodes
     qDeleteAll(created_nodes_);
     created_nodes_.clear();

app/render/previewautocacher.h

@@ -167,6 +167,10 @@ private:
 
   QTimer delayed_requeue_timer_;
 
+  TimeRangeList audio_needing_conform_;
+
+  qint64 last_conform_task_;
+
 private slots:
   /**
    * @brief Handler for when the NodeGraph reports a video change over a certain time range
@@ -213,6 +217,8 @@ private slots:
    */
   void RequeueFrames();
 
+  void ConformFinished();
+
 };
 
 }

app/render/rendermanager.cpp

@@ -181,12 +181,12 @@ RenderTicketPtr RenderManager::RenderFrame(ViewerOutput *viewer, ColorManager* c
   return ticket;
 }
 
-RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange& r, bool generate_waveforms, bool prioritize)
+RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange& r, RenderMode::Mode mode, bool generate_waveforms, bool prioritize)
 {
-  return RenderAudio(viewer, r, viewer->GetAudioParams(), generate_waveforms, prioritize);
+  return RenderAudio(viewer, r, viewer->GetAudioParams(), mode, generate_waveforms, prioritize);
 }
 
-RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange &r, const AudioParams &params, bool generate_waveforms, bool prioritize)
+RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange &r, const AudioParams &params, RenderMode::Mode mode, bool generate_waveforms, bool prioritize)
 {
   // Create ticket
   RenderTicketPtr ticket = std::make_shared<RenderTicket>();
@@ -194,6 +194,7 @@ RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange
   ticket->setProperty("viewer", Node::PtrToValue(viewer));
   ticket->setProperty("time", QVariant::fromValue(r));
   ticket->setProperty("type", kTypeAudio);
+  ticket->setProperty("mode", mode);
   ticket->setProperty("enablewaveforms", generate_waveforms);
   ticket->setProperty("aparam", QVariant::fromValue(params));
 

app/render/rendermanager.h

@@ -105,8 +105,8 @@ public:
    *
    * This function is thread-safe.
    */
-  RenderTicketPtr RenderAudio(ViewerOutput* viewer, const TimeRange& r, const AudioParams& params, bool generate_waveforms, bool prioritize = false);
-  RenderTicketPtr RenderAudio(ViewerOutput *viewer, const TimeRange& r, bool generate_waveforms, bool prioritize = false);
+  RenderTicketPtr RenderAudio(ViewerOutput* viewer, const TimeRange& r, const AudioParams& params, RenderMode::Mode mode, bool generate_waveforms, bool prioritize = false);
+  RenderTicketPtr RenderAudio(ViewerOutput *viewer, const TimeRange& r, RenderMode::Mode mode, bool generate_waveforms, bool prioritize = false);
 
   RenderTicketPtr SaveFrameToCache(FrameHashCache* cache, FramePtr frame, const QByteArray& hash, bool prioritize = false);
 

app/render/renderprocessor.cpp

@@ -443,13 +443,15 @@ QVariant RenderProcessor::ProcessAudioFootage(const FootageJob &stream, const Ti
   if (decoder) {
     const AudioParams& audio_params = ticket_->property("aparam").value<AudioParams>();
 
-    SampleBufferPtr frame = decoder->RetrieveAudio(input_time, audio_params,
-                                                   stream.cache_path(),
-                                                   stream.loop_mode(),
-                                                   &IsCancelled());
+    Decoder::RetrieveAudioData status = decoder->RetrieveAudio(input_time, audio_params,
+                                                               stream.cache_path(),
+                                                               stream.loop_mode(),
+                                                               static_cast<RenderMode::Mode>(ticket_->property("mode").toInt()));
 
-    if (frame) {
-      value = QVariant::fromValue(frame);
+    if (status.status == Decoder::kOK && status.samples) {
+      value = QVariant::fromValue(status.samples);
+    } else if (status.status == Decoder::kWaitingForConform) {
+      ticket_->setProperty("incomplete", true);
     }
   }
 

app/task/conform/conform.cpp

@@ -20,42 +20,33 @@
 
 #include "conform.h"
 
-#include "codec/decoder.h"
-
 namespace olive {
 
-ConformTask::ConformTask(Footage* footage, int index, const AudioParams& params) :
-  footage_(footage),
-  index_(index),
-  params_(params)
+ConformTask::ConformTask(const QString &decoder_id, const Decoder::CodecStream &stream, const AudioParams& params, const QString &output_filename) :
+  decoder_id_(decoder_id),
+  stream_(stream),
+  params_(params),
+  output_filename_(output_filename)
 {
-  SetTitle(tr("Conforming Audio %1:%2").arg(footage_->filename(), QString::number(index_)));
+  SetTitle(tr("Conforming Audio %1:%2").arg(stream.filename(), QString::number(stream.stream())));
 }
 
 bool ConformTask::Run()
 {
-  // Conforming is done by the renderer now, but I would like to use something like this just to
-  // show progress
+  DecoderPtr decoder = Decoder::CreateFromID(decoder_id_);
 
-  /*if (stream_->footage()->decoder().isEmpty()) {
-    SetError(tr("Failed to find decoder to conform audio stream"));
+  if (!decoder->Open(stream_)) {
+    SetError(tr("Failed to open decoder for audio conform"));
     return false;
-  } else {
-    DecoderPtr decoder = Decoder::CreateFromID(stream_->footage()->decoder());
+  }
 
-    decoder->set_stream(stream_);
+  connect(decoder.get(), &Decoder::IndexProgress, this, &ConformTask::ProgressChanged);
 
-    connect(decoder.get(), &Decoder::IndexProgress, this, &ConformTask::ProgressChanged);
+  bool ret = decoder->ConformAudio(output_filename_, params_, &IsCancelled());
 
-    if (!decoder->ConformAudio(&IsCancelled(), params_)) {
-      SetError(tr("Failed to conform audio"));
-      return false;
-    } else {
-      return true;
-    }
-  }*/
+  decoder->Close();
 
-  return true;
+  return ret;
 }
 
 }

app/task/conform/conform.h

@@ -21,6 +21,7 @@
 #ifndef CONFORMTASK_H
 #define CONFORMTASK_H
 
+#include "codec/decoder.h"
 #include "node/project/footage/footage.h"
 #include "render/audioparams.h"
 #include "task/task.h"
@@ -31,18 +32,20 @@ class ConformTask : public Task
 {
   Q_OBJECT
 public:
-  ConformTask(Footage* stream, int index, const AudioParams& params);
+  ConformTask(const QString &decoder_id, const Decoder::CodecStream &stream, const AudioParams& params, const QString &output_filename);
 
 protected:
   virtual bool Run() override;
 
 private:
-  Footage* footage_;
+  QString decoder_id_;
 
-  int index_;
+  Decoder::CodecStream stream_;
 
   AudioParams params_;
 
+  QString output_filename_;
+
 };
 
 }

app/task/render/render.cpp

@@ -71,7 +71,7 @@ bool RenderTask::Render(ColorManager* manager,
       watcher->setProperty("range", QVariant::fromValue(this_range));
       PrepareWatcher(watcher, &watcher_thread);
       IncrementRunningTickets();
-      watcher->SetTicket(RenderManager::instance()->RenderAudio(viewer_, this_range, audio_params_, false));
+      watcher->SetTicket(RenderManager::instance()->RenderAudio(viewer_, this_range, audio_params_, mode, false));
 
       r = end;
     }

app/task/task.h

@@ -99,6 +99,8 @@ public slots:
     // Print how long this task took for debugging purposes
     qDebug() << this << "took" << (QDateTime::currentMSecsSinceEpoch() - start_time_);
 
+    emit Finished(this, ret);
+
     return ret;
   }
 
@@ -159,6 +161,13 @@ signals:
    */
   void ProgressChanged(double d);
 
+  /**
+   * @brief Emitted when task is finished
+   *
+   * Do NOT delete immediately after this signal, call deleteLater() instead.
+   */
+  void Finished(Task *task, bool succeeded);
+
 private:
   QString title_;