renderer: reimported audio rendering functions

app/node/output/viewer/viewer.cpp

@@ -86,9 +86,9 @@ QString ViewerOutput::Description() const
 
 void ViewerOutput::InvalidateCache(const TimeRange &range, NodeInput *from, NodeInput *source)
 {
-  if (from == texture_input_ || from == samples_input_) {
-    emit GraphChangedFrom(source);
+  emit GraphChangedFrom(source);
 
+  if (from == texture_input_ || from == samples_input_) {
     TimeRange invalidated_range(qMax(rational(), range.in()),
                                 qMin(GetLength(), range.out()));
 

app/render/audioplaybackcache.cpp

@@ -57,6 +57,59 @@ void AudioPlaybackCache::SetParameters(const AudioRenderingParams &params)
   emit ParametersChanged();
 }
 
+void AudioPlaybackCache::WritePCM(const TimeRange &range, SampleBufferPtr samples)
+{
+  QFile f(filename_);
+  qDebug() << "Writing PCM to" << filename_;
+  if (f.open(QFile::ReadWrite)) {
+    qint64 start_offset = params_.time_to_bytes(range.in());
+    qint64 max_len = params_.time_to_bytes(range.out());
+    qint64 write_len = max_len - start_offset;
+
+    QByteArray a = samples->toPackedData();
+
+    if (f.size() < max_len) {
+      f.resize(max_len);
+    }
+
+    f.seek(start_offset);
+    f.write(a);
+
+    if (write_len > a.size()) {
+      // Fill remaining space with silence
+      QByteArray s(write_len - a.size(), 0x00);
+      f.write(s);
+    }
+
+    f.close();
+  } else {
+    qWarning() << "Failed to write PCM data to" << filename_;
+  }
+}
+
+void AudioPlaybackCache::WriteSilence(const TimeRange &range)
+{
+  QFile f(filename_);
+  if (f.open(QFile::ReadWrite)) {
+    qint64 start_offset = params_.time_to_bytes(range.in());
+    qint64 max_len = params_.time_to_bytes(range.out());
+    qint64 write_len = max_len - start_offset;
+
+    if (f.size() < max_len) {
+      f.resize(max_len);
+    }
+
+    f.seek(start_offset);
+
+    QByteArray a(write_len, 0x00);
+    f.write(a);
+
+    f.close();
+  } else {
+    qWarning() << "Failed to write PCM data to" << filename_;
+  }
+}
+
 const QString &AudioPlaybackCache::GetCacheFilename() const
 {
   return filename_;

app/render/audioplaybackcache.h

@@ -39,7 +39,9 @@ public:
 
   void SetParameters(const AudioRenderingParams& params);
 
-  void WritePCM(SampleBufferPtr samples);
+  void WritePCM(const TimeRange &range, SampleBufferPtr samples);
+
+  void WriteSilence(const TimeRange &range);
 
   const QString& GetCacheFilename() const;
 

app/render/backend/renderbackend.cpp

@@ -25,6 +25,8 @@
 
 #include "config/config.h"
 #include "core.h"
+#include "task/conform/conform.h"
+#include "task/taskmanager.h"
 #include "window/mainwindow/mainwindow.h"
 
 OLIVE_NAMESPACE_ENTER
@@ -37,7 +39,8 @@ RenderBackend::RenderBackend(QObject *parent) :
   divider_(1),
   render_mode_(RenderMode::kOnline),
   pix_fmt_(PixelFormat::PIX_FMT_RGBA32F),
-  sample_fmt_(SampleFormat::SAMPLE_FMT_FLT)
+  sample_fmt_(SampleFormat::SAMPLE_FMT_FLT),
+  ic_from_conform_(false)
 {
   // FIXME: Don't create in CLI mode
   cancel_dialog_ = new RenderCancelDialog(Core::instance()->main_window());
@@ -203,11 +206,11 @@ RenderWorker *RenderBackend::GetInstanceFromPool(QVector<RenderWorker *> &worker
         instance->Init(viewer_node_);
       }
 
-      connect(instance,
-              &RenderWorker::FinishedJob,
-              this,
-              &RenderBackend::WorkerFinished,
-              Qt::QueuedConnection);
+      connect(instance, &RenderWorker::FinishedJob,
+              this, &RenderBackend::WorkerFinished, Qt::QueuedConnection);
+
+      connect(instance, &RenderWorker::AudioConformUnavailable,
+              this, &RenderBackend::AudioConformUnavailable, Qt::QueuedConnection);
     } else {
       instance = worker_pool.at(instance_queuer % worker_pool.size());
       instance_queuer++;
@@ -217,11 +220,128 @@ RenderWorker *RenderBackend::GetInstanceFromPool(QVector<RenderWorker *> &worker
   return instance;
 }
 
+void RenderBackend::ListenForConformSignal(AudioStreamPtr s)
+{
+  foreach (const ConformWaitInfo& info, conform_wait_info_) {
+    if (info.stream == s) {
+      // We've probably already connected to this one
+      return;
+    }
+  }
+
+  connect(s.get(), &AudioStream::ConformAppended, this, &RenderBackend::AudioConformUpdated);
+}
+
+void RenderBackend::StopListeningForConformSignal(AudioStream *s)
+{
+  foreach (const ConformWaitInfo& info, conform_wait_info_) {
+    if (info.stream.get() == s) {
+      // There are still conforms we're waiting for, don't disconnect
+      return;
+    }
+  }
+
+  disconnect(s, &AudioStream::ConformAppended, this, &RenderBackend::AudioConformUpdated);
+}
+
+void RenderBackend::AudioConformUnavailable(StreamPtr stream, TimeRange range, rational stream_time, AudioRenderingParams params)
+{
+  ConformWaitInfo info = {stream, params, range, stream_time};
+
+  if (conform_wait_info_.contains(info)) {
+    return;
+  }
+
+  AudioStreamPtr audio_stream = std::static_pointer_cast<AudioStream>(stream);
+
+  if (audio_stream->try_start_conforming(params)) {
+
+    // Start indexing process
+    ListenForConformSignal(audio_stream);
+
+    conform_wait_info_.append(info);
+
+    ConformTask* conform_task = new ConformTask(audio_stream, params);
+
+    TaskManager::instance()->AddTask(conform_task);
+
+  } else if (audio_stream->has_conformed_version(params)) {
+
+    // Conform JUST finished, requeue this time
+    ic_from_conform_ = true;
+    AudioInvalidated(range);
+    ic_from_conform_ = false;
+
+  } else {
+
+    // A conform task is already running, so we'll just wait for it
+    ListenForConformSignal(audio_stream);
+
+    conform_wait_info_.append(info);
+
+  }
+}
+
+void RenderBackend::AudioConformUpdated(AudioRenderingParams params)
+{
+  AudioStream *stream = static_cast<AudioStream*>(sender());
+
+  for (int i=0;i<conform_wait_info_.size();i++) {
+    const ConformWaitInfo& info = conform_wait_info_.at(i);
+
+    if (info.stream.get() == stream
+        && info.params == params) {
+
+      // Make a copy so the values we use aren't corrupt
+      ConformWaitInfo copy = info;
+
+      // Remove this entry from the list
+      conform_wait_info_.removeAt(i);
+      i--;
+
+      // Send invalidate cache signal
+      ic_from_conform_ = true;
+      AudioInvalidated(copy.affected_range);
+      ic_from_conform_ = false;
+
+    }
+  }
+
+  StopListeningForConformSignal(stream);
+}
+
 void RenderBackend::AudioInvalidated(const TimeRange& r)
 {
+  if (!ic_from_conform_) {
+    // Cancel any ranges waiting on a conform here since obviously the contents have changed
+    for (int i=0;i<conform_wait_info_.size();i++) {
+      ConformWaitInfo& info = conform_wait_info_[i];
+
+      // FIXME: Code shamelessly copied from TimeRangeList::RemoveTimeRange()
+
+      if (r.Contains(info.affected_range)) {
+        conform_wait_info_.removeAt(i);
+        i--;
+      } else if (info.affected_range.Contains(r, false, false)) {
+        ConformWaitInfo copy = info;
+
+        info.affected_range.set_out(r.in());
+        copy.affected_range.set_in(r.out());
+
+        conform_wait_info_.append(copy);
+      } else if (info.affected_range.in() < r.in() && info.affected_range.out() > r.in()) {
+        info.affected_range.set_out(r.in());
+      } else if (info.affected_range.in() < r.out() && info.affected_range.out() > r.out()) {
+        info.affected_range.set_in(r.out());
+      }
+    }
+  }
+
   QFutureWatcher<SampleBufferPtr>* watcher = new QFutureWatcher<SampleBufferPtr>();
   connect(watcher, &QFutureWatcher<SampleBufferPtr>::finished, this, &RenderBackend::AudioRendered);
 
+  audio_jobs_.insert(watcher, r);
+
   watcher->setFuture(QtConcurrent::run(&video_thread_pool_,
                                        GetInstanceFromPool(audio_instance_pool_, audio_thread_pool_, audio_instance_queuer_),
                                        &RenderWorker::RenderAudio,
@@ -232,10 +352,16 @@ void RenderBackend::AudioRendered()
 {
   QFutureWatcher<SampleBufferPtr>* watcher = static_cast<QFutureWatcher<SampleBufferPtr>*>(sender());
 
-  if (watcher->result()) {
-    qDebug() << "AUDIO: Received" << watcher->result()->sample_count_per_channel() << "samples!";
-  } else {
-    qDebug() << "AUDIO: Received null";
+  if (audio_jobs_.contains(watcher)) {
+    TimeRange r = audio_jobs_.take(watcher);
+
+    if (watcher->result()) {
+      qDebug() << "Received" << watcher->result()->sample_count_per_channel() << "samples";
+      viewer_node_->audio_playback_cache()->WritePCM(r, watcher->result());
+    } else {
+      qDebug() << "Received null";
+      viewer_node_->audio_playback_cache()->WriteSilence(r);
+    }
   }
 
   watcher->deleteLater();

app/render/backend/renderbackend.h

@@ -104,18 +104,31 @@ private:
 
   // AUDIO MEMBERS
   SampleFormat::Format sample_fmt_;
+  QHash< QFutureWatcher<SampleBufferPtr>*, TimeRange > audio_jobs_;
 
   struct ConformWaitInfo {
     StreamPtr stream;
+    AudioRenderingParams params;
     TimeRange affected_range;
     rational stream_time;
 
     bool operator==(const ConformWaitInfo& rhs) const;
   };
 
-  QList<ConformWaitInfo> footage_wait_info_;
+  QList<ConformWaitInfo> conform_wait_info_;
+
+  void ListenForConformSignal(AudioStreamPtr s);
+
+  void StopListeningForConformSignal(AudioStream *s);
+
+  bool ic_from_conform_;
 
 private slots:
+  void AudioConformUnavailable(StreamPtr stream, TimeRange range,
+                               rational stream_time, AudioRenderingParams params);
+
+  void AudioConformUpdated(OLIVE_NAMESPACE::AudioRenderingParams params);
+
   void AudioInvalidated(const TimeRange &r);
 
   void AudioRendered();

app/render/backend/renderworker.cpp

@@ -109,6 +109,15 @@ SampleBufferPtr RenderWorker::RenderAudio(const TimeRange &range)
 
   UpdateData(true);
 
+  audio_render_time_ = range;
+
+  qDebug() << "Audio deps:";
+
+  QList<Node*> d = viewer_->samples_input()->GetDependencies();
+  foreach (Node* n, d) {
+    qDebug() << "  " << n;
+  }
+
   NodeValueTable table = ProcessInput(viewer_->samples_input(), range);
 
   QVariant samples = table.Get(NodeParam::kSamples);
@@ -132,12 +141,8 @@ void RenderWorker::UpdateData(bool block_for_update)
 
 NodeValueTable RenderWorker::GenerateBlockTable(const TrackOutput *track, const TimeRange &range)
 {
-  qDebug() << "Hello from track" << track << "type" << track->track_type();
-
   if (track->track_type() == Timeline::kTrackTypeAudio) {
 
-    qDebug() << "Hello?";
-
     QList<Block*> active_blocks = track->BlocksAtTimeRange(range);
 
     // All these blocks will need to output to a buffer so we create one here
@@ -309,6 +314,8 @@ void RenderWorker::ProcessNodeEvent(const Node *node, const TimeRange &range, No
 
 void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable *table)
 {
+  qDebug() << "Stream type" << stream->type();
+
   if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) {
 
     ImageStreamPtr video_stream = std::static_pointer_cast<ImageStream>(stream);
@@ -348,6 +355,7 @@ void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &inp
 
   } else if (stream->type() == Stream::kAudio) {
 
+    qDebug() << "Hello!";
     table->Push(GetDataFromStream(stream, input_time));
 
   }
@@ -358,7 +366,25 @@ NodeValue RenderWorker::GetDataFromStream(StreamPtr stream, const TimeRange &inp
   DecoderPtr decoder = ResolveDecoderFromInput(stream);
 
   if (decoder) {
-    return FrameToTexture(decoder, stream, input_time);
+    if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) {
+      return FrameToTexture(decoder, stream, input_time);
+    } else if (stream->type() == Stream::kAudio) {
+      qDebug() << "Decoding audio!";
+      if (decoder->HasConformedVersion(audio_params())) {
+        qDebug() << "  Retrieving audio!";
+        SampleBufferPtr frame = decoder->RetrieveAudio(input_time.in(), input_time.length(),
+                                                       audio_params());
+
+        if (frame) {
+          qDebug() << "  Returning audio!";
+          return NodeValue(NodeParam::kSamples, QVariant::fromValue(frame));
+        }
+      } else {
+        qDebug() << "  Conform doesn't exist! AAAAA";
+        emit AudioConformUnavailable(decoder->stream(), audio_render_time_,
+                                     input_time.out(), audio_params());
+      }
+    }
   }
 
   return NodeValue();
@@ -498,6 +524,7 @@ void RenderWorker::Init(ViewerOutput* viewer)
 
   Queue(viewer->texture_input());
   Queue(viewer->samples_input());
+  Queue(viewer->track_input(Timeline::kTrackTypeAudio));
   ProcessQueue();
 }
 

app/render/backend/renderworker.h

@@ -124,7 +124,8 @@ protected:
   }
 
 signals:
-  void AudioConformUnavailable();
+  void AudioConformUnavailable(StreamPtr stream, TimeRange range,
+                               rational stream_time, AudioRenderingParams params);
 
   void FinishedJob();
 
@@ -160,6 +161,7 @@ private:
   DecoderCache decoder_cache_;
 
   ViewerOutput* viewer_;
+  TimeRange audio_render_time_;
   QList<NodeInput*> queued_updates_;
   QHash<Node*, Node*> copy_map_;
   bool available_;