renderer: generate audio waveforms in render threads

Solves UI lag issues with long audio sequences.

app/audio/audiovisualwaveform.cpp

@@ -162,6 +162,28 @@ void AudioVisualWaveform::OverwriteSums(const AudioVisualWaveform &sums, const r
   length_ = qMax(length_, dest + length);
 }
 
+void AudioVisualWaveform::OverwriteSilence(const rational &start, const rational &length)
+{
+  for (auto it=mipmapped_data_.begin(); it!=mipmapped_data_.end(); it++) {
+    rational rate = it->first;
+
+    Sample& our_arr = it->second;
+
+    double rate_dbl = rate.toDouble();
+
+    // Get our destination sample
+    int our_start_index = time_to_samples(start, rate_dbl);
+    int our_length_index = time_to_samples(length, rate_dbl);
+    int our_end_index = our_start_index + our_length_index;
+
+    if (our_arr.size() < our_end_index) {
+      our_arr.resize(our_end_index);
+    }
+
+    memset(reinterpret_cast<char*>(our_arr.data()) + our_start_index, 0, our_length_index);
+  }
+}
+
 void AudioVisualWaveform::Shift(const rational &from, const rational &to)
 {
   for (auto it=mipmapped_data_.begin(); it!=mipmapped_data_.end(); it++) {

app/audio/audiovisualwaveform.h

@@ -88,6 +88,8 @@ public:
    */
   void OverwriteSums(const AudioVisualWaveform& sums, const rational& dest, const rational& offset = 0, const rational &length = 0);
 
+  void OverwriteSilence(const rational &start, const rational &length);
+
   void Shift(const rational& from, const rational& to);
 
   Sample GetSummaryFromTime(const rational& start, const rational& length) const;

app/render/audioplaybackcache.cpp

@@ -48,6 +48,7 @@ void AudioPlaybackCache::SetParameters(const AudioParams &params)
   }
 
   params_ = params;
+  visual_.set_channel_count(params_.channel_count());
 
   // Restart empty file so there's always "something" to play
   ClearPlaylist();
@@ -55,7 +56,7 @@ void AudioPlaybackCache::SetParameters(const AudioParams &params)
   emit ParametersChanged();
 }
 
-void AudioPlaybackCache::WritePCM(const TimeRange &range, SampleBufferPtr samples, const qint64 &job_time)
+void AudioPlaybackCache::WritePCM(const TimeRange &range, SampleBufferPtr samples, const AudioVisualWaveform *waveform, const qint64 &job_time)
 {
   QList<TimeRange> valid_ranges = GetValidRanges(range, job_time);
   if (valid_ranges.isEmpty()) {
@@ -83,6 +84,7 @@ void AudioPlaybackCache::WritePCM(const TimeRange &range, SampleBufferPtr sample
   foreach (const TimeRange& r, valid_ranges) {
     rational this_segment_in = 0;
 
+    // Write PCM to playlist
     for (auto it=playlist_.begin(); it!=playlist_.end(); it++) {
       rational this_segment_out = this_segment_in + params_.bytes_to_time((*it).size());
 
@@ -138,6 +140,13 @@ void AudioPlaybackCache::WritePCM(const TimeRange &range, SampleBufferPtr sample
       // Each segment is contiguous, so this out will be the next segment's in
       this_segment_in = this_segment_out;
     }
+
+    // Write visual
+    if (waveform) {
+      visual_.OverwriteSums(*waveform, r.in(), r.in() - range.in(), r.length());
+    } else {
+      visual_.OverwriteSilence(r.in(), r.length());
+    }
   }
 
   foreach (const TimeRange& v, ranges_we_validated) {
@@ -149,7 +158,7 @@ void AudioPlaybackCache::WriteSilence(const TimeRange &range, qint64 job_time)
 {
   // WritePCM will automatically fill non-existent bytes with silence, so we just have to send
   // it an empty sample buffer
-  WritePCM(range, nullptr, job_time);
+  WritePCM(range, nullptr, nullptr, job_time);
 }
 
 void AudioPlaybackCache::ShiftEvent(const rational &from_in_time, const rational &to_in_time)

app/render/audioplaybackcache.h

@@ -21,6 +21,7 @@
 #ifndef AUDIOPLAYBACKCACHE_H
 #define AUDIOPLAYBACKCACHE_H
 
+#include "audio/audiovisualwaveform.h"
 #include "common/timerange.h"
 #include "codec/samplebuffer.h"
 #include "render/playbackcache.h"
@@ -65,7 +66,7 @@ public:
 
   void SetParameters(const AudioParams& params);
 
-  void WritePCM(const TimeRange &range, SampleBufferPtr samples, const qint64& job_time);
+  void WritePCM(const TimeRange &range, SampleBufferPtr samples, const AudioVisualWaveform *waveform, const qint64& job_time);
 
   void WriteSilence(const TimeRange &range, qint64 job_time);
 
@@ -181,6 +182,11 @@ public:
    */
   PlaybackDevice* CreatePlaybackDevice(QObject *parent = nullptr) const;
 
+  const AudioVisualWaveform &visual() const
+  {
+    return visual_;
+  }
+
 signals:
   void ParametersChanged();
 
@@ -212,6 +218,8 @@ private:
 
   AudioParams params_;
 
+  AudioVisualWaveform visual_;
+
 };
 
 }

app/render/previewautocacher.cpp

@@ -171,8 +171,11 @@ void PreviewAutoCacher::AudioRendered()
     if (watcher->HasResult()) {
       const TimeRange &range = audio_tasks_.value(watcher);
 
+      AudioVisualWaveform waveform = watcher->GetTicket()->property("waveform").value<AudioVisualWaveform>();
+
       viewer_node_->audio_playback_cache()->WritePCM(range,
                                                      watcher->Get().value<SampleBufferPtr>(),
+                                                     &waveform,
                                                      watcher->GetTicket()->GetJobTime());
 
       bool pcm_is_usable = true;
@@ -506,7 +509,7 @@ void PreviewAutoCacher::TryRender()
 
   if (!invalidated_audio_.isEmpty()) {
     foreach (const TimeRange& range, invalidated_audio_) {
-      std::list<TimeRange> chunks = range.Split(2);
+      std::list<TimeRange> chunks = range.Split(30);
 
       foreach (const TimeRange& r, chunks) {
         RenderTicketWatcher* watcher = new RenderTicketWatcher();

app/render/renderprocessor.cpp

@@ -173,7 +173,16 @@ void RenderProcessor::Run()
       table = GenerateTable(texture_output.node(), texture_output.output(), time);
     }
 
-    ticket_->Finish(table.Get(NodeValue::kSamples));
+    QVariant sample_variant = table.Get(NodeValue::kSamples);
+    SampleBufferPtr samples = sample_variant.value<SampleBufferPtr>();
+    if (samples && ticket_->property("enablewaveforms").toBool()) {
+      AudioVisualWaveform vis;
+      vis.set_channel_count(samples->audio_params().channel_count());
+      vis.OverwriteSamples(samples, samples->audio_params().sample_rate());
+      ticket_->setProperty("waveform", QVariant::fromValue(vis));
+    }
+
+    ticket_->Finish(sample_variant);
     break;
   }
   case RenderManager::kTypeVideoDownload:

app/widget/viewer/audiowaveformview.cpp

@@ -60,8 +60,7 @@ void AudioWaveformView::SetViewer(AudioPlaybackCache *playback)
     pool_.clear();
     pool_.waitForDone();
 
-    disconnect(playback_, &AudioPlaybackCache::Validated, this, &AudioWaveformView::RenderRange);
-    //disconnect(playback_, &AudioPlaybackCache::ParametersChanged, this, &AudioWaveformView::RenderRange);
+    disconnect(playback_, &AudioPlaybackCache::Validated, this, static_cast<void(AudioWaveformView::*)()>(&AudioWaveformView::update));
 
     SetTimebase(0);
   }
@@ -69,14 +68,9 @@ void AudioWaveformView::SetViewer(AudioPlaybackCache *playback)
   playback_ = playback;
 
   if (playback_) {
-    connect(playback_, &AudioPlaybackCache::Validated, this, &AudioWaveformView::RenderRange);
-    //connect(playback_, &AudioPlaybackCache::ParametersChanged, this, &AudioWaveformView::RenderRange);
+    connect(playback_, &AudioPlaybackCache::Validated, this, static_cast<void(AudioWaveformView::*)()>(&AudioWaveformView::update));
 
     SetTimebase(playback_->GetParameters().sample_rate_as_time_base());
-
-    waveform_.set_channel_count(playback_->GetParameters().channel_count());
-
-    RenderRange(TimeRange(0, playback_->GetLength()));
   }
 }
 
@@ -101,7 +95,7 @@ void AudioWaveformView::paintEvent(QPaintEvent *event)
 
   // Draw waveform
   p.setPen(QColor(64, 255, 160)); // FIXME: Hardcoded color
-  AudioVisualWaveform::DrawWaveform(&p, rect(), GetScale(), waveform_, SceneToTime(GetScroll()));
+  AudioVisualWaveform::DrawWaveform(&p, rect(), GetScale(), playback_->visual(), SceneToTime(GetScroll()));
 
   // Draw playhead
   p.setPen(PLAYHEAD_COLOR);
@@ -110,42 +104,4 @@ void AudioWaveformView::paintEvent(QPaintEvent *event)
   p.drawLine(playhead_x, 0, playhead_x, height());
 }
 
-void AudioWaveformView::RenderRange(TimeRange range)
-{
-  // Limit range to length
-  range = TimeRange(qMax(rational(0), range.in()), qMin(playback_->GetLength(), range.out()));
-
-  // Floor to second increments
-  int64_t start = qFloor(range.in().toDouble());
-  int64_t end = qCeil(range.out().toDouble());
-
-  for (; start!=end; start++) {
-    TimeRange this_range(start, start+1);
-
-    QFutureWatcher<AudioVisualWaveform>* watcher = new QFutureWatcher<AudioVisualWaveform>();
-    connect(watcher, &QFutureWatcher<AudioVisualWaveform>::finished, this, &AudioWaveformView::BackgroundFinished);
-
-    jobs_.insert(this_range, watcher);
-
-    watcher->setFuture(QtConcurrent::run(&pool_, GenerateWaveform, playback_->CreatePlaybackDevice(), playback_->GetParameters(), this_range));
-  }
-}
-
-void AudioWaveformView::BackgroundFinished()
-{
-  QFutureWatcher<AudioVisualWaveform>* watcher = static_cast<QFutureWatcher<AudioVisualWaveform>*>(sender());
-
-  for (auto it=jobs_.begin(); it!=jobs_.end(); it++) {
-    if (it.value() == watcher) {
-      AudioVisualWaveform rendered = watcher->result();
-      waveform_.OverwriteSums(rendered, it.key().in());
-      jobs_.erase(it);
-      update();
-      break;
-    }
-  }
-
-  delete watcher;
-}
-
 }

app/widget/viewer/audiowaveformview.h

@@ -24,7 +24,6 @@
 #include <QtConcurrent/QtConcurrent>
 #include <QWidget>
 
-#include "audio/audiovisualwaveform.h"
 #include "render/audioparams.h"
 #include "render/audioplaybackcache.h"
 #include "widget/timeruler/seekablewidget.h"
@@ -37,32 +36,16 @@ class AudioWaveformView : public SeekableWidget
 public:
   AudioWaveformView(QWidget* parent = nullptr);
 
-  //void SetData(const QString& file, const AudioRenderingParams& params);
-
   void SetViewer(AudioPlaybackCache *playback);
 
-  const AudioVisualWaveform* waveform() const
-  {
-    return &waveform_;
-  }
-
 protected:
   virtual void paintEvent(QPaintEvent* event) override;
 
 private:
-  void RenderRange(TimeRange range);
-
   QThreadPool pool_;
 
   AudioPlaybackCache *playback_;
 
-  AudioVisualWaveform waveform_;
-
-  QHash<TimeRange, QFutureWatcher<AudioVisualWaveform>*> jobs_;
-
-private slots:
-  void BackgroundFinished();
-
 };
 
 }

app/widget/viewer/viewer.cpp

@@ -455,7 +455,7 @@ void ViewerWidget::StartAudioOutput()
     AudioManager::instance()->StartOutput(audio_cache,
                                           audio_cache->GetParameters().time_to_bytes(GetTime()),
                                           playback_speed_);
-    emit AudioManager::instance()->OutputWaveformStarted(waveform_view_->waveform(),
+    emit AudioManager::instance()->OutputWaveformStarted(&audio_cache->visual(),
                                                          GetTime(), playback_speed_);
   }
 }