timeline/renderer: implemented basic waveform display support

app/audio/CMakeLists.txt

@@ -26,6 +26,8 @@ set(OLIVE_SOURCES
   audio/outputmanager.cpp
   audio/sampleformat.h
   audio/sampleformat.cpp
+  audio/sumsamples.h
+  audio/sumsamples.cpp
   audio/tempoprocessor.h
   audio/tempoprocessor.cpp
   PARENT_SCOPE

app/audio/sumsamples.cpp

@@ -0,0 +1,64 @@
+#include "sumsamples.h"
+
+#include <QDebug>
+
+const int SampleSummer::kSumSampleRate = 200;
+
+QVector<SampleSummer::Sum> SampleSummer::SumSamples(const float *samples, int nb_samples, int nb_channels)
+{
+  return SumSamplesInternal<float>(samples, nb_samples, nb_channels);
+}
+
+QVector<SampleSummer::Sum> SampleSummer::SumSamples(const qfloat16 *samples, int nb_samples, int nb_channels)
+{
+  return SumSamplesInternal<qfloat16>(samples, nb_samples, nb_channels);
+}
+
+QVector<SampleSummer::Sum> SampleSummer::ReSumSamples(const SampleSummer::Sum *samples, int nb_samples, int nb_channels)
+{
+  QVector<SampleSummer::Sum> summed_samples(nb_channels);
+
+  for (int i=0;i<nb_samples;i++) {
+    int channel = i%nb_channels;
+
+    const SampleSummer::Sum& sample = samples[i];
+
+    if (sample.min < summed_samples[channel].min) {
+      summed_samples[channel].min = sample.min;
+    }
+
+    if (sample.max > summed_samples[channel].max) {
+      summed_samples[channel].max = sample.max;
+    }
+  }
+
+  return summed_samples;
+}
+
+SampleSummer::Sum::Sum()
+{
+  min = 0;
+  max = 0;
+}
+
+template<typename T>
+QVector<SampleSummer::Sum> SampleSummer::SumSamplesInternal(const T *samples, int nb_samples, int nb_channels)
+{
+  QVector<SampleSummer::Sum> summed_samples(nb_channels);
+
+  for (int i=0;i<nb_samples;i++) {
+    int channel = i%nb_channels;
+
+    float sample = samples[i];
+
+    if (sample < summed_samples[channel].min) {
+      summed_samples[channel].min = sample;
+    }
+
+    if (sample > summed_samples[channel].max) {
+      summed_samples[channel].max = sample;
+    }
+  }
+
+  return summed_samples;
+}

app/audio/sumsamples.h

@@ -0,0 +1,30 @@
+#ifndef SUMSAMPLES_H
+#define SUMSAMPLES_H
+
+#include <QFloat16>
+#include <QVector>
+
+class SampleSummer {
+public:
+  struct Sum {
+    Sum();
+
+    qfloat16 min;
+    qfloat16 max;
+  };
+
+  // FIXME: Move to config
+  static const int kSumSampleRate;
+
+  static QVector<Sum> SumSamples(const float* samples, int nb_samples, int nb_channels);
+  static QVector<Sum> SumSamples(const qfloat16* samples, int nb_samples, int nb_channels);
+
+  static QVector<Sum> ReSumSamples(const SampleSummer::Sum* samples, int nb_samples, int nb_channels);
+
+private:
+  template <typename T>
+  static QVector<Sum> SumSamplesInternal(const T* samples, int nb_samples, int nb_channels);
+
+};
+
+#endif // SUMSAMPLES_H

app/render/backend/audio/audiobackend.cpp

@@ -24,7 +24,7 @@ bool AudioBackend::InitInternal()
   // Initiate one thread per CPU core
   for (int i=0;i<threads().size();i++) {
     // Create one processor object for each thread
-    AudioWorker* processor = new AudioWorker(decoder_cache());
+    AudioWorker* processor = new AudioWorker(decoder_cache(), &copy_map_);
     processor->SetParameters(params());
     processors_.append(processor);
   }
@@ -39,12 +39,13 @@ void AudioBackend::CloseInternal()
 bool AudioBackend::CompileInternal()
 {
   // This backend doesn't compile anything yet
-  return true;
+  return AudioRenderBackend::CompileInternal();
 }
 
 void AudioBackend::DecompileInternal()
 {
   // This backend doesn't compile anything yet
+  AudioRenderBackend::DecompileInternal();
 }
 
 void AudioBackend::ConnectWorkerToThis(RenderWorker *worker)

app/render/backend/audio/audioworker.cpp

@@ -1,7 +1,7 @@
 #include "audioworker.h"
 
-AudioWorker::AudioWorker(DecoderCache* decoder_cache, QObject *parent) :
-  AudioRenderWorker(decoder_cache, parent)
+AudioWorker::AudioWorker(DecoderCache* decoder_cache, QHash<Node *, Node *> *copy_map, QObject *parent) :
+  AudioRenderWorker(decoder_cache, copy_map, parent)
 {
 }
 

app/render/backend/audio/audioworker.h

@@ -6,7 +6,7 @@
 class AudioWorker : public AudioRenderWorker
 {
 public:
-  AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr);
+  AudioWorker(DecoderCache* decoder_cache, QHash<Node*, Node*>* copy_map, QObject* parent = nullptr);
 
 protected:
   virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override;

app/render/backend/audiorenderbackend.cpp

@@ -41,6 +41,20 @@ void AudioRenderBackend::DisconnectViewer(ViewerOutput *node)
   disconnect(node, &ViewerOutput::AudioGraphChanged, this, &AudioRenderBackend::QueueRecompile);
 }
 
+bool AudioRenderBackend::CompileInternal()
+{
+  for (int i=0;i<copied_graph_.nodes().size();i++) {
+    copy_map_.insert(copied_graph_.nodes().at(i), source_node_list_.at(i));
+  }
+
+  return true;
+}
+
+void AudioRenderBackend::DecompileInternal()
+{
+  copy_map_.clear();
+}
+
 bool AudioRenderBackend::GenerateCacheIDInternal(QCryptographicHash &hash)
 {
   if (!params_.is_valid()) {

app/render/backend/audiorenderbackend.h

@@ -29,6 +29,10 @@ protected:
 
   virtual void DisconnectViewer(ViewerOutput* node) override;
 
+  virtual bool CompileInternal() override;
+
+  virtual void DecompileInternal() override;
+
   /**
    * @brief Internal function for generating the cache ID
    */
@@ -40,6 +44,8 @@ protected:
 
   virtual void ConnectWorkerToThis(RenderWorker* worker) override;
 
+  QHash<Node*, Node*> copy_map_;
+
 private:
   struct ConformWaitInfo {
     StreamPtr stream;

app/render/backend/audiorenderworker.cpp

@@ -1,9 +1,15 @@
 #include "audiorenderworker.h"
 
-#include "audio/audiomanager.h"
+#include <QDir>
+#include <QFloat16>
 
-AudioRenderWorker::AudioRenderWorker(DecoderCache* decoder_cache, QObject *parent) :
-  RenderWorker(decoder_cache, parent)
+#include "audio/audiomanager.h"
+#include "audio/sumsamples.h"
+#include "config/config.h"
+
+AudioRenderWorker::AudioRenderWorker(DecoderCache* decoder_cache, QHash<Node *, Node *> *copy_map, QObject *parent) :
+  RenderWorker(decoder_cache, parent),
+  copy_map_(copy_map)
 {
 }
 
@@ -82,6 +88,43 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
         memcpy(block_range_buffer.data()+destination_offset,
                samples_from_this_block.data(),
                actual_copy_size);
+
+        // Save waveform to file
+        QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString());
+        QDir waveform_loc = local_appdata_dir.filePath("waveform");
+        waveform_loc.mkpath(".");
+        QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast<quintptr>(copy_map_->value(b)))));
+        wave_file.open(QFile::ReadWrite);
+
+        // FIXME: Assumes 32-bit float
+        float* flt_samples = reinterpret_cast<float*>(samples_from_this_block.data());
+        int nb_sample = samples_from_this_block.size() / sizeof(float);
+
+        // FIXME: Hardcoded sample rate
+        // We use S16 here as a size-compatible substitute for qfloat16/half
+        AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S16);
+        int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()) * waveform_params.channel_count();
+
+        qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()) * 2;
+        qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()) * 2;
+        qint64 end_offset = start_offset + length_offset;
+
+        if (wave_file.size() < end_offset) {
+          wave_file.resize(end_offset);
+        }
+
+        wave_file.seek(start_offset);
+
+        for (int i=0;i<nb_sample;i+=chunk_size) {
+          QVector<SampleSummer::Sum> summary = SampleSummer::SumSamples(&flt_samples[i],
+                                                                        qMin(chunk_size, nb_sample - i),
+                                                                        audio_params().channel_count());
+
+          wave_file.write(reinterpret_cast<const char*>(summary.constData()),
+                          summary.size() * sizeof(SampleSummer::Sum));
+        }
+
+        wave_file.close();
       }
     }
 

app/render/backend/audiorenderworker.h

@@ -7,7 +7,7 @@ class AudioRenderWorker : public RenderWorker
 {
   Q_OBJECT
 public:
-  AudioRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr);
+  AudioRenderWorker(DecoderCache* decoder_cache, QHash<Node*, Node*>* copy_map, QObject* parent = nullptr);
 
   void SetParameters(const AudioRenderingParams& audio_params);
 
@@ -26,6 +26,8 @@ protected:
 private:
   AudioRenderingParams audio_params_;
 
+  QHash<Node*, Node*>* copy_map_;
+
 };
 
 #endif // AUDIORENDERWORKER_H

app/render/backend/renderbackend.h

@@ -108,6 +108,10 @@ protected:
 
   QHash<TimeRange, qint64> render_job_info_;
 
+  QList<Node*> source_node_list_;
+
+  NodeGraph copied_graph_;
+
 protected slots:
   void QueueRecompile();
 
@@ -139,9 +143,6 @@ private:
 
   QString cache_id_;
 
-  QList<Node*> source_node_list_;
-  NodeGraph copied_graph_;
-
   bool recompile_queued_;
   bool input_update_queued_;
 

app/widget/timelinewidget/view/timelineviewbase.cpp

@@ -31,8 +31,6 @@ TimelineViewBase::TimelineViewBase(QWidget *parent) :
 
   SetDefaultDragMode(NoDrag);
 
-  setViewportUpdateMode(FullViewportUpdate);
-
   connect(&scene_, SIGNAL(changed(const QList<QRectF>&)), this, SLOT(UpdateSceneRect()));
   connect(Core::instance(), &Core::ToolChanged, this, &TimelineViewBase::ApplicationToolChanged);
 

app/widget/timelinewidget/view/timelineviewblockitem.cpp

@@ -22,12 +22,16 @@
 
 #include <QBrush>
 #include <QCoreApplication>
+#include <QDir>
+#include <QFloat16>
 #include <QGraphicsScene>
 #include <QGraphicsSceneMouseEvent>
 #include <QPainter>
 #include <QStyleOptionGraphicsItem>
 
+#include "audio/sumsamples.h"
 #include "common/qtutils.h"
+#include "config/config.h"
 #include "node/block/transition/transition.h"
 
 TimelineViewBlockItem::TimelineViewBlockItem(QGraphicsItem* parent) :
@@ -97,6 +101,57 @@ void TimelineViewBlockItem::paint(QPainter *painter, const QStyleOptionGraphicsI
       painter->fillRect(rect(), QColor(0, 0, 0, 64));
     }
 
+    // Draw waveform if one is available
+    QFile wave_file(QDir(QDir(Config::Current()["DiskCachePath"].toString()).filePath("waveform")).filePath(QString::number(reinterpret_cast<quintptr>(block_))));
+    if (wave_file.open(QFile::ReadOnly)){
+      painter->setPen(QColor(64, 64, 64));
+
+      QByteArray w = wave_file.readAll();
+
+      // FIXME: Hardcoded channel count
+      int channels = 2;
+
+      const SampleSummer::Sum* samples = reinterpret_cast<const SampleSummer::Sum*>(w.constData());
+      int nb_samples = w.size() / sizeof(SampleSummer::Sum);
+
+      int sample_index, next_sample_index = 0;
+
+      QVector<SampleSummer::Sum> summary;
+      int summary_index = -1;
+
+      int channel_height = rect().height() / channels;
+      int channel_half_height = channel_height / 2;
+
+      for (int i=0;i<rect().width();i++) {
+        sample_index = next_sample_index;
+
+        if (sample_index == nb_samples) {
+          break;
+        }
+
+        next_sample_index = qMin(nb_samples,
+                                 qFloor(static_cast<double>(SampleSummer::kSumSampleRate) * static_cast<double>(i+1) / this->GetScale()) * channels);
+
+        if (summary_index != sample_index) {
+          summary = SampleSummer::ReSumSamples(&samples[sample_index],
+                                               qMax(channels, next_sample_index - sample_index),
+                                               channels);
+          summary_index = sample_index;
+        }
+
+        for (int j=0;j<summary.size();j++) {
+          int channel_mid = channel_height * j + channel_half_height;
+
+          painter->drawLine(i,
+                            channel_mid + qRound(summary.at(j).min * channel_half_height),
+                            i,
+                            channel_mid + qRound(summary.at(j).max * channel_half_height));
+        }
+      }
+
+      wave_file.close();
+    }
+
     painter->setPen(Qt::white);
     painter->drawLine(rect().topLeft(), QPointF(rect().right(), rect().top()));
     painter->drawLine(rect().topLeft(), QPointF(rect().left(), rect().bottom() - 1));