renderer: switched audio rendering over to planar buffers

While planar audio cannot be used for playback, it's much easier for processing.
This should lead to cleaner and easier to write code in the future.

app/audio/sumsamples.cpp

@@ -14,6 +14,21 @@ QVector<SampleSummer::Sum> SampleSummer::SumSamples(const qfloat16 *samples, int
   return SumSamplesInternal<qfloat16>(samples, nb_samples, nb_channels);
 }
 
+QVector<SampleSummer::Sum> SampleSummer::SumSamples(SampleBufferPtr samples, int start_index, int length)
+{
+  QVector<SampleSummer::Sum> summed_samples(samples->audio_params().channel_count());
+
+  int end_index = start_index + length;
+
+  for (int i=start_index;i<end_index;i++) {
+    for (int channel=0;channel<samples->audio_params().channel_count();channel++) {
+      ClampMinMax<float>(summed_samples[channel], samples->data()[channel][i]);
+    }
+  }
+
+  return summed_samples;
+}
+
 QVector<SampleSummer::Sum> SampleSummer::ReSumSamples(const SampleSummer::Sum *samples, int nb_samples, int nb_channels)
 {
   QVector<SampleSummer::Sum> summed_samples(nb_channels);
@@ -47,18 +62,20 @@ QVector<SampleSummer::Sum> SampleSummer::SumSamplesInternal(const T *samples, in
   QVector<SampleSummer::Sum> summed_samples(nb_channels);
 
   for (int i=0;i<nb_samples;i++) {
-    int channel = i%nb_channels;
-
-    float sample = samples[i];
-
-    if (sample < summed_samples[channel].min) {
-      summed_samples[channel].min = sample;
-    }
-
-    if (sample > summed_samples[channel].max) {
-      summed_samples[channel].max = sample;
-    }
+    ClampMinMax<T>(summed_samples[i%nb_channels], samples[i]);
   }
 
   return summed_samples;
 }
+
+template<typename T>
+void SampleSummer::ClampMinMax(SampleSummer::Sum &sum, T value)
+{
+  if (value < sum.min) {
+    sum.min = value;
+  }
+
+  if (value > sum.max) {
+    sum.max = value;
+  }
+}

app/audio/sumsamples.h

@@ -4,6 +4,8 @@
 #include <QFloat16>
 #include <QVector>
 
+#include "codec/samplebuffer.h"
+
 class SampleSummer {
 public:
   struct Sum {
@@ -18,6 +20,7 @@ public:
 
   static QVector<Sum> SumSamples(const float* samples, int nb_samples, int nb_channels);
   static QVector<Sum> SumSamples(const qfloat16* samples, int nb_samples, int nb_channels);
+  static QVector<Sum> SumSamples(SampleBufferPtr samples, int start_index, int length);
 
   static QVector<Sum> ReSumSamples(const SampleSummer::Sum* samples, int nb_samples, int nb_channels);
 
@@ -25,6 +28,9 @@ private:
   template <typename T>
   static QVector<Sum> SumSamplesInternal(const T* samples, int nb_samples, int nb_channels);
 
+  template <typename T>
+  static void ClampMinMax(Sum &sum, T value);
+
 };
 
 #endif // SUMSAMPLES_H

app/codec/CMakeLists.txt

@@ -25,6 +25,8 @@ set(OLIVE_SOURCES
   codec/encoder.cpp
   codec/frame.h
   codec/frame.cpp
+  codec/samplebuffer.h
+  codec/samplebuffer.cpp
   codec/waveinput.h
   codec/waveinput.cpp
   codec/waveoutput.h

app/codec/decoder.cpp

@@ -66,7 +66,7 @@ FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const int &/*divid
   return nullptr;
 }
 
-FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/)
+SampleBufferPtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/)
 {
   return nullptr;
 }

app/codec/decoder.h

@@ -26,6 +26,7 @@
 #include <stdint.h>
 
 #include "codec/frame.h"
+#include "codec/samplebuffer.h"
 #include "codec/waveoutput.h"
 #include "common/constructors.h"
 #include "common/rational.h"
@@ -168,7 +169,7 @@ public:
    * A FramePtr of valid data at this timecode of the requested length or nullptr if there was nothing to retrieve at
    * the provided timecode or the media could not be opened.
    */
-  virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params);
+  virtual SampleBufferPtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params);
 
   virtual bool SupportsVideo();
   virtual bool SupportsAudio();

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -400,9 +400,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
   // We found the frame, we'll return a copy
   if (return_frame) {
     FramePtr copy = Frame::Create();
-    copy->set_width(return_frame->width());
-    copy->set_height(return_frame->height());
-    copy->set_format(return_frame->format());
+    copy->set_video_params(return_frame->video_params());
     copy->set_timestamp(return_frame->timestamp());
     copy->set_sample_aspect_ratio(return_frame->sample_aspect_ratio());
     copy->allocate();
@@ -415,7 +413,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
   return nullptr;
 }
 
-FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params)
+SampleBufferPtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params)
 {
   QMutexLocker locker(&mutex_);
 
@@ -428,29 +426,19 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &
     return nullptr;
   }
 
-  //Conform(params, cancelled);
-
   WaveInput input(GetConformedFilename(params));
 
   if (input.open()) {
     const AudioRenderingParams& input_params = input.params();
 
-    FramePtr audio_frame = Frame::Create();
-    audio_frame->set_audio_params(input_params);
-    audio_frame->set_sample_count(input_params.time_to_samples(length));
-    audio_frame->allocate();
-
-    qint64 actual_read = input.read(input_params.time_to_bytes(timecode),
-                                    audio_frame->data(),
-                                    audio_frame->allocated_size());
-
-    if (actual_read < audio_frame->allocated_size()) {
-      memset(audio_frame->data() + actual_read, 0, audio_frame->allocated_size() - actual_read);
-    }
-
+    // Read bytes from wav
+    QByteArray packed_data = input.read(input_params.time_to_bytes(timecode), input_params.time_to_bytes(length));
     input.close();
 
-    return audio_frame;
+    // Create sample buffer
+    SampleBufferPtr sample_buffer = SampleBuffer::CreateFromPackedData(input_params, packed_data);
+
+    return sample_buffer;
   }
 
   return nullptr;

app/codec/ffmpeg/ffmpegdecoder.h

@@ -55,7 +55,7 @@ public:
   virtual bool Open() override;
   virtual RetrieveState GetRetrieveState(const rational &time) override;
   virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override;
-  virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override;
+  virtual SampleBufferPtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override;
   virtual void Close() override;
 
   virtual QString id() override;

app/codec/ffmpeg/ffmpegframecache.cpp

@@ -85,9 +85,7 @@ Frame* FFmpegFrameCache::Get(const VideoRenderingParams &params)
 
   // Otherwise we'll need to create one
   Frame* f = new Frame();
-  f->set_width(params.width());
-  f->set_height(params.height());
-  f->set_format(params.format());
+  f->set_video_params(params);
   f->allocate();
 
   return f;

app/codec/frame.cpp

@@ -24,10 +24,6 @@
 #include <QtGlobal>
 
 Frame::Frame() :
-  width_(0),
-  height_(0),
-  format_(PixelFormat::PIX_FMT_INVALID),
-  sample_count_(0),
   timestamp_(0),
   sample_aspect_ratio_(1)
 {
@@ -38,24 +34,29 @@ FramePtr Frame::Create()
   return std::make_shared<Frame>();
 }
 
-const int &Frame::width() const
+const VideoRenderingParams &Frame::video_params() const
 {
-  return width_;
+  return params_;
 }
 
-void Frame::set_width(const int &width)
+void Frame::set_video_params(const VideoRenderingParams &params)
 {
-  width_ = width;
+  params_ = params;
+}
+
+const int &Frame::width() const
+{
+  return params_.width();
 }
 
 const int &Frame::height() const
 {
-  return height_;
+  return params_.height();
 }
 
-void Frame::set_height(const int &height)
+const PixelFormat::Format &Frame::format() const
 {
-  height_ = height;
+  return params_.format();
 }
 
 const rational &Frame::sample_aspect_ratio() const
@@ -68,16 +69,6 @@ void Frame::set_sample_aspect_ratio(const rational &aspect_ratio)
   sample_aspect_ratio_ = aspect_ratio;
 }
 
-const AudioRenderingParams &Frame::audio_params() const
-{
-  return audio_params_;
-}
-
-void Frame::set_audio_params(const AudioRenderingParams &params)
-{
-  audio_params_ = params;
-}
-
 const rational &Frame::timestamp() const
 {
   return timestamp_;
@@ -98,31 +89,11 @@ void Frame::set_native_timestamp(const int64_t &timestamp)
   native_timestamp_ = timestamp;
 }
 
-const PixelFormat::Format &Frame::format() const
-{
-  return format_;
-}
-
-void Frame::set_format(const PixelFormat::Format &format)
-{
-  format_ = format;
-}
-
 QByteArray Frame::ToByteArray() const
 {
   return data_;
 }
 
-const int &Frame::sample_count() const
-{
-  return sample_count_;
-}
-
-void Frame::set_sample_count(const int &audio_sample_count)
-{
-  sample_count_ = audio_sample_count;
-}
-
 char *Frame::data()
 {
   return data_.data();
@@ -136,11 +107,11 @@ const char *Frame::const_data() const
 void Frame::allocate()
 {
   // Assume this frame is intended to be a video frame
-  if (width_ > 0 && height_ > 0) {
-    data_.resize(PixelFormat::GetBufferSize(static_cast<PixelFormat::Format>(format_), width_, height_));
-  } else if (sample_count_ > 0) {
-    data_.resize(audio_params_.samples_to_bytes(sample_count_));
+  if (!params_.is_valid()) {
+    qWarning() << "Tried to allocate a frame with invalid parameters";
   }
+
+  data_.resize(PixelFormat::GetBufferSize(params_.format(), params_.width(), params_.height()));
 }
 
 bool Frame::is_allocated() const

app/codec/frame.h

@@ -25,8 +25,8 @@
 #include <QVector>
 
 #include "common/rational.h"
-#include "render/audioparams.h"
 #include "render/pixelformat.h"
+#include "render/videoparams.h"
 
 class Frame;
 using FramePtr = std::shared_ptr<Frame>;
@@ -41,27 +41,16 @@ public:
 
   static FramePtr Create();
 
-  /**
-   * @brief Get frame's width in pixels
-   */
-  const int& width() const;
-  void set_width(const int& width);
+  const VideoRenderingParams& video_params() const;
+  void set_video_params(const VideoRenderingParams& params);
 
-  /**
-   * @brief Get frame's height in pixels
-   */
+  const int& width() const;
   const int& height() const;
-  void set_height(const int& height);
+  const PixelFormat::Format& format() const;
 
   const rational& sample_aspect_ratio() const;
   void set_sample_aspect_ratio(const rational& sample_aspect_ratio);
 
-  const AudioRenderingParams& audio_params() const;
-  void set_audio_params(const AudioRenderingParams& params);
-
-  const int &sample_count() const;
-  void set_sample_count(const int &sample_count);
-
   /**
    * @brief Get frame's timestamp.
    *
@@ -73,16 +62,6 @@ public:
   const int64_t& native_timestamp();
   void set_native_timestamp(const int64_t& timestamp);
 
-  /**
-   * @brief Get frame's format
-   *
-   * @return
-   *
-   * Currently this will either be an olive::PixelFormat (video) or an olive::SampleFormat (audio).
-   */
-  const PixelFormat::Format& format() const;
-  void set_format(const PixelFormat::Format& format);
-
   /**
    * @brief Returns a copy of the data in this frame as a QByteArray
    *
@@ -127,15 +106,7 @@ public:
   int allocated_size() const;
 
 private:
-  int width_;
-
-  int height_;
-
-  PixelFormat::Format format_;
-
-  AudioRenderingParams audio_params_;
-
-  int sample_count_;
+  VideoRenderingParams params_;
 
   QByteArray data_;
 

app/codec/oiio/oiiodecoder.cpp

@@ -183,9 +183,9 @@ FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const int& divider
 
   FramePtr frame = Frame::Create();
 
-  frame->set_width(buffer_->spec().width / divider);
-  frame->set_height(buffer_->spec().height / divider);
-  frame->set_format(pix_fmt_);
+  frame->set_video_params(VideoRenderingParams(buffer_->spec().width / divider,
+                                               buffer_->spec().height / divider,
+                                               pix_fmt_));
   frame->allocate();
 
   if (divider == 1) {

app/codec/samplebuffer.cpp

@@ -0,0 +1,211 @@
+#include "samplebuffer.h"
+
+SampleBuffer::SampleBuffer() :
+  sample_count_per_channel_(0),
+  data_(nullptr)
+{
+}
+
+SampleBuffer::~SampleBuffer()
+{
+  destroy();
+}
+
+SampleBufferPtr SampleBuffer::Create()
+{
+  return std::make_shared<SampleBuffer>();
+}
+
+SampleBufferPtr SampleBuffer::CreateAllocated(const AudioRenderingParams &audio_params, int samples_per_channel)
+{
+  SampleBufferPtr buffer = Create();
+
+  buffer->set_audio_params(audio_params);
+  buffer->set_sample_count_per_channel(samples_per_channel);
+  buffer->allocate();
+
+  return buffer;
+}
+
+SampleBufferPtr SampleBuffer::CreateFromPackedData(const AudioRenderingParams &audio_params, const QByteArray &bytes)
+{
+  if (!audio_params.is_valid()) {
+    qWarning() << "Tried to create from packed data with invalid parameters";
+    return nullptr;
+  }
+
+  int samples_per_channel = audio_params.bytes_to_samples(bytes.size());
+  SampleBufferPtr buffer = CreateAllocated(audio_params, samples_per_channel);
+
+  int total_samples = samples_per_channel * audio_params.channel_count();
+
+  const float* packed_data = reinterpret_cast<const float*>(bytes.constData());
+
+  for (int i=0;i<total_samples;i++) {
+    int channel = i%audio_params.channel_count();
+    int index = i / audio_params.channel_count();
+
+    buffer->data_[channel][index] = packed_data[i];
+  }
+
+  return buffer;
+}
+
+const AudioRenderingParams &SampleBuffer::audio_params() const
+{
+  return audio_params_;
+}
+
+void SampleBuffer::set_audio_params(const AudioRenderingParams &params)
+{
+  if (data_) {
+    qWarning() << "Tried to set parameters on allocated sample buffer";
+    return;
+  }
+
+  audio_params_ = params;
+}
+
+const int &SampleBuffer::sample_count_per_channel() const
+{
+  return sample_count_per_channel_;
+}
+
+void SampleBuffer::set_sample_count_per_channel(const int &sample_count)
+{
+  if (data_) {
+    qWarning() << "Tried to set sample count on allocated sample buffer";
+    return;
+  }
+
+  sample_count_per_channel_ = sample_count;
+}
+
+float **SampleBuffer::data()
+{
+  return data_;
+}
+
+float *SampleBuffer::channel_data(int channel)
+{
+  return data_[channel];
+}
+
+bool SampleBuffer::is_allocated() const
+{
+  return data_;
+}
+
+void SampleBuffer::allocate()
+{
+  if (!audio_params_.is_valid()) {
+    qWarning() << "Tried to allocate sample buffer with invalid audio parameters";
+    return;
+  }
+
+  if (!sample_count_per_channel_) {
+    qWarning() << "Tried to allocate sample buffer with zero sample count";
+    return;
+  }
+
+  if (data_) {
+    qWarning() << "Tried to allocate already allocated sample buffer";
+    return;
+  }
+
+  allocate_sample_buffer(&data_, audio_params_.channel_count(), sample_count_per_channel_);
+}
+
+void SampleBuffer::destroy()
+{
+  destroy_sample_buffer(&data_, audio_params_.channel_count());
+}
+
+void SampleBuffer::reverse()
+{
+  if (!is_allocated()) {
+    qWarning() << "Tried to reverse an unallocated sample buffer";
+    return;
+  }
+
+  int half_nb_sample = sample_count_per_channel_ / 2;
+
+  for (int i=0;i<half_nb_sample;i++) {
+    int opposite_ind = sample_count_per_channel_ - i - 1;
+
+    for (int j=0;j<audio_params_.channel_count();j++) {
+      std::swap(data_[j][i], data_[j][opposite_ind]);
+    }
+  }
+}
+
+void SampleBuffer::speed(double speed)
+{
+  if (!is_allocated()) {
+    qWarning() << "Tried to speed an unallocated sample buffer";
+    return;
+  }
+
+  int max_adjusted_nb_samples = qRound(static_cast<double>(sample_count_per_channel_) * speed);
+
+  float** input_data = data_;
+  float** output_data;
+
+  allocate_sample_buffer(&output_data, audio_params_.channel_count(), max_adjusted_nb_samples);
+
+  for (int i=0;i<max_adjusted_nb_samples;i++) {
+    int input_index = qRound(static_cast<double>(i) * speed);
+
+    for (int j=0;j<audio_params_.channel_count();j++) {
+      output_data[j][i] = input_data[j][input_index];
+    }
+  }
+
+  destroy_sample_buffer(&input_data, audio_params_.channel_count());
+
+  data_ = output_data;
+}
+
+QByteArray SampleBuffer::toPackedData() const
+{
+  QByteArray packed_data;
+
+  if (is_allocated()) {
+    packed_data.resize(audio_params_.samples_to_bytes(sample_count_per_channel_));
+
+    float* output_data = reinterpret_cast<float*>(packed_data.data());
+
+    int output_index = 0;
+
+    for (int j=0;j<sample_count_per_channel_;j++) {
+      for (int i=0;i<audio_params_.channel_count();i++) {
+        output_data[output_index] = data_[i][j];
+
+        output_index++;
+      }
+    }
+  }
+
+  return packed_data;
+}
+
+void SampleBuffer::allocate_sample_buffer(float ***data, int nb_channels, int nb_samples)
+{
+  *data = new float* [nb_channels];
+
+  for (int i=0;i<nb_channels;i++) {
+    (*data)[i] = new float[nb_samples];
+  }
+}
+
+void SampleBuffer::destroy_sample_buffer(float ***data, int nb_channels)
+{
+  if (*data) {
+    for (int i=0;i<nb_channels;i++) {
+      delete [] (*data)[i];
+    }
+
+    delete *data;
+    *data = nullptr;
+  }
+}

app/codec/samplebuffer.h

@@ -0,0 +1,67 @@
+#ifndef SAMPLEBUFFER_H
+#define SAMPLEBUFFER_H
+
+#include <memory>
+
+#include "common/constructors.h"
+#include "render/audioparams.h"
+
+class SampleBuffer;
+using SampleBufferPtr = std::shared_ptr<SampleBuffer>;
+
+/**
+ * @brief A buffer of audio samples
+ *
+ * Audio samples in this structure are always stored in PLANAR (separated by channel). This is done to simplify audio
+ * rendering code. This replaces the old system of using QByteArrays (containing packed audio) and while SampleBuffer
+ * replaces many of those in the rendering/processing side of things, QByteArrays are currently still in use for
+ * playback, including reading to and from the cache.
+ */
+class SampleBuffer
+{
+public:
+  SampleBuffer();
+
+  virtual ~SampleBuffer();
+
+  static SampleBufferPtr Create();
+  static SampleBufferPtr CreateAllocated(const AudioRenderingParams& audio_params, int samples_per_channel);
+  static SampleBufferPtr CreateFromPackedData(const AudioRenderingParams& audio_params, const QByteArray& bytes);
+
+  DISABLE_COPY_MOVE(SampleBuffer)
+
+  const AudioRenderingParams& audio_params() const;
+  void set_audio_params(const AudioRenderingParams& params);
+
+  const int &sample_count_per_channel() const;
+  void set_sample_count_per_channel(const int &sample_count_per_channel);
+
+  float** data();
+  float* channel_data(int channel);
+  float* sample_data(int index);
+
+  bool is_allocated() const;
+  void allocate();
+  void destroy();
+
+  void reverse();
+  void speed(double speed);
+
+  QByteArray toPackedData() const;
+
+private:
+  static void allocate_sample_buffer(float*** data, int nb_channels, int nb_samples);
+
+  static void destroy_sample_buffer(float*** data, int nb_channels);
+
+  AudioRenderingParams audio_params_;
+
+  int sample_count_per_channel_;
+
+  float** data_;
+
+};
+
+Q_DECLARE_METATYPE(SampleBufferPtr)
+
+#endif // SAMPLEBUFFER_H

app/core.cpp

@@ -374,6 +374,7 @@ void Core::DeclareTypesForQt()
   qRegisterMetaType<OpenGLTextureCache::ReferencePtr>();
   qRegisterMetaType<NodeValueTable>();
   qRegisterMetaType<FramePtr>();
+  qRegisterMetaType<SampleBufferPtr>();
   qRegisterMetaType<AudioRenderingParams>();
   qRegisterMetaType<NodeKeyframe::Type>();
   qRegisterMetaType<Decoder::RetrieveState>();

app/dialog/preferences/tabs/preferencesqualitytab.cpp

@@ -4,6 +4,7 @@
 #include <QLabel>
 #include <QVBoxLayout>
 
+#include "audio/sampleformat.h"
 #include "render/colormanager.h"
 #include "render/pixelformat.h"
 

app/node/audio/pan/pan.cpp

@@ -47,7 +47,7 @@ NodeInput *PanNode::ProcessesSamplesFrom() const
   return samples_input_;
 }
 
-void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams &params, const float *input, float *output, int index) const
+void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams &params, const SampleBufferPtr input, SampleBufferPtr output, int index) const
 {
   if (params.channel_count() != 2) {
     // This node currently only works for stereo audio
@@ -56,18 +56,14 @@ void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderi
 
   float pan_val = (*values)[panning_input_].Get(NodeParam::kFloat).toFloat();
 
-  output[index] = input[index];
+  for (int i=0;i<params.channel_count();i++) {
+    output->data()[i][index] = input->data()[i][index];
+  }
 
-  if (index%2 == 0) {
-    // Sample is left channel
   if (pan_val > 0) {
-      output[index] *= (1.0F - pan_val);
-    }
-  } else {
-    // Sample is right channel
-    if (pan_val < 0) {
-      output[index] *= (1.0F - qAbs(pan_val));
-    }
+    output->data()[0][index] *= (1.0F - pan_val);
+  } else if (pan_val < 0) {
+    output->data()[1][index] *= (1.0F - qAbs(pan_val));
   }
 }
 

app/node/audio/pan/pan.h

@@ -17,7 +17,7 @@ public:
 
   virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
   virtual NodeInput* ProcessesSamplesFrom() const override;
-  virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override;
+  virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override;
 
   virtual void Retranslate() override;
 

app/node/audio/volume/volume.cpp

@@ -46,11 +46,13 @@ NodeInput *VolumeNode::ProcessesSamplesFrom() const
   return samples_input_;
 }
 
-void VolumeNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const float* input, float* output, int index) const
+void VolumeNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const
 {
   float volume_val = (*values)[volume_input_].Get(NodeParam::kFloat).toFloat();
 
-  output[index] = input[index] * volume_val;
+  for (int i=0;i<params.channel_count();i++) {
+    output->data()[i][index] = input->data()[i][index] * volume_val;
+  }
 }
 
 void VolumeNode::Retranslate()

app/node/audio/volume/volume.h

@@ -17,7 +17,7 @@ public:
 
   virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
   virtual NodeInput* ProcessesSamplesFrom() const override;
-  virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override;
+  virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override;
 
   virtual void Retranslate() override;
 

app/node/node.cpp

@@ -482,13 +482,8 @@ NodeInput* Node::ProcessesSamplesFrom() const
   return nullptr;
 }
 
-void Node::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams &params, const float *input, float *output, int index) const
+void Node::ProcessSamples(const NodeValueDatabase*, const AudioRenderingParams&, const SampleBufferPtr, SampleBufferPtr, int) const
 {
-  Q_UNUSED(values)
-  Q_UNUSED(params)
-  Q_UNUSED(input)
-  Q_UNUSED(output)
-  Q_UNUSED(index)
 }
 
 NodeParam *Node::GetParameterWithID(const QString &id) const

app/node/node.h

@@ -26,6 +26,7 @@
 #include <QPointF>
 #include <QXmlStreamWriter>
 
+#include "codec/samplebuffer.h"
 #include "common/rational.h"
 #include "common/xmlutils.h"
 #include "node/dependency.h"
@@ -190,7 +191,7 @@ public:
   /**
    * @brief If ProcessesSamples() is true, this is the function that will process them.
    */
-  virtual void ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const float* input, float* output, int index) const;
+  virtual void ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const;
 
   /**
    * @brief Returns the parameter with the specified ID (or nullptr if it doesn't exist)

app/node/param.h

@@ -137,7 +137,7 @@ public:
     /**
      * Audio samples type
      *
-     * Resolves to `QVector4D`.
+     * Resolves to `SampleBufferPtr`.
      */
     kSamples = 0x400,
 

app/render/backend/audio/audiobackend.cpp

@@ -62,7 +62,7 @@ void AudioBackend::ThreadCompletedCache(NodeDependency dep, NodeValueTable data,
   if (job_time == render_job_info_.value(dep.range())) {
     render_job_info_.remove(dep.range());
 
-    QByteArray cached_samples = data.Get(NodeParam::kSamples).toByteArray();
+    QByteArray cached_samples = data.Get(NodeParam::kSamples).value<SampleBufferPtr>()->toPackedData();
 
     int offset = params().time_to_bytes(dep.in());
     int length = params().time_to_bytes(dep.range().length());

app/render/backend/audio/audioworker.cpp

@@ -12,11 +12,10 @@ void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR
   }
 
   if (decoder->HasConformedVersion(audio_params())) {
-    FramePtr frame = nullptr;
-    frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params());
+    SampleBufferPtr frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params());
 
     if (frame) {
-      table->Push(NodeParam::kSamples, frame->ToByteArray());
+      table->Push(NodeParam::kSamples, QVariant::fromValue(frame));
     }
   } else {
     emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params());
@@ -41,10 +40,10 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c
     return;
   }
 
-  QByteArray input_buffer = samples_var.toByteArray();
-  QByteArray output_buffer(input_buffer.size(), 0);
+  SampleBufferPtr input_buffer = samples_var.value<SampleBufferPtr>();
+  SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(input_buffer->audio_params(), input_buffer->sample_count_per_channel());
 
-  int sample_count = input_buffer.size() / audio_params().bytes_per_sample_per_channel();
+  int sample_count = input_buffer->sample_count_per_channel();
 
   // FIXME: Hardcoded float sample format
   for (int i=0;i<sample_count;i++) {
@@ -69,10 +68,10 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c
 
     node->ProcessSamples(&input_params,
                          audio_params(),
-                         reinterpret_cast<const float*>(input_buffer.constData()),
-                         reinterpret_cast<float*>(output_buffer.data()),
+                         input_buffer,
+                         output_buffer,
                          i);
   }
 
-  output_params->Push(NodeParam::kSamples, output_buffer);
+  output_params->Push(NodeParam::kSamples, QVariant::fromValue(output_buffer));
 }

app/render/backend/audiorenderworker.cpp

@@ -35,7 +35,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
   QList<Block*> active_blocks = track->BlocksAtTimeRange(range);
 
   // All these blocks will need to output to a buffer so we create one here
-  QByteArray block_range_buffer(audio_params_.time_to_bytes(range.length()), 0);
+  SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params_, audio_params_.time_to_samples(range.length()));
 
   NodeValueTable merged_table;
 
@@ -46,69 +46,56 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
 
     // Destination buffer
     NodeValueTable table = ProcessNode(NodeDependency(b, range_for_block));
-    QByteArray samples_from_this_block = table.Take(NodeParam::kSamples).toByteArray();
+    QVariant sample_val = table.Take(NodeParam::kSamples);
+
+    if (sample_val.isNull()) {
+      continue;
+    }
+
+    SampleBufferPtr samples_from_this_block = sample_val.value<SampleBufferPtr>();
 
-    if (!samples_from_this_block.isEmpty()) {
     // Stretch samples here
     rational abs_speed = qAbs(b->speed());
 
     if (abs_speed != 1) {
-        QByteArray speed_adjusted_samples;
-
-        double clip_speed = abs_speed.toDouble();
-
-        int sample_count = audio_params_.bytes_to_samples(samples_from_this_block.size());
-
-        for (double i=0;i<sample_count;i+=clip_speed) {
-          int sample_index = qFloor(i);
-          int byte_index = audio_params_.samples_to_bytes(sample_index);
-
-          QByteArray sample_at_this_index = samples_from_this_block.mid(byte_index, audio_params_.samples_to_bytes(1));
-          speed_adjusted_samples.append(sample_at_this_index);
-        }
-
-        samples_from_this_block = speed_adjusted_samples;
+      samples_from_this_block->speed(abs_speed.toDouble());
     }
 
     if (b->is_reversed()) {
       // Reverse the audio buffer
-        AudioManager::ReverseBuffer(samples_from_this_block.data(), samples_from_this_block.size(), audio_params_.samples_to_bytes(1));
+      samples_from_this_block->reverse();
     }
 
-      int destination_offset = audio_params_.time_to_bytes(range_for_block.in() - range.in());
-      int actual_copy_size = qMin(audio_params_.time_to_bytes(range_for_block.length()), samples_from_this_block.size());
+    int destination_offset = audio_params_.time_to_samples(range_for_block.in() - range.in()) * sizeof(float);
+    int max_dest_sz = audio_params_.time_to_samples(range_for_block.length()) * sizeof(float);
+    int input_sz = samples_from_this_block->sample_count_per_channel() * sizeof(float);
+    int actual_copy_size = qMin(max_dest_sz, input_sz);
 
-      if (destination_offset < 0
-          || destination_offset + actual_copy_size > block_range_buffer.size()) {
-        qCritical() << "Tried to copy audio beyond the size of the destination buffer";
-        qCritical() << "  Destination size:" << block_range_buffer.size();
-        qCritical() << "  Destination offset:" << destination_offset;
-        qCritical() << "  Copy size:" << actual_copy_size;
-        abort();
-      } else {
-        memcpy(block_range_buffer.data()+destination_offset,
-               samples_from_this_block.data(),
+    for (int i=0;i<audio_params_.channel_count();i++) {
+      memcpy(block_range_buffer->data()[i] + destination_offset,
+             samples_from_this_block->data()[i],
              actual_copy_size);
 
+      if (input_sz < max_dest_sz) {
+        memset(block_range_buffer->data()[i] + destination_offset + input_sz, 0, max_dest_sz - input_sz);
+      }
+    }
+
+    {
       // Save waveform to file
       Block* src_block = static_cast<Block*>(copy_map_->value(b));
       QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString());
       QDir waveform_loc = local_appdata_dir.filePath("waveform");
       waveform_loc.mkpath(".");
       QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast<quintptr>(src_block))));
-        wave_file.open(QFile::ReadWrite);
 
-        // FIXME: Assumes 32-bit float
-        float* flt_samples = reinterpret_cast<float*>(samples_from_this_block.data());
-        int nb_sample = samples_from_this_block.size() / sizeof(float);
+      if (wave_file.open(QFile::ReadWrite)) {
+        // We use S32 as a size-compatible substitute for SampleSummer::Sum which is 4 bytes in size
+        AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S32);
+        int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate());
 
-        // FIXME: Hardcoded sample rate
-        // We use S16 here as a size-compatible substitute for qfloat16/half
-        AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S16);
-        int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()) * waveform_params.channel_count();
-
-        qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()) * 2;
-        qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()) * 2;
+        qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in());
+        qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length());
         qint64 end_offset = start_offset + length_offset;
 
         if (wave_file.size() < end_offset) {
@@ -117,10 +104,10 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
 
         wave_file.seek(start_offset);
 
-        for (int i=0;i<nb_sample;i+=chunk_size) {
-          QVector<SampleSummer::Sum> summary = SampleSummer::SumSamples(&flt_samples[i],
-                                                                        qMin(chunk_size, nb_sample - i),
-                                                                        audio_params().channel_count());
+        for (int i=0;i<samples_from_this_block->sample_count_per_channel();i+=chunk_size) {
+          QVector<SampleSummer::Sum> summary = SampleSummer::SumSamples(samples_from_this_block,
+                                                                        i,
+                                                                        qMin(chunk_size, samples_from_this_block->sample_count_per_channel() - i));
 
           wave_file.write(reinterpret_cast<const char*>(summary.constData()),
                           summary.size() * sizeof(SampleSummer::Sum));
@@ -137,7 +124,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
     NodeValueTable::Merge({merged_table, table});
   }
 
-  merged_table.Push(NodeParam::kSamples, block_range_buffer);
+  merged_table.Push(NodeParam::kSamples, QVariant::fromValue(block_range_buffer));
 
   return merged_table;
 }

app/render/backend/videorenderworker.cpp

@@ -317,9 +317,7 @@ void VideoRenderWorker::Download(const rational& time, QVariant texture, QString
   } else {
 
     FramePtr frame = Frame::Create();
-    frame->set_width(video_params().width());
-    frame->set_height(video_params().height());
-    frame->set_format(video_params().format());
+    frame->set_video_params(video_params());
     frame->allocate();
 
     TextureToBuffer(texture, frame->data());

app/render/pixelformat.cpp

@@ -237,10 +237,8 @@ FramePtr PixelFormat::ConvertPixelFormat(FramePtr frame, const PixelFormat::Form
   FramePtr converted = Frame::Create();
 
   // Copy parameters
-  converted->set_width(frame->width());
-  converted->set_height(frame->height());
+  converted->set_video_params(frame->video_params());
   converted->set_timestamp(frame->timestamp());
-  converted->set_format(dest_format);
   converted->allocate();
 
   OIIO::ImageBuf src(OIIO::ImageSpec(frame->width(), frame->height(), ChannelCount(frame->format()), GetOIIOTypeDesc(frame->format())), frame->data());

app/render/videoparams.cpp

@@ -34,6 +34,12 @@ VideoRenderingParams::VideoRenderingParams() :
 {
 }
 
+VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const PixelFormat::Format &format) :
+  VideoParams(width, height, rational()),
+  format_(format)
+{
+}
+
 VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const rational &time_base, const PixelFormat::Format &format, const RenderMode::Mode& mode, const int &divider) :
   VideoParams(width, height, time_base),
   format_(format),

app/render/videoparams.h

@@ -25,6 +25,7 @@ private:
 class VideoRenderingParams : public VideoParams {
 public:
   VideoRenderingParams();
+  VideoRenderingParams(const int& width, const int& height, const PixelFormat::Format& format);
   VideoRenderingParams(const int& width, const int& height, const rational& time_base, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1);
   VideoRenderingParams(const VideoParams& params, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1);
 

app/widget/viewer/viewerglwidget.cpp

@@ -101,9 +101,7 @@ void ViewerGLWidget::SetImage(const QString &fn)
         load_buffer_.destroy();
         texture_.Destroy();
 
-        load_buffer_.set_width(input->spec().width);
-        load_buffer_.set_height(input->spec().height);
-        load_buffer_.set_format(image_format);
+        load_buffer_.set_video_params(VideoRenderingParams(input->spec().width, input->spec().height, image_format));
         load_buffer_.allocate();
 
         texture_.Create(context(), input->spec().width, input->spec().height, image_format);