renderer: switched audio rendering over to planar buffers

While planar audio cannot be used for playback, it's much easier for processing.
This should lead to cleaner and easier to write code in the future.

app/codec/CMakeLists.txt

@@ -25,6 +25,8 @@ set(OLIVE_SOURCES
   codec/encoder.cpp
   codec/frame.h
   codec/frame.cpp
+  codec/samplebuffer.h
+  codec/samplebuffer.cpp
   codec/waveinput.h
   codec/waveinput.cpp
   codec/waveoutput.h

app/codec/decoder.cpp

@@ -66,7 +66,7 @@ FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const int &/*divid
   return nullptr;
 }
 
-FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/)
+SampleBufferPtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/)
 {
   return nullptr;
 }

app/codec/decoder.h

@@ -26,6 +26,7 @@
 #include <stdint.h>
 
 #include "codec/frame.h"
+#include "codec/samplebuffer.h"
 #include "codec/waveoutput.h"
 #include "common/constructors.h"
 #include "common/rational.h"
@@ -168,7 +169,7 @@ public:
    * A FramePtr of valid data at this timecode of the requested length or nullptr if there was nothing to retrieve at
    * the provided timecode or the media could not be opened.
    */
-  virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params);
+  virtual SampleBufferPtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params);
 
   virtual bool SupportsVideo();
   virtual bool SupportsAudio();

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -400,9 +400,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
   // We found the frame, we'll return a copy
   if (return_frame) {
     FramePtr copy = Frame::Create();
-    copy->set_width(return_frame->width());
-    copy->set_height(return_frame->height());
-    copy->set_format(return_frame->format());
+    copy->set_video_params(return_frame->video_params());
     copy->set_timestamp(return_frame->timestamp());
     copy->set_sample_aspect_ratio(return_frame->sample_aspect_ratio());
     copy->allocate();
@@ -415,7 +413,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
   return nullptr;
 }
 
-FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params)
+SampleBufferPtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams &params)
 {
   QMutexLocker locker(&mutex_);
 
@@ -428,29 +426,19 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &
     return nullptr;
   }
 
-  //Conform(params, cancelled);
-
   WaveInput input(GetConformedFilename(params));
 
   if (input.open()) {
     const AudioRenderingParams& input_params = input.params();
 
-    FramePtr audio_frame = Frame::Create();
-    audio_frame->set_audio_params(input_params);
-    audio_frame->set_sample_count(input_params.time_to_samples(length));
-    audio_frame->allocate();
-
-    qint64 actual_read = input.read(input_params.time_to_bytes(timecode),
-                                    audio_frame->data(),
-                                    audio_frame->allocated_size());
-
-    if (actual_read < audio_frame->allocated_size()) {
-      memset(audio_frame->data() + actual_read, 0, audio_frame->allocated_size() - actual_read);
-    }
-
+    // Read bytes from wav
+    QByteArray packed_data = input.read(input_params.time_to_bytes(timecode), input_params.time_to_bytes(length));
     input.close();
 
-    return audio_frame;
+    // Create sample buffer
+    SampleBufferPtr sample_buffer = SampleBuffer::CreateFromPackedData(input_params, packed_data);
+
+    return sample_buffer;
   }
 
   return nullptr;

app/codec/ffmpeg/ffmpegdecoder.h

@@ -55,7 +55,7 @@ public:
   virtual bool Open() override;
   virtual RetrieveState GetRetrieveState(const rational &time) override;
   virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override;
-  virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override;
+  virtual SampleBufferPtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override;
   virtual void Close() override;
 
   virtual QString id() override;

app/codec/ffmpeg/ffmpegframecache.cpp

@@ -85,9 +85,7 @@ Frame* FFmpegFrameCache::Get(const VideoRenderingParams &params)
 
   // Otherwise we'll need to create one
   Frame* f = new Frame();
-  f->set_width(params.width());
-  f->set_height(params.height());
-  f->set_format(params.format());
+  f->set_video_params(params);
   f->allocate();
 
   return f;

app/codec/frame.cpp

@@ -24,10 +24,6 @@
 #include <QtGlobal>
 
 Frame::Frame() :
-  width_(0),
-  height_(0),
-  format_(PixelFormat::PIX_FMT_INVALID),
-  sample_count_(0),
   timestamp_(0),
   sample_aspect_ratio_(1)
 {
@@ -38,24 +34,29 @@ FramePtr Frame::Create()
   return std::make_shared<Frame>();
 }
 
-const int &Frame::width() const
+const VideoRenderingParams &Frame::video_params() const
 {
-  return width_;
+  return params_;
 }
 
-void Frame::set_width(const int &width)
+void Frame::set_video_params(const VideoRenderingParams &params)
 {
-  width_ = width;
+  params_ = params;
+}
+
+const int &Frame::width() const
+{
+  return params_.width();
 }
 
 const int &Frame::height() const
 {
-  return height_;
+  return params_.height();
 }
 
-void Frame::set_height(const int &height)
+const PixelFormat::Format &Frame::format() const
 {
-  height_ = height;
+  return params_.format();
 }
 
 const rational &Frame::sample_aspect_ratio() const
@@ -68,16 +69,6 @@ void Frame::set_sample_aspect_ratio(const rational &aspect_ratio)
   sample_aspect_ratio_ = aspect_ratio;
 }
 
-const AudioRenderingParams &Frame::audio_params() const
-{
-  return audio_params_;
-}
-
-void Frame::set_audio_params(const AudioRenderingParams &params)
-{
-  audio_params_ = params;
-}
-
 const rational &Frame::timestamp() const
 {
   return timestamp_;
@@ -98,31 +89,11 @@ void Frame::set_native_timestamp(const int64_t &timestamp)
   native_timestamp_ = timestamp;
 }
 
-const PixelFormat::Format &Frame::format() const
-{
-  return format_;
-}
-
-void Frame::set_format(const PixelFormat::Format &format)
-{
-  format_ = format;
-}
-
 QByteArray Frame::ToByteArray() const
 {
   return data_;
 }
 
-const int &Frame::sample_count() const
-{
-  return sample_count_;
-}
-
-void Frame::set_sample_count(const int &audio_sample_count)
-{
-  sample_count_ = audio_sample_count;
-}
-
 char *Frame::data()
 {
   return data_.data();
@@ -136,11 +107,11 @@ const char *Frame::const_data() const
 void Frame::allocate()
 {
   // Assume this frame is intended to be a video frame
-  if (width_ > 0 && height_ > 0) {
-    data_.resize(PixelFormat::GetBufferSize(static_cast<PixelFormat::Format>(format_), width_, height_));
-  } else if (sample_count_ > 0) {
-    data_.resize(audio_params_.samples_to_bytes(sample_count_));
+  if (!params_.is_valid()) {
+    qWarning() << "Tried to allocate a frame with invalid parameters";
   }
+
+  data_.resize(PixelFormat::GetBufferSize(params_.format(), params_.width(), params_.height()));
 }
 
 bool Frame::is_allocated() const

app/codec/frame.h

@@ -25,8 +25,8 @@
 #include <QVector>
 
 #include "common/rational.h"
-#include "render/audioparams.h"
 #include "render/pixelformat.h"
+#include "render/videoparams.h"
 
 class Frame;
 using FramePtr = std::shared_ptr<Frame>;
@@ -41,27 +41,16 @@ public:
 
   static FramePtr Create();
 
-  /**
-   * @brief Get frame's width in pixels
-   */
-  const int& width() const;
-  void set_width(const int& width);
+  const VideoRenderingParams& video_params() const;
+  void set_video_params(const VideoRenderingParams& params);
 
-  /**
-   * @brief Get frame's height in pixels
-   */
+  const int& width() const;
   const int& height() const;
-  void set_height(const int& height);
+  const PixelFormat::Format& format() const;
 
   const rational& sample_aspect_ratio() const;
   void set_sample_aspect_ratio(const rational& sample_aspect_ratio);
 
-  const AudioRenderingParams& audio_params() const;
-  void set_audio_params(const AudioRenderingParams& params);
-
-  const int &sample_count() const;
-  void set_sample_count(const int &sample_count);
-
   /**
    * @brief Get frame's timestamp.
    *
@@ -73,16 +62,6 @@ public:
   const int64_t& native_timestamp();
   void set_native_timestamp(const int64_t& timestamp);
 
-  /**
-   * @brief Get frame's format
-   *
-   * @return
-   *
-   * Currently this will either be an olive::PixelFormat (video) or an olive::SampleFormat (audio).
-   */
-  const PixelFormat::Format& format() const;
-  void set_format(const PixelFormat::Format& format);
-
   /**
    * @brief Returns a copy of the data in this frame as a QByteArray
    *
@@ -127,15 +106,7 @@ public:
   int allocated_size() const;
 
 private:
-  int width_;
-
-  int height_;
-
-  PixelFormat::Format format_;
-
-  AudioRenderingParams audio_params_;
-
-  int sample_count_;
+  VideoRenderingParams params_;
 
   QByteArray data_;
 

app/codec/oiio/oiiodecoder.cpp

@@ -183,9 +183,9 @@ FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const int& divider
 
   FramePtr frame = Frame::Create();
 
-  frame->set_width(buffer_->spec().width / divider);
-  frame->set_height(buffer_->spec().height / divider);
-  frame->set_format(pix_fmt_);
+  frame->set_video_params(VideoRenderingParams(buffer_->spec().width / divider,
+                                               buffer_->spec().height / divider,
+                                               pix_fmt_));
   frame->allocate();
 
   if (divider == 1) {

app/codec/samplebuffer.cpp

@@ -0,0 +1,211 @@
+#include "samplebuffer.h"
+
+SampleBuffer::SampleBuffer() :
+  sample_count_per_channel_(0),
+  data_(nullptr)
+{
+}
+
+SampleBuffer::~SampleBuffer()
+{
+  destroy();
+}
+
+SampleBufferPtr SampleBuffer::Create()
+{
+  return std::make_shared<SampleBuffer>();
+}
+
+SampleBufferPtr SampleBuffer::CreateAllocated(const AudioRenderingParams &audio_params, int samples_per_channel)
+{
+  SampleBufferPtr buffer = Create();
+
+  buffer->set_audio_params(audio_params);
+  buffer->set_sample_count_per_channel(samples_per_channel);
+  buffer->allocate();
+
+  return buffer;
+}
+
+SampleBufferPtr SampleBuffer::CreateFromPackedData(const AudioRenderingParams &audio_params, const QByteArray &bytes)
+{
+  if (!audio_params.is_valid()) {
+    qWarning() << "Tried to create from packed data with invalid parameters";
+    return nullptr;
+  }
+
+  int samples_per_channel = audio_params.bytes_to_samples(bytes.size());
+  SampleBufferPtr buffer = CreateAllocated(audio_params, samples_per_channel);
+
+  int total_samples = samples_per_channel * audio_params.channel_count();
+
+  const float* packed_data = reinterpret_cast<const float*>(bytes.constData());
+
+  for (int i=0;i<total_samples;i++) {
+    int channel = i%audio_params.channel_count();
+    int index = i / audio_params.channel_count();
+
+    buffer->data_[channel][index] = packed_data[i];
+  }
+
+  return buffer;
+}
+
+const AudioRenderingParams &SampleBuffer::audio_params() const
+{
+  return audio_params_;
+}
+
+void SampleBuffer::set_audio_params(const AudioRenderingParams &params)
+{
+  if (data_) {
+    qWarning() << "Tried to set parameters on allocated sample buffer";
+    return;
+  }
+
+  audio_params_ = params;
+}
+
+const int &SampleBuffer::sample_count_per_channel() const
+{
+  return sample_count_per_channel_;
+}
+
+void SampleBuffer::set_sample_count_per_channel(const int &sample_count)
+{
+  if (data_) {
+    qWarning() << "Tried to set sample count on allocated sample buffer";
+    return;
+  }
+
+  sample_count_per_channel_ = sample_count;
+}
+
+float **SampleBuffer::data()
+{
+  return data_;
+}
+
+float *SampleBuffer::channel_data(int channel)
+{
+  return data_[channel];
+}
+
+bool SampleBuffer::is_allocated() const
+{
+  return data_;
+}
+
+void SampleBuffer::allocate()
+{
+  if (!audio_params_.is_valid()) {
+    qWarning() << "Tried to allocate sample buffer with invalid audio parameters";
+    return;
+  }
+
+  if (!sample_count_per_channel_) {
+    qWarning() << "Tried to allocate sample buffer with zero sample count";
+    return;
+  }
+
+  if (data_) {
+    qWarning() << "Tried to allocate already allocated sample buffer";
+    return;
+  }
+
+  allocate_sample_buffer(&data_, audio_params_.channel_count(), sample_count_per_channel_);
+}
+
+void SampleBuffer::destroy()
+{
+  destroy_sample_buffer(&data_, audio_params_.channel_count());
+}
+
+void SampleBuffer::reverse()
+{
+  if (!is_allocated()) {
+    qWarning() << "Tried to reverse an unallocated sample buffer";
+    return;
+  }
+
+  int half_nb_sample = sample_count_per_channel_ / 2;
+
+  for (int i=0;i<half_nb_sample;i++) {
+    int opposite_ind = sample_count_per_channel_ - i - 1;
+
+    for (int j=0;j<audio_params_.channel_count();j++) {
+      std::swap(data_[j][i], data_[j][opposite_ind]);
+    }
+  }
+}
+
+void SampleBuffer::speed(double speed)
+{
+  if (!is_allocated()) {
+    qWarning() << "Tried to speed an unallocated sample buffer";
+    return;
+  }
+
+  int max_adjusted_nb_samples = qRound(static_cast<double>(sample_count_per_channel_) * speed);
+
+  float** input_data = data_;
+  float** output_data;
+
+  allocate_sample_buffer(&output_data, audio_params_.channel_count(), max_adjusted_nb_samples);
+
+  for (int i=0;i<max_adjusted_nb_samples;i++) {
+    int input_index = qRound(static_cast<double>(i) * speed);
+
+    for (int j=0;j<audio_params_.channel_count();j++) {
+      output_data[j][i] = input_data[j][input_index];
+    }
+  }
+
+  destroy_sample_buffer(&input_data, audio_params_.channel_count());
+
+  data_ = output_data;
+}
+
+QByteArray SampleBuffer::toPackedData() const
+{
+  QByteArray packed_data;
+
+  if (is_allocated()) {
+    packed_data.resize(audio_params_.samples_to_bytes(sample_count_per_channel_));
+
+    float* output_data = reinterpret_cast<float*>(packed_data.data());
+
+    int output_index = 0;
+
+    for (int j=0;j<sample_count_per_channel_;j++) {
+      for (int i=0;i<audio_params_.channel_count();i++) {
+        output_data[output_index] = data_[i][j];
+
+        output_index++;
+      }
+    }
+  }
+
+  return packed_data;
+}
+
+void SampleBuffer::allocate_sample_buffer(float ***data, int nb_channels, int nb_samples)
+{
+  *data = new float* [nb_channels];
+
+  for (int i=0;i<nb_channels;i++) {
+    (*data)[i] = new float[nb_samples];
+  }
+}
+
+void SampleBuffer::destroy_sample_buffer(float ***data, int nb_channels)
+{
+  if (*data) {
+    for (int i=0;i<nb_channels;i++) {
+      delete [] (*data)[i];
+    }
+
+    delete *data;
+    *data = nullptr;
+  }
+}

app/codec/samplebuffer.h

@@ -0,0 +1,67 @@
+#ifndef SAMPLEBUFFER_H
+#define SAMPLEBUFFER_H
+
+#include <memory>
+
+#include "common/constructors.h"
+#include "render/audioparams.h"
+
+class SampleBuffer;
+using SampleBufferPtr = std::shared_ptr<SampleBuffer>;
+
+/**
+ * @brief A buffer of audio samples
+ *
+ * Audio samples in this structure are always stored in PLANAR (separated by channel). This is done to simplify audio
+ * rendering code. This replaces the old system of using QByteArrays (containing packed audio) and while SampleBuffer
+ * replaces many of those in the rendering/processing side of things, QByteArrays are currently still in use for
+ * playback, including reading to and from the cache.
+ */
+class SampleBuffer
+{
+public:
+  SampleBuffer();
+
+  virtual ~SampleBuffer();
+
+  static SampleBufferPtr Create();
+  static SampleBufferPtr CreateAllocated(const AudioRenderingParams& audio_params, int samples_per_channel);
+  static SampleBufferPtr CreateFromPackedData(const AudioRenderingParams& audio_params, const QByteArray& bytes);
+
+  DISABLE_COPY_MOVE(SampleBuffer)
+
+  const AudioRenderingParams& audio_params() const;
+  void set_audio_params(const AudioRenderingParams& params);
+
+  const int &sample_count_per_channel() const;
+  void set_sample_count_per_channel(const int &sample_count_per_channel);
+
+  float** data();
+  float* channel_data(int channel);
+  float* sample_data(int index);
+
+  bool is_allocated() const;
+  void allocate();
+  void destroy();
+
+  void reverse();
+  void speed(double speed);
+
+  QByteArray toPackedData() const;
+
+private:
+  static void allocate_sample_buffer(float*** data, int nb_channels, int nb_samples);
+
+  static void destroy_sample_buffer(float*** data, int nb_channels);
+
+  AudioRenderingParams audio_params_;
+
+  int sample_count_per_channel_;
+
+  float** data_;
+
+};
+
+Q_DECLARE_METATYPE(SampleBufferPtr)
+
+#endif // SAMPLEBUFFER_H