correctly affect speed/reversing when shuttling audio

app/audio/CMakeLists.txt

@@ -26,5 +26,7 @@ set(OLIVE_SOURCES
   audio/outputmanager.cpp
   audio/sampleformat.h
   audio/sampleformat.cpp
+  audio/tempoprocessor.h
+  audio/tempoprocessor.cpp
   PARENT_SCOPE
 )

app/audio/audiomanager.cpp

@@ -80,9 +80,9 @@ void AudioManager::PushToOutput(const QByteArray &samples)
   output_manager_.Push(samples);
 }
 
-void AudioManager::StartOutput(QIODevice *device)
+void AudioManager::StartOutput(QIODevice *device, int playback_speed)
 {
-  output_manager_.PullFromDevice(device);
+  output_manager_.PullFromDevice(device, playback_speed);
 }
 
 void AudioManager::StopOutput()
@@ -147,6 +147,8 @@ void AudioManager::SetOutputParams(const AudioRenderingParams &params)
   if (output_params_ != params) {
     output_params_ = params;
 
+    output_manager_.SetParameters(params);
+
     // Refresh output device
     SetOutputDevice(output_device_info_);
   }
@@ -168,6 +170,24 @@ const QList<QAudioDeviceInfo> &AudioManager::ListOutputDevices()
   return output_devices_;
 }
 
+void AudioManager::ReverseBuffer(char *buffer, int buffer_size, int sample_size)
+{
+  int half_buffer_sz = buffer_size / 2;
+  char* temp_buffer = new char[sample_size];
+
+  for (int src_index=0;src_index<half_buffer_sz;src_index+=sample_size) {
+    char* src_ptr = buffer + src_index;
+    char* dst_ptr = buffer + buffer_size - sample_size - src_index;
+
+    // Simple swap
+    memcpy(temp_buffer, src_ptr, static_cast<size_t>(sample_size));
+    memcpy(src_ptr, dst_ptr, static_cast<size_t>(sample_size));
+    memcpy(dst_ptr, temp_buffer, static_cast<size_t>(sample_size));
+  }
+
+  delete [] temp_buffer;
+}
+
 AudioManager::AudioManager() :
   input_(nullptr),
   input_file_(nullptr),

app/audio/audiomanager.h

@@ -79,7 +79,7 @@ public:
    *
    * This takes ownership of the QIODevice and will delete it when StopOutput() is called
    */
-  void StartOutput(QIODevice* device);
+  void StartOutput(QIODevice* device, int playback_speed);
 
   /**
    * @brief Stop audio output immediately
@@ -95,6 +95,8 @@ public:
   const QList<QAudioDeviceInfo>& ListInputDevices();
   const QList<QAudioDeviceInfo>& ListOutputDevices();
 
+  static void ReverseBuffer(char* buffer, int size, int resolution);
+
 signals:
   void DeviceListReady();
 

app/audio/outputdeviceproxy.cpp

@@ -1,5 +1,6 @@
 #include "outputdeviceproxy.h"
 
+#include "audiomanager.h"
 #include "bufferaverage.h"
 
 AudioOutputDeviceProxy::AudioOutputDeviceProxy() :
@@ -8,9 +9,24 @@ AudioOutputDeviceProxy::AudioOutputDeviceProxy() :
 {
 }
 
-void AudioOutputDeviceProxy::SetDevice(QIODevice *device)
+void AudioOutputDeviceProxy::SetParameters(const AudioRenderingParams &params)
+{
+  params_ = params;
+}
+
+void AudioOutputDeviceProxy::SetDevice(QIODevice *device, int playback_speed)
 {
   device_ = device;
+
+  if (!device_->isOpen()) {
+    device_->open(QIODevice::ReadOnly);
+  }
+
+  playback_speed_ = playback_speed;
+
+  if (qAbs(playback_speed_) != 1) {
+    tempo_processor_.Open(params_, qAbs(playback_speed_));
+  }
 }
 
 void AudioOutputDeviceProxy::SetSendAverages(bool send)
@@ -23,14 +39,39 @@ void AudioOutputDeviceProxy::close()
   QIODevice::close();
 
   device_->close();
+
+  if (tempo_processor_.IsOpen()) {
+    tempo_processor_.Close();
+  }
 }
 
 qint64 AudioOutputDeviceProxy::readData(char *data, qint64 maxlen)
 {
   if (device_) {
-    qint64 read_count = device_->read(data, maxlen);
 
-    if (send_averages_ && read_count > 0) {
+    qint64 read_count;
+
+
+
+    if (tempo_processor_.IsOpen()) {
+
+      while ((read_count = tempo_processor_.Pull(data, static_cast<int>(maxlen))) == 0) {
+        int dev_read = static_cast<int>(ReverseAwareRead(data, maxlen));
+
+        if (!dev_read) {
+          break;
+        }
+
+        tempo_processor_.Push(data, dev_read);
+      }
+
+    } else {
+      // If we aren't doing any tempo processing, simply passthrough the read signal
+      read_count = ReverseAwareRead(data, maxlen);
+    }
+
+    // If we read any
+    if (read_count > 0 && send_averages_) {
       emit ProcessedAverages(AudioBufferAverage::ProcessAverages(data, static_cast<int>(read_count)));
     }
 
@@ -44,3 +85,32 @@ qint64 AudioOutputDeviceProxy::writeData(const char *data, qint64 maxSize)
 {
   return -1;
 }
+
+qint64 AudioOutputDeviceProxy::ReverseAwareRead(char *data, qint64 maxlen)
+{
+  qint64 new_pos;
+
+  if (playback_speed_ < 0) {
+    // If we're reversing, we'll seek back by maxlen bytes before we read
+    new_pos = device_->pos() - maxlen;
+
+    if (new_pos < 0) {
+      maxlen = device_->pos();
+
+      new_pos = 0;
+    }
+
+    device_->seek(new_pos);
+  }
+
+  qint64 read_count = device_->read(data, maxlen);
+
+  if (playback_speed_ < 0) {
+    device_->seek(new_pos);
+
+    // Reverse the samples here
+    AudioManager::ReverseBuffer(data, static_cast<int>(read_count), params_.samples_to_bytes(1));
+  }
+
+  return read_count;
+}

app/audio/outputdeviceproxy.h

@@ -3,13 +3,17 @@
 
 #include <QIODevice>
 
+#include "tempoprocessor.h"
+
 class AudioOutputDeviceProxy : public QIODevice
 {
   Q_OBJECT
 public:
   AudioOutputDeviceProxy();
 
-  void SetDevice(QIODevice* device);
+  void SetParameters(const AudioRenderingParams& params);
+
+  void SetDevice(QIODevice* device, int playback_speed);
 
   void SetSendAverages(bool send);
 
@@ -24,10 +28,18 @@ protected:
   virtual qint64 writeData(const char *data, qint64 maxSize) override;
 
 private:
+  qint64 ReverseAwareRead(char* data, qint64 maxlen);
+
   QIODevice* device_;
 
+  TempoProcessor tempo_processor_;
+
   bool send_averages_;
 
+  AudioRenderingParams params_;
+
+  int playback_speed_;
+
 };
 
 #endif // AUDIOOUTPUTDEVICEPROXY_H

app/audio/outputmanager.cpp

@@ -70,7 +70,12 @@ void AudioOutputManager::ResetToPushMode()
   }
 }
 
-void AudioOutputManager::PullFromDevice(QIODevice *device)
+void AudioOutputManager::SetParameters(const AudioRenderingParams &params)
+{
+  device_proxy_.SetParameters(params);
+}
+
+void AudioOutputManager::PullFromDevice(QIODevice *device, int playback_speed)
 {
   if (!output_ || !device) {
     return;
@@ -82,7 +87,7 @@ void AudioOutputManager::PullFromDevice(QIODevice *device)
   pushed_samples_.clear();
 
   // Pull from the device
-  device_proxy_.SetDevice(device);
+  device_proxy_.SetDevice(device, playback_speed);
   device_proxy_.open(QIODevice::ReadOnly);
   output_->start(&device_proxy_);
 }

app/audio/outputmanager.h

@@ -51,10 +51,12 @@ public:
    * This will clear any pushed samples or QIODevices currently being read and will start reading from this next time
    * the audio output requests data.
    */
-  void PullFromDevice(QIODevice* device);
+  void PullFromDevice(QIODevice* device, int playback_speed);
 
   void ResetToPushMode();
 
+  void SetParameters(const AudioRenderingParams& params);
+
 signals:
   /**
    * @brief Signal emitted when samples are sent to the output device

app/audio/tempoprocessor.cpp

@@ -0,0 +1,252 @@
+#include "tempoprocessor.h"
+
+extern "C" {
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+#include <libavutil/opt.h>
+}
+
+#include <QDebug>
+
+#include "codec/ffmpeg/ffmpegcommon.h"
+
+TempoProcessor::TempoProcessor() :
+  filter_graph_(nullptr),
+  buffersrc_ctx_(nullptr),
+  buffersink_ctx_(nullptr),
+  open_(false)
+{
+
+}
+
+bool TempoProcessor::IsOpen() const
+{
+  return open_;
+}
+
+const double &TempoProcessor::GetSpeed() const
+{
+  return speed_;
+}
+
+bool TempoProcessor::Open(const AudioRenderingParams &params, const double& speed)
+{
+  if (open_) {
+    return true;
+  }
+
+  params_ = params;
+  speed_ = speed;
+
+  // Create AVFilterGraph instance
+  filter_graph_ = avfilter_graph_alloc();
+  if (!filter_graph_) {
+    qCritical() << "Failed to create AVFilterGraph";
+    Close();
+    return false;
+  }
+
+  // Set up audio buffer args
+  char filter_args[200];
+  snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64,
+           1,
+           params_.sample_rate(),
+           params_.sample_rate(),
+           FFmpegCommon::GetFFmpegSampleFormat(params_.format()),
+           params.channel_layout());
+
+  // Create buffer and buffersink
+  if (avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_) < 0) {
+    qCritical() << "Failed to create audio buffer source";
+    Close();
+    return false;
+  }
+
+  if (avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_) < 0) {
+    qCritical() << "Failed to create audio buffer sink";
+    Close();
+    return false;
+  }
+
+  // Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside
+  // those boundaries, we need to daisychain more than one together.
+  double base = (speed_ > 1.0) ? 2.0 : 0.5;
+  double speed_log = log(speed_) / log(base);
+
+  // This is the number of how many 0.5 or 2.0 tempos we need to daisychain
+  int whole = qFloor(speed_log);
+
+  // Set speed_log to the remainder
+  speed_log -= whole;
+
+  AVFilterContext* previous_filter = buffersrc_ctx_;
+
+  for (int i=0;i<=whole;i++) {
+    double filter_tempo = (i == whole) ? qPow(base, speed_log) : base;
+
+    if (qFuzzyCompare(filter_tempo, 1.0)) {
+      // This filter would do nothing
+      continue;
+    }
+
+    previous_filter = CreateTempoFilter(filter_graph_,
+                                        previous_filter,
+                                        filter_tempo);
+
+    if (!previous_filter) {
+      qCritical() << "Failed to create audio tempo filter";
+      Close();
+      return false;
+    }
+  }
+
+  // Link the last filter to the buffersink
+  if (avfilter_link(previous_filter, 0, buffersink_ctx_, 0) != 0) {
+    qCritical() << "Failed to link final filter and buffer sink";
+    Close();
+    return false;
+  }
+
+  // Config graph
+  if (avfilter_graph_config(filter_graph_, nullptr) < 0) {
+    qCritical() << "Failed to configure filter graph";
+    Close();
+    return false;
+  }
+
+  timestamp_ = 0;
+
+  open_ = true;
+
+  flushed_ = false;
+
+  return true;
+}
+
+void TempoProcessor::Push(const char *data, int length)
+{
+  if (flushed_) {
+    if (length > 0) {
+      qCritical() << "Tried to push" << length << "bytes after TempoProcessor was closed";
+    }
+    return;
+  }
+
+  AVFrame* src_frame;
+
+  if (length == 0) {
+    // No audio data, flush the last out of the filter graph
+    src_frame = nullptr;
+    flushed_ = true;
+  } else {
+    src_frame = av_frame_alloc();
+
+    if (!src_frame) {
+      qCritical() << "Failed to allocate source frame";
+      return;
+    }
+
+    // Allocate a buffer for the number of samples we got
+    src_frame->sample_rate = params_.sample_rate();
+    src_frame->format = FFmpegCommon::GetFFmpegSampleFormat(params_.format());
+    src_frame->channel_layout = params_.channel_layout();
+    src_frame->nb_samples = params_.bytes_to_samples(length);
+    src_frame->pts = timestamp_;
+    timestamp_ += src_frame->nb_samples;
+
+    if (av_frame_get_buffer(src_frame, 0) < 0) {
+      qCritical() << "Failed to allocate buffer for source frame";
+      av_frame_free(&src_frame);
+      return;
+    }
+
+    // Copy buffer from data array to frame
+    memcpy(src_frame->data[0], data, length);
+  }
+
+  int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, src_frame, AV_BUFFERSRC_FLAG_KEEP_REF);
+
+  if (ret < 0) {
+    qCritical() << "Failed to feed buffer source" << ret;
+  }
+
+  if (src_frame) {
+    av_frame_free(&src_frame);
+  }
+}
+
+int TempoProcessor::Pull(char *data, int max_length)
+{
+  if (!processed_frame_) {
+    processed_frame_ = av_frame_alloc();
+
+    // Try to pull samples from the buffersink
+    int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame_);
+
+    if (ret < 0) {
+      // We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the
+      // error might be fatal...
+      if (ret != AVERROR(EAGAIN)) {
+        qCritical() << "Failed to pull from buffersink" << ret;
+      }
+
+      av_frame_free(&processed_frame_);
+
+      return 0;
+    }
+
+    processed_frame_byte_index_ = 0;
+    processed_frame_max_bytes_ = params_.samples_to_bytes(processed_frame_->nb_samples);
+  }
+
+  // Determine how many bytes we should copy into the data array
+  int copy_length = qMin(max_length, processed_frame_max_bytes_ - processed_frame_byte_index_);
+
+  // Copy the bytes
+  memcpy(data, processed_frame_->data[0] + processed_frame_byte_index_, copy_length);
+
+  // Add the copied amount to the current index
+  processed_frame_byte_index_ += copy_length;
+
+  // If the index has reached the limit of this processed frame, we can dispose of the frame now
+  if (processed_frame_byte_index_ == processed_frame_max_bytes_) {
+    av_frame_free(&processed_frame_);
+    processed_frame_ = nullptr;
+  }
+
+  return copy_length;
+}
+
+void TempoProcessor::Close()
+{
+  open_ = false;
+
+  if (filter_graph_) {
+    avfilter_graph_free(&filter_graph_);
+    filter_graph_ = nullptr;
+  }
+
+  if (processed_frame_) {
+    av_frame_free(&processed_frame_);
+    processed_frame_ = nullptr;
+  }
+
+  buffersrc_ctx_ = nullptr;
+  buffersink_ctx_ = nullptr;
+}
+
+AVFilterContext *TempoProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo)
+{
+  // Set up tempo param, which is taken as a C string
+  char speed_param[20];
+  snprintf(speed_param, 20, "%f", tempo);
+
+  AVFilterContext* tempo_ctx = nullptr;
+
+  if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0
+      && avfilter_link(link, 0, tempo_ctx, 0) == 0) {
+    return tempo_ctx;
+  }
+
+  return nullptr;
+}

app/audio/tempoprocessor.h

@@ -0,0 +1,53 @@
+#ifndef TEMPOPROCESSOR_H
+#define TEMPOPROCESSOR_H
+
+#include <inttypes.h>
+
+extern "C" {
+#include <libavfilter/avfilter.h>
+}
+
+#include "render/audioparams.h"
+
+class TempoProcessor
+{
+public:
+  TempoProcessor();
+
+  bool IsOpen() const;
+
+  const double& GetSpeed() const;
+
+  bool Open(const AudioRenderingParams& params, const double &speed);
+
+  void Push(const char *data, int length);
+
+  int Pull(char* data, int max_length);
+
+  void Close();
+
+private:
+  static AVFilterContext* CreateTempoFilter(AVFilterGraph *graph, AVFilterContext *link, const double& tempo);
+
+  AVFilterGraph* filter_graph_;
+
+  AVFilterContext* buffersrc_ctx_;
+
+  AVFilterContext* buffersink_ctx_;
+
+  AVFrame* processed_frame_;
+  int processed_frame_byte_index_;
+  int processed_frame_max_bytes_;
+
+  AudioRenderingParams params_;
+
+  int64_t timestamp_;
+
+  double speed_;
+
+  bool open_;
+
+  bool flushed_;
+};
+
+#endif // TEMPOPROCESSOR_H

app/render/backend/audiorenderworker.cpp

@@ -73,21 +73,8 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
       }
 
       if (b->is_reversed()) {
-        int sample_size = audio_params_.samples_to_bytes(1);
-        int half_buffer_sz = samples_from_this_block.size() / 2;
-        char* temp_buffer = new char[sample_size];
-
-        for (int src_index=0;src_index<half_buffer_sz;src_index+=sample_size) {
-          char* src_ptr = samples_from_this_block.data() + src_index;
-          char* dst_ptr = samples_from_this_block.data() + samples_from_this_block.size() - sample_size - src_index;
-
-          // Simple swap
-          memcpy(temp_buffer, src_ptr, static_cast<size_t>(sample_size));
-          memcpy(src_ptr, dst_ptr, static_cast<size_t>(sample_size));
-          memcpy(dst_ptr, temp_buffer, static_cast<size_t>(sample_size));
-        }
-
-        delete [] temp_buffer;
+        // Reverse the audio buffer
+        AudioManager::ReverseBuffer(samples_from_this_block.data(), samples_from_this_block.size(), audio_params_.samples_to_bytes(1));
       }
 
       copied_size = samples_from_this_block.size();

app/widget/viewer/viewer.cpp

@@ -257,16 +257,17 @@ void ViewerWidget::PlayInternal(int speed)
     return;
   }
 
+  playback_speed_ = speed;
+
   QIODevice* audio_src = audio_renderer_->GetAudioPullDevice();
   if (audio_src != nullptr && audio_src->open(QIODevice::ReadOnly)) {
     audio_src->seek(audio_renderer_->params().time_to_bytes(GetTime()));
     AudioManager::instance()->SetOutputParams(audio_renderer_->params());
-    AudioManager::instance()->StartOutput(audio_src);
+    AudioManager::instance()->StartOutput(audio_src, playback_speed_);
   }
 
   start_msec_ = QDateTime::currentMSecsSinceEpoch();
   start_timestamp_ = ruler_->GetTime();
-  playback_speed_ = speed;
 
   playback_timer_.start();