finished portaudio switch

This may have bugs in it, but as far as I can tell it's fully functional.

.github/workflows/ci.yml

@@ -317,7 +317,7 @@ jobs:
           brew update
           brew upgrade
           brew tap olive-editor/homebrew
-          brew install -f qt5 ffmpeg-olive openimageio-olive opencolorio opentimelineio
+          brew install -f qt5 ffmpeg-olive openimageio-olive opencolorio opentimelineio portaudio
           echo "/usr/local/opt/qt@5/bin" >> $GITHUB_PATH
 
       - name: Acquire Google Crashpad

app/audio/audiomanager.cpp

@@ -28,18 +28,6 @@ namespace olive {
 
 AudioManager* AudioManager::instance_ = nullptr;
 
-QString AudioManager::GetAudioBackendName(AudioManager::Backend b)
-{
-  switch (b) {
-  case kAudioBackendQt:
-    return tr("Qt");
-  case kAudioBackendCount:
-    break;
-  }
-
-  return tr("Unknown");
-}
-
 void AudioManager::CreateInstance()
 {
   if (instance_ == nullptr) {
@@ -87,17 +75,57 @@ bool AudioManager::IsRefreshingInputs()
   return is_refreshing_inputs_;
 }
 
-void AudioManager::PushToOutput(const QByteArray &samples)
+void AudioManager::SetOutputNotifyInterval(int n)
 {
-  if (!output_stream_) {
-    // Start output with no callback so it'll be in "push" mode
-    StartOutputStream();
-  }
-
-  Pa_WriteStream(output_stream_, samples.constData(), output_params_.bytes_to_samples(samples.size()));
+  output_buffer_->set_notify_interval(n);
 }
 
-static PaSampleFormat GetPortAudioSampleFormat(AudioParams::Format fmt)
+int OutputCallback(const void *input, void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo *timeInfo, PaStreamCallbackFlags statusFlags, void *userData)
+{
+  PreviewAudioDevice *device = static_cast<PreviewAudioDevice*>(userData);
+
+  qint64 max_read = frameCount * device->bytes_per_frame();
+  qint64 read_count = device->read(reinterpret_cast<char*>(output), max_read);
+  if (read_count < max_read) {
+    memset(reinterpret_cast<uint8_t*>(output) + read_count, 0, max_read - read_count);
+  }
+
+  return paContinue;
+}
+
+void AudioManager::PushToOutput(const AudioParams &params, const QByteArray &samples)
+{
+  if (output_params_ != params || output_stream_ == nullptr) {
+    output_params_ = params;
+
+    CloseOutputStream();
+
+    PaStreamParameters p;
+
+    p.channelCount = output_params_.channel_count();
+    p.device = output_device_;
+    p.hostApiSpecificStreamInfo = nullptr;
+    p.sampleFormat = GetPortAudioSampleFormat(output_params_.format());
+    p.suggestedLatency = Pa_GetDeviceInfo(output_device_)->defaultLowOutputLatency;
+
+    Pa_OpenStream(&output_stream_, nullptr, &p, output_params_.sample_rate(), paFramesPerBufferUnspecified, paNoFlag, OutputCallback, output_buffer_.get());
+
+    output_buffer_->set_bytes_per_frame(output_params_.samples_to_bytes(1));
+  }
+
+  output_buffer_->write(samples);
+
+  if (!Pa_IsStreamActive(output_stream_)) {
+    Pa_StartStream(output_stream_);
+  }
+}
+
+void AudioManager::ClearBufferedOutput()
+{
+  output_buffer_->clear();
+}
+
+PaSampleFormat AudioManager::GetPortAudioSampleFormat(AudioParams::Format fmt)
 {
   switch (fmt) {
   case AudioParams::kFormatUnsigned8:
@@ -118,38 +146,34 @@ static PaSampleFormat GetPortAudioSampleFormat(AudioParams::Format fmt)
   return 0;
 }
 
-int OutputCallback(const void *input, void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo *timeInfo, PaStreamCallbackFlags statusFlags, void *userData)
+void AudioManager::CloseOutputStream()
 {
-  PreviewAudioDevice *device = static_cast<PreviewAudioDevice*>(userData);
-
-  device->read(reinterpret_cast<char*>(output), frameCount * device->bytes_per_frame());
-
-  return paContinue;
-}
-
-void AudioManager::StartOutput(std::shared_ptr<PreviewAudioDevice> device)
-{
-  // First stop any current device
-  StopOutputStream();
-
-  // Store device
-  output_device_ = device;
-
-  // Start output stream that pulls from this device
-  StartOutputStream(OutputCallback);
+  if (output_stream_) {
+    if (Pa_IsStreamActive(output_stream_)) {
+      StopOutput();
+    }
+    Pa_CloseStream(output_stream_);
+    output_stream_ = nullptr;
+  }
 }
 
 void AudioManager::StopOutput()
 {
-  // Stop playback first so the callback won't get called again
-  StopOutputStream();
-
-  // Then clear our reference to the output devicec
-  output_device_ = nullptr;
+  // Abort the stream so playback stops immediately
+  if (output_stream_) {
+    Pa_AbortStream(output_stream_);
+    ClearBufferedOutput();
+  }
 }
 
-void AudioManager::SetOutputDevice(const QAudioDeviceInfo &info)
+void AudioManager::SetOutputDevice(PaDeviceIndex device)
 {
+  qInfo() << "Setting output audio device to" << Pa_GetDeviceInfo(device)->name;
+
+  output_device_ = device;
+
+  CloseOutputStream();
+
   /*qInfo() << "Setting output audio device to" << info.deviceName();
 
   StopOutput();
@@ -178,21 +202,11 @@ void AudioManager::SetOutputDevice(const QAudioDeviceInfo &info)
   }*/
 }
 
-void AudioManager::SetOutputParams(const AudioParams &params)
+void AudioManager::SetInputDevice(PaDeviceIndex device)
 {
-  if (output_params_ != params) {
-    // If an output stream is running, stop it now
-    StopOutputStream();
+  qInfo() << "Setting input audio device to" << Pa_GetDeviceInfo(device)->name;
 
-    // Update parameters
-    output_params_ = params;
-  }
-}
-
-void AudioManager::SetInputDevice(const QAudioDeviceInfo &info)
-{
-  input_ = std::unique_ptr<QAudioInput>(new QAudioInput(info, QAudioFormat(), this));
-  input_device_info_ = info;
+  input_device_ = device;
 }
 
 const QList<QAudioDeviceInfo> &AudioManager::ListInputDevices()
@@ -208,51 +222,27 @@ const QList<QAudioDeviceInfo> &AudioManager::ListOutputDevices()
 AudioManager::AudioManager() :
   is_refreshing_inputs_(false),
   is_refreshing_outputs_(false),
-  output_stream_(nullptr),
-  input_(nullptr),
-  input_file_(nullptr)
+  output_stream_(nullptr)
 {
   //RefreshDevices();
 
   Pa_Initialize();
 
-  output_ = Pa_GetDefaultOutputDevice();
+  SetOutputDevice(Pa_GetDefaultOutputDevice());
+  SetInputDevice(Pa_GetDefaultInputDevice());
+
+  output_buffer_ = std::make_unique<PreviewAudioDevice>();
+  output_buffer_->open(PreviewAudioDevice::ReadWrite);
+  connect(output_buffer_.get(), &PreviewAudioDevice::Notify, this, &AudioManager::OutputNotify);
 }
 
 AudioManager::~AudioManager()
 {
-  StopOutputStream();
+  CloseOutputStream();
 
   Pa_Terminate();
 }
 
-void AudioManager::StartOutputStream(PaStreamCallback *streamCallback)
-{
-  if (output_stream_) {
-    StopOutputStream();
-  }
-
-  PaStreamParameters p;
-
-  p.channelCount = output_params_.channel_count();
-  p.device = output_;
-  p.hostApiSpecificStreamInfo = nullptr;
-  p.sampleFormat = GetPortAudioSampleFormat(output_params_.format());
-  p.suggestedLatency = Pa_GetDeviceInfo(output_)->defaultLowOutputLatency;
-
-  Pa_OpenStream(&output_stream_, nullptr, &p, output_params_.sample_rate(), paFramesPerBufferUnspecified, paNoFlag, streamCallback, output_device_.get());
-  Pa_StartStream(output_stream_);
-}
-
-void AudioManager::StopOutputStream()
-{
-  if (output_stream_) {
-    Pa_AbortStream(output_stream_);
-    Pa_CloseStream(output_stream_);
-    output_stream_ = nullptr;
-  }
-}
-
 void AudioManager::OutputDevicesRefreshed()
 {
   /*QFutureWatcher< QList<QAudioDeviceInfo> >* watcher = static_cast<QFutureWatcher< QList<QAudioDeviceInfo> >*>(sender());
@@ -282,7 +272,7 @@ void AudioManager::OutputDevicesRefreshed()
 
 void AudioManager::InputDevicesRefreshed()
 {
-  QFutureWatcher< QList<QAudioDeviceInfo> >* watcher = static_cast<QFutureWatcher< QList<QAudioDeviceInfo> >*>(sender());
+  /*QFutureWatcher< QList<QAudioDeviceInfo> >* watcher = static_cast<QFutureWatcher< QList<QAudioDeviceInfo> >*>(sender());
 
   input_devices_ = watcher->result();
   watcher->deleteLater();
@@ -304,7 +294,7 @@ void AudioManager::InputDevicesRefreshed()
     }
   }
 
-  emit InputListReady();
+  emit InputListReady();*/
 }
 
 }

app/audio/audiomanager.h

@@ -45,13 +45,6 @@ class AudioManager : public QObject
 {
   Q_OBJECT
 public:
-  enum Backend {
-    kAudioBackendQt,
-    kAudioBackendCount
-  };
-
-  static QString GetAudioBackendName(Backend b);
-
   static void CreateInstance();
   static void DestroyInstance();
 
@@ -63,23 +56,17 @@ public:
 
   bool IsRefreshingInputs();
 
-  void PushToOutput(const QByteArray& samples);
+  void SetOutputNotifyInterval(int n);
 
-  /**
-   * @brief Start playing audio from AudioPlaybackCache
-   */
-  void StartOutput(std::shared_ptr<PreviewAudioDevice> device);
+  void PushToOutput(const AudioParams &params, const QByteArray& samples);
+
+  void ClearBufferedOutput();
 
-  /**
-   * @brief Stop audio output immediately
-   */
   void StopOutput();
 
-  void SetOutputDevice(const QAudioDeviceInfo& info);
+  void SetOutputDevice(PaDeviceIndex device);
 
-  void SetOutputParams(const AudioParams& params);
-
-  void SetInputDevice(const QAudioDeviceInfo& info);
+  void SetInputDevice(PaDeviceIndex device);
 
   const QList<QAudioDeviceInfo>& ListInputDevices();
   const QList<QAudioDeviceInfo>& ListOutputDevices();
@@ -87,6 +74,8 @@ public:
 signals:
   void OutputListReady();
 
+  void OutputNotify();
+
   void InputListReady();
 
 private:
@@ -94,9 +83,9 @@ private:
 
   virtual ~AudioManager() override;
 
-  void StartOutputStream(PaStreamCallback *streamCallback = nullptr);
+  static PaSampleFormat GetPortAudioSampleFormat(AudioParams::Format fmt);
 
-  void StopOutputStream();
+  void CloseOutputStream();
 
   QList<QAudioDeviceInfo> input_devices_;
   QList<QAudioDeviceInfo> output_devices_;
@@ -106,14 +95,12 @@ private:
 
   static AudioManager* instance_;
 
-  PaDeviceIndex output_;
+  PaDeviceIndex output_device_;
   PaStream *output_stream_;
   AudioParams output_params_;
-  std::shared_ptr<QIODevice> output_device_;
+  std::unique_ptr<PreviewAudioDevice> output_buffer_;
 
-  std::unique_ptr<QAudioInput> input_;
-  QAudioDeviceInfo input_device_info_;
-  QIODevice* input_file_;
+  PaDeviceIndex input_device_;
 
 private slots:
   void OutputDevicesRefreshed();

app/render/previewaudiodevice.cpp

@@ -22,17 +22,10 @@
 
 namespace olive {
 
-PreviewAudioDevice::PreviewAudioDevice(int bytes_per_frame, QObject *parent) :
-  bytes_per_frame_(bytes_per_frame),
+PreviewAudioDevice::PreviewAudioDevice(QObject *parent) :
   notify_interval_(0),
   bytes_read_(0)
 {
-  // These pointers are always valid
-  using_ = &internal_buffer_[0];
-  pushing_ = &internal_buffer_[1];
-
-  // Default to swap being true because we'll have nothing in the main buffer at first
-  swap_requested_ = true;
 }
 
 PreviewAudioDevice::~PreviewAudioDevice()
@@ -47,13 +40,9 @@ bool PreviewAudioDevice::isSequential() const
 
 qint64 PreviewAudioDevice::readData(char *data, qint64 maxSize)
 {
-  if (swap_requested_) {
-    SwapBuffers(kFullLock);
-    swap_requested_ = false;
-  }
+  QMutexLocker locker(&lock_);
 
-  // This function should NEVER touch the buffer in `pushing_`
-  qint64 copy_length = qMin(maxSize, qint64(using_->size()));
+  qint64 copy_length = qMin(maxSize, qint64(buffer_.size()));
 
   if (copy_length) {
     qint64 new_bytes_read = bytes_read_ + copy_length;
@@ -66,13 +55,8 @@ qint64 PreviewAudioDevice::readData(char *data, qint64 maxSize)
 
     bytes_read_ = new_bytes_read;
 
-    memcpy(data, using_->constData(), copy_length);
-    *using_ = using_->mid(copy_length);
-  }
-
-  if (using_->isEmpty() && !SwapBuffers(kTryLock)) {
-    // Ask push function to swap if it can. If it can't, we'll catch it next read.
-    swap_requested_ = true;
+    memcpy(data, buffer_.constData(), copy_length);
+    buffer_ = buffer_.mid(copy_length);
   }
 
   return copy_length;
@@ -80,41 +64,19 @@ qint64 PreviewAudioDevice::readData(char *data, qint64 maxSize)
 
 qint64 PreviewAudioDevice::writeData(const char *data, qint64 length)
 {
-  // This function should NEVER touch the buffer in `using_`
   QMutexLocker locker(&lock_);
-  pushing_->append(data, length);
 
-  // If swap requested, do this now
-  if (swap_requested_) {
-    SwapBuffers(kDontLock);
-    swap_requested_ = false;
-  }
+  buffer_.append(data, length);
 
   return length;
 }
 
-bool PreviewAudioDevice::SwapBuffers(LockMethod m)
+void PreviewAudioDevice::clear()
 {
-  switch (m) {
-  case kDontLock:
-    break;
-  case kTryLock:
-    if (!lock_.tryLock()) {
-      return false;
-    }
-    break;
-  case kFullLock:
-    lock_.lock();
-    break;
-  }
+  QMutexLocker locker(&lock_);
 
-  std::swap(using_, pushing_);
-
-  if (m != kDontLock) {
-    lock_.unlock();
-  }
-
-  return true;
+  buffer_.clear();
+  bytes_read_ = 0;
 }
 
 }

app/render/previewaudiodevice.h

@@ -29,7 +29,7 @@ class PreviewAudioDevice : public QIODevice
 {
   Q_OBJECT
 public:
-  PreviewAudioDevice(int bytes_per_frame, QObject *parent = nullptr);
+  PreviewAudioDevice(QObject *parent = nullptr);
 
   virtual ~PreviewAudioDevice() override;
 
@@ -46,31 +46,25 @@ public:
     return bytes_per_frame_;
   }
 
+  void set_bytes_per_frame(int b)
+  {
+    bytes_per_frame_ = b;
+  }
+
   void set_notify_interval(qint64 i)
   {
     notify_interval_ = i;
   }
 
+  void clear();
+
 signals:
   void Notify();
 
 private:
-  enum LockMethod {
-    kDontLock,
-    kTryLock,
-    kFullLock
-  };
-
-  bool SwapBuffers(LockMethod m);
-
   QMutex lock_;
 
-  QByteArray internal_buffer_[2];
-
-  QByteArray *using_;
-  QByteArray *pushing_;
-
-  QAtomicInt swap_requested_;
+  QByteArray buffer_;
 
   int bytes_per_frame_;
 

app/widget/viewer/viewer.cpp

@@ -58,8 +58,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) :
   color_menu_enabled_(true),
   time_changed_from_timer_(false),
   prequeuing_video_(false),
-  prequeuing_audio_(0),
-  audio_playback_device_(nullptr)
+  prequeuing_audio_(0)
 {
   // Set up main layout
   QVBoxLayout* layout = new QVBoxLayout(this);
@@ -409,14 +408,6 @@ void ViewerWidget::ClearVideoAutoCacherQueue()
   auto_cacher_.CancelVideoTasks();
 }
 
-void ViewerWidget::StartAudioOutput()
-{
-  AudioManager::instance()->SetOutputParams(GetConnectedNode()->GetAudioParams());
-  AudioManager::instance()->StartOutput(audio_playback_device_);
-  AudioMonitor::StartWaveformOnAll(&GetConnectedNode()->audio_playback_cache()->visual(),
-                                   GetTime(), playback_speed_);
-}
-
 void ViewerWidget::QueueNextAudioBuffer()
 {
   rational queue_end = audio_playback_queue_time_ + (kAudioPlaybackInterval * playback_speed_);
@@ -444,7 +435,7 @@ void ViewerWidget::ReceivedAudioBufferForPlayback()
 
     if (watcher->HasResult()) {
       SampleBufferPtr samples = watcher->Get().value<SampleBufferPtr>();
-      if (samples && audio_playback_device_) {
+      if (samples) {
         // If the samples must be reversed, reverse them now
         if (playback_speed_ < 0) {
           samples->reverse();
@@ -464,14 +455,18 @@ void ViewerWidget::ReceivedAudioBufferForPlayback()
 
         // TempoProcessor may have emptied the array
         if (!pack.isEmpty()) {
-          audio_playback_device_->write(pack);
-
           if (prequeuing_audio_) {
+            // Add to prequeued audio buffer
             prequeuing_audio_--;
 
+            prequeued_audio_.append(pack);
+
             if (!prequeuing_audio_) {
               FinishPlayPreprocess();
             }
+          } else {
+            // Push directly to audio manager
+            AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack);
           }
         }
       }
@@ -499,8 +494,8 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing()
       }*/
 
       QByteArray data = packed_processor_.Convert(samples);
-      AudioManager::instance()->SetOutputParams(samples->audio_params());
-      AudioManager::instance()->PushToOutput(data);
+      AudioManager::instance()->ClearBufferedOutput();
+      AudioManager::instance()->PushToOutput(samples->audio_params(), data);
       AudioMonitor::PushBytesOnAll(data);
     }
   }
@@ -668,20 +663,17 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only)
 
   AudioParams ap = GetConnectedNode()->GetAudioParams();
   if (ap.is_valid()) {
-    audio_playback_device_ = std::make_shared<PreviewAudioDevice>(ap.bytes_per_sample_per_channel() * ap.channel_count());
-    audio_playback_device_->set_notify_interval(ap.time_to_bytes(kAudioPlaybackInterval));
-    connect(audio_playback_device_.get(), &PreviewAudioDevice::Notify, this, &ViewerWidget::QueueNextAudioBuffer, Qt::QueuedConnection);
+    AudioManager::instance()->SetOutputNotifyInterval(ap.time_to_bytes(kAudioPlaybackInterval));
+    connect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer);
 
-    if (audio_playback_device_->open(QIODevice::ReadWrite)) {
-      if (std::abs(playback_speed_) > 1) {
-        tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_));
-      }
+    if (std::abs(playback_speed_) > 1) {
+      tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_));
+    }
 
-      prequeuing_audio_ = 2; // Queue two buffers ahead of time
-      audio_playback_queue_time_ = GetTime();
-      for (int i=0; i<prequeuing_audio_; i++) {
-        QueueNextAudioBuffer();
-      }
+    prequeuing_audio_ = 2; // Queue two buffers ahead of time
+    audio_playback_queue_time_ = GetTime();
+    for (int i=0; i<prequeuing_audio_; i++) {
+      QueueNextAudioBuffer();
     }
   }
 }
@@ -689,9 +681,6 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only)
 void ViewerWidget::PauseInternal()
 {
   if (IsPlaying()) {
-    AudioManager::instance()->StopOutput();
-    AudioMonitor::StopOnAll();
-
     playback_speed_ = 0;
     controls_->ShowPlayButton();
 
@@ -707,8 +696,11 @@ void ViewerWidget::PauseInternal()
     playback_queue_.clear();
     playback_backup_timer_.stop();
 
-    disconnect(audio_playback_device_.get(), &PreviewAudioDevice::Notify, this, &ViewerWidget::QueueNextAudioBuffer);
-    audio_playback_device_ = nullptr;
+    // Handle audio
+    AudioManager::instance()->StopOutput();
+    AudioMonitor::StopOnAll();
+    prequeued_audio_.clear();
+    disconnect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer);
     qDeleteAll(audio_playback_queue_);
     audio_playback_queue_.clear();
     if (tempo_processor_.IsOpen()) {
@@ -730,7 +722,7 @@ void ViewerWidget::PushScrubbedAudio()
 
     if (params.is_valid()) {
       // NOTE: Hardcoded scrubbing interval (20ms)
-      rational interval = rational(50, 1000);
+      rational interval = rational(20, 1000);
 
       RenderTicketWatcher *watcher = new RenderTicketWatcher();
       connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::ReceivedAudioBufferForScrubbing);
@@ -837,8 +829,13 @@ void ViewerWidget::FinishPlayPreprocess()
 
   int64_t playback_start_time = GetTimestamp();
 
-  if (audio_playback_device_) {
-    StartAudioOutput();
+  // Start audio waveform playback
+  if (!prequeued_audio_.isEmpty()) {
+    AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), prequeued_audio_);
+    prequeued_audio_.clear();
+
+    AudioMonitor::StartWaveformOnAll(&GetConnectedNode()->audio_playback_cache()->visual(),
+                                     GetTime(), playback_speed_);
   }
 
   playback_timer_.Start(playback_start_time, playback_speed_, timebase_dbl());

app/widget/viewer/viewer.h

@@ -256,11 +256,11 @@ private:
 
   QVector<RenderTicketWatcher*> queue_watchers_;
 
-  std::shared_ptr<PreviewAudioDevice> audio_playback_device_;
   std::list<RenderTicketWatcher*> audio_playback_queue_;
   rational audio_playback_queue_time_;
   PackedProcessor packed_processor_;
   TempoProcessor tempo_processor_;
+  QByteArray prequeued_audio_;
   static const int kAudioPlaybackInterval;
 
   static QVector<ViewerWidget*> instances_;
@@ -308,8 +308,6 @@ private slots:
 
   void Dropped(QDropEvent* event);
 
-  void StartAudioOutput();
-
   void QueueNextAudioBuffer();
 
   void ReceivedAudioBufferForPlayback();