From f49aa7d974e0d84c49fd55ea637de964dfd52916 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Fri, 1 Oct 2021 15:38:56 -0700 Subject: [PATCH] finished portaudio switch This may have bugs in it, but as far as I can tell it's fully functional. --- .github/workflows/ci.yml | 2 +- app/audio/audiomanager.cpp | 172 ++++++++++++++---------------- app/audio/audiomanager.h | 39 +++---- app/render/previewaudiodevice.cpp | 58 ++-------- app/render/previewaudiodevice.h | 24 ++--- app/widget/viewer/viewer.cpp | 67 ++++++------ app/widget/viewer/viewer.h | 4 +- 7 files changed, 147 insertions(+), 219 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 694218757..9f09fb3b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -317,7 +317,7 @@ jobs: brew update brew upgrade brew tap olive-editor/homebrew - brew install -f qt5 ffmpeg-olive openimageio-olive opencolorio opentimelineio + brew install -f qt5 ffmpeg-olive openimageio-olive opencolorio opentimelineio portaudio echo "/usr/local/opt/qt@5/bin" >> $GITHUB_PATH - name: Acquire Google Crashpad diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index 63f93da13..25b871b37 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -28,18 +28,6 @@ namespace olive { AudioManager* AudioManager::instance_ = nullptr; -QString AudioManager::GetAudioBackendName(AudioManager::Backend b) -{ - switch (b) { - case kAudioBackendQt: - return tr("Qt"); - case kAudioBackendCount: - break; - } - - return tr("Unknown"); -} - void AudioManager::CreateInstance() { if (instance_ == nullptr) { @@ -87,17 +75,57 @@ bool AudioManager::IsRefreshingInputs() return is_refreshing_inputs_; } -void AudioManager::PushToOutput(const QByteArray &samples) +void AudioManager::SetOutputNotifyInterval(int n) { - if (!output_stream_) { - // Start output with no callback so it'll be in "push" mode - StartOutputStream(); - } - - Pa_WriteStream(output_stream_, samples.constData(), output_params_.bytes_to_samples(samples.size())); + output_buffer_->set_notify_interval(n); } -static PaSampleFormat GetPortAudioSampleFormat(AudioParams::Format fmt) +int OutputCallback(const void *input, void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo *timeInfo, PaStreamCallbackFlags statusFlags, void *userData) +{ + PreviewAudioDevice *device = static_cast(userData); + + qint64 max_read = frameCount * device->bytes_per_frame(); + qint64 read_count = device->read(reinterpret_cast(output), max_read); + if (read_count < max_read) { + memset(reinterpret_cast(output) + read_count, 0, max_read - read_count); + } + + return paContinue; +} + +void AudioManager::PushToOutput(const AudioParams ¶ms, const QByteArray &samples) +{ + if (output_params_ != params || output_stream_ == nullptr) { + output_params_ = params; + + CloseOutputStream(); + + PaStreamParameters p; + + p.channelCount = output_params_.channel_count(); + p.device = output_device_; + p.hostApiSpecificStreamInfo = nullptr; + p.sampleFormat = GetPortAudioSampleFormat(output_params_.format()); + p.suggestedLatency = Pa_GetDeviceInfo(output_device_)->defaultLowOutputLatency; + + Pa_OpenStream(&output_stream_, nullptr, &p, output_params_.sample_rate(), paFramesPerBufferUnspecified, paNoFlag, OutputCallback, output_buffer_.get()); + + output_buffer_->set_bytes_per_frame(output_params_.samples_to_bytes(1)); + } + + output_buffer_->write(samples); + + if (!Pa_IsStreamActive(output_stream_)) { + Pa_StartStream(output_stream_); + } +} + +void AudioManager::ClearBufferedOutput() +{ + output_buffer_->clear(); +} + +PaSampleFormat AudioManager::GetPortAudioSampleFormat(AudioParams::Format fmt) { switch (fmt) { case AudioParams::kFormatUnsigned8: @@ -118,38 +146,34 @@ static PaSampleFormat GetPortAudioSampleFormat(AudioParams::Format fmt) return 0; } -int OutputCallback(const void *input, void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo *timeInfo, PaStreamCallbackFlags statusFlags, void *userData) +void AudioManager::CloseOutputStream() { - PreviewAudioDevice *device = static_cast(userData); - - device->read(reinterpret_cast(output), frameCount * device->bytes_per_frame()); - - return paContinue; -} - -void AudioManager::StartOutput(std::shared_ptr device) -{ - // First stop any current device - StopOutputStream(); - - // Store device - output_device_ = device; - - // Start output stream that pulls from this device - StartOutputStream(OutputCallback); + if (output_stream_) { + if (Pa_IsStreamActive(output_stream_)) { + StopOutput(); + } + Pa_CloseStream(output_stream_); + output_stream_ = nullptr; + } } void AudioManager::StopOutput() { - // Stop playback first so the callback won't get called again - StopOutputStream(); - - // Then clear our reference to the output devicec - output_device_ = nullptr; + // Abort the stream so playback stops immediately + if (output_stream_) { + Pa_AbortStream(output_stream_); + ClearBufferedOutput(); + } } -void AudioManager::SetOutputDevice(const QAudioDeviceInfo &info) +void AudioManager::SetOutputDevice(PaDeviceIndex device) { + qInfo() << "Setting output audio device to" << Pa_GetDeviceInfo(device)->name; + + output_device_ = device; + + CloseOutputStream(); + /*qInfo() << "Setting output audio device to" << info.deviceName(); StopOutput(); @@ -178,21 +202,11 @@ void AudioManager::SetOutputDevice(const QAudioDeviceInfo &info) }*/ } -void AudioManager::SetOutputParams(const AudioParams ¶ms) +void AudioManager::SetInputDevice(PaDeviceIndex device) { - if (output_params_ != params) { - // If an output stream is running, stop it now - StopOutputStream(); + qInfo() << "Setting input audio device to" << Pa_GetDeviceInfo(device)->name; - // Update parameters - output_params_ = params; - } -} - -void AudioManager::SetInputDevice(const QAudioDeviceInfo &info) -{ - input_ = std::unique_ptr(new QAudioInput(info, QAudioFormat(), this)); - input_device_info_ = info; + input_device_ = device; } const QList &AudioManager::ListInputDevices() @@ -208,51 +222,27 @@ const QList &AudioManager::ListOutputDevices() AudioManager::AudioManager() : is_refreshing_inputs_(false), is_refreshing_outputs_(false), - output_stream_(nullptr), - input_(nullptr), - input_file_(nullptr) + output_stream_(nullptr) { //RefreshDevices(); Pa_Initialize(); - output_ = Pa_GetDefaultOutputDevice(); + SetOutputDevice(Pa_GetDefaultOutputDevice()); + SetInputDevice(Pa_GetDefaultInputDevice()); + + output_buffer_ = std::make_unique(); + output_buffer_->open(PreviewAudioDevice::ReadWrite); + connect(output_buffer_.get(), &PreviewAudioDevice::Notify, this, &AudioManager::OutputNotify); } AudioManager::~AudioManager() { - StopOutputStream(); + CloseOutputStream(); Pa_Terminate(); } -void AudioManager::StartOutputStream(PaStreamCallback *streamCallback) -{ - if (output_stream_) { - StopOutputStream(); - } - - PaStreamParameters p; - - p.channelCount = output_params_.channel_count(); - p.device = output_; - p.hostApiSpecificStreamInfo = nullptr; - p.sampleFormat = GetPortAudioSampleFormat(output_params_.format()); - p.suggestedLatency = Pa_GetDeviceInfo(output_)->defaultLowOutputLatency; - - Pa_OpenStream(&output_stream_, nullptr, &p, output_params_.sample_rate(), paFramesPerBufferUnspecified, paNoFlag, streamCallback, output_device_.get()); - Pa_StartStream(output_stream_); -} - -void AudioManager::StopOutputStream() -{ - if (output_stream_) { - Pa_AbortStream(output_stream_); - Pa_CloseStream(output_stream_); - output_stream_ = nullptr; - } -} - void AudioManager::OutputDevicesRefreshed() { /*QFutureWatcher< QList >* watcher = static_cast >*>(sender()); @@ -282,7 +272,7 @@ void AudioManager::OutputDevicesRefreshed() void AudioManager::InputDevicesRefreshed() { - QFutureWatcher< QList >* watcher = static_cast >*>(sender()); + /*QFutureWatcher< QList >* watcher = static_cast >*>(sender()); input_devices_ = watcher->result(); watcher->deleteLater(); @@ -304,7 +294,7 @@ void AudioManager::InputDevicesRefreshed() } } - emit InputListReady(); + emit InputListReady();*/ } } diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index 1a8174bc1..7c2fa9882 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -45,13 +45,6 @@ class AudioManager : public QObject { Q_OBJECT public: - enum Backend { - kAudioBackendQt, - kAudioBackendCount - }; - - static QString GetAudioBackendName(Backend b); - static void CreateInstance(); static void DestroyInstance(); @@ -63,23 +56,17 @@ public: bool IsRefreshingInputs(); - void PushToOutput(const QByteArray& samples); + void SetOutputNotifyInterval(int n); - /** - * @brief Start playing audio from AudioPlaybackCache - */ - void StartOutput(std::shared_ptr device); + void PushToOutput(const AudioParams ¶ms, const QByteArray& samples); + + void ClearBufferedOutput(); - /** - * @brief Stop audio output immediately - */ void StopOutput(); - void SetOutputDevice(const QAudioDeviceInfo& info); + void SetOutputDevice(PaDeviceIndex device); - void SetOutputParams(const AudioParams& params); - - void SetInputDevice(const QAudioDeviceInfo& info); + void SetInputDevice(PaDeviceIndex device); const QList& ListInputDevices(); const QList& ListOutputDevices(); @@ -87,6 +74,8 @@ public: signals: void OutputListReady(); + void OutputNotify(); + void InputListReady(); private: @@ -94,9 +83,9 @@ private: virtual ~AudioManager() override; - void StartOutputStream(PaStreamCallback *streamCallback = nullptr); + static PaSampleFormat GetPortAudioSampleFormat(AudioParams::Format fmt); - void StopOutputStream(); + void CloseOutputStream(); QList input_devices_; QList output_devices_; @@ -106,14 +95,12 @@ private: static AudioManager* instance_; - PaDeviceIndex output_; + PaDeviceIndex output_device_; PaStream *output_stream_; AudioParams output_params_; - std::shared_ptr output_device_; + std::unique_ptr output_buffer_; - std::unique_ptr input_; - QAudioDeviceInfo input_device_info_; - QIODevice* input_file_; + PaDeviceIndex input_device_; private slots: void OutputDevicesRefreshed(); diff --git a/app/render/previewaudiodevice.cpp b/app/render/previewaudiodevice.cpp index b46fff7c0..c275c2c12 100644 --- a/app/render/previewaudiodevice.cpp +++ b/app/render/previewaudiodevice.cpp @@ -22,17 +22,10 @@ namespace olive { -PreviewAudioDevice::PreviewAudioDevice(int bytes_per_frame, QObject *parent) : - bytes_per_frame_(bytes_per_frame), +PreviewAudioDevice::PreviewAudioDevice(QObject *parent) : notify_interval_(0), bytes_read_(0) { - // These pointers are always valid - using_ = &internal_buffer_[0]; - pushing_ = &internal_buffer_[1]; - - // Default to swap being true because we'll have nothing in the main buffer at first - swap_requested_ = true; } PreviewAudioDevice::~PreviewAudioDevice() @@ -47,13 +40,9 @@ bool PreviewAudioDevice::isSequential() const qint64 PreviewAudioDevice::readData(char *data, qint64 maxSize) { - if (swap_requested_) { - SwapBuffers(kFullLock); - swap_requested_ = false; - } + QMutexLocker locker(&lock_); - // This function should NEVER touch the buffer in `pushing_` - qint64 copy_length = qMin(maxSize, qint64(using_->size())); + qint64 copy_length = qMin(maxSize, qint64(buffer_.size())); if (copy_length) { qint64 new_bytes_read = bytes_read_ + copy_length; @@ -66,13 +55,8 @@ qint64 PreviewAudioDevice::readData(char *data, qint64 maxSize) bytes_read_ = new_bytes_read; - memcpy(data, using_->constData(), copy_length); - *using_ = using_->mid(copy_length); - } - - if (using_->isEmpty() && !SwapBuffers(kTryLock)) { - // Ask push function to swap if it can. If it can't, we'll catch it next read. - swap_requested_ = true; + memcpy(data, buffer_.constData(), copy_length); + buffer_ = buffer_.mid(copy_length); } return copy_length; @@ -80,41 +64,19 @@ qint64 PreviewAudioDevice::readData(char *data, qint64 maxSize) qint64 PreviewAudioDevice::writeData(const char *data, qint64 length) { - // This function should NEVER touch the buffer in `using_` QMutexLocker locker(&lock_); - pushing_->append(data, length); - // If swap requested, do this now - if (swap_requested_) { - SwapBuffers(kDontLock); - swap_requested_ = false; - } + buffer_.append(data, length); return length; } -bool PreviewAudioDevice::SwapBuffers(LockMethod m) +void PreviewAudioDevice::clear() { - switch (m) { - case kDontLock: - break; - case kTryLock: - if (!lock_.tryLock()) { - return false; - } - break; - case kFullLock: - lock_.lock(); - break; - } + QMutexLocker locker(&lock_); - std::swap(using_, pushing_); - - if (m != kDontLock) { - lock_.unlock(); - } - - return true; + buffer_.clear(); + bytes_read_ = 0; } } diff --git a/app/render/previewaudiodevice.h b/app/render/previewaudiodevice.h index a285ad088..6d796e2ff 100644 --- a/app/render/previewaudiodevice.h +++ b/app/render/previewaudiodevice.h @@ -29,7 +29,7 @@ class PreviewAudioDevice : public QIODevice { Q_OBJECT public: - PreviewAudioDevice(int bytes_per_frame, QObject *parent = nullptr); + PreviewAudioDevice(QObject *parent = nullptr); virtual ~PreviewAudioDevice() override; @@ -46,31 +46,25 @@ public: return bytes_per_frame_; } + void set_bytes_per_frame(int b) + { + bytes_per_frame_ = b; + } + void set_notify_interval(qint64 i) { notify_interval_ = i; } + void clear(); + signals: void Notify(); private: - enum LockMethod { - kDontLock, - kTryLock, - kFullLock - }; - - bool SwapBuffers(LockMethod m); - QMutex lock_; - QByteArray internal_buffer_[2]; - - QByteArray *using_; - QByteArray *pushing_; - - QAtomicInt swap_requested_; + QByteArray buffer_; int bytes_per_frame_; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 997ce745a..4888a14e5 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -58,8 +58,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) : color_menu_enabled_(true), time_changed_from_timer_(false), prequeuing_video_(false), - prequeuing_audio_(0), - audio_playback_device_(nullptr) + prequeuing_audio_(0) { // Set up main layout QVBoxLayout* layout = new QVBoxLayout(this); @@ -409,14 +408,6 @@ void ViewerWidget::ClearVideoAutoCacherQueue() auto_cacher_.CancelVideoTasks(); } -void ViewerWidget::StartAudioOutput() -{ - AudioManager::instance()->SetOutputParams(GetConnectedNode()->GetAudioParams()); - AudioManager::instance()->StartOutput(audio_playback_device_); - AudioMonitor::StartWaveformOnAll(&GetConnectedNode()->audio_playback_cache()->visual(), - GetTime(), playback_speed_); -} - void ViewerWidget::QueueNextAudioBuffer() { rational queue_end = audio_playback_queue_time_ + (kAudioPlaybackInterval * playback_speed_); @@ -444,7 +435,7 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() if (watcher->HasResult()) { SampleBufferPtr samples = watcher->Get().value(); - if (samples && audio_playback_device_) { + if (samples) { // If the samples must be reversed, reverse them now if (playback_speed_ < 0) { samples->reverse(); @@ -464,14 +455,18 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() // TempoProcessor may have emptied the array if (!pack.isEmpty()) { - audio_playback_device_->write(pack); - if (prequeuing_audio_) { + // Add to prequeued audio buffer prequeuing_audio_--; + prequeued_audio_.append(pack); + if (!prequeuing_audio_) { FinishPlayPreprocess(); } + } else { + // Push directly to audio manager + AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack); } } } @@ -499,8 +494,8 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing() }*/ QByteArray data = packed_processor_.Convert(samples); - AudioManager::instance()->SetOutputParams(samples->audio_params()); - AudioManager::instance()->PushToOutput(data); + AudioManager::instance()->ClearBufferedOutput(); + AudioManager::instance()->PushToOutput(samples->audio_params(), data); AudioMonitor::PushBytesOnAll(data); } } @@ -668,20 +663,17 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) AudioParams ap = GetConnectedNode()->GetAudioParams(); if (ap.is_valid()) { - audio_playback_device_ = std::make_shared(ap.bytes_per_sample_per_channel() * ap.channel_count()); - audio_playback_device_->set_notify_interval(ap.time_to_bytes(kAudioPlaybackInterval)); - connect(audio_playback_device_.get(), &PreviewAudioDevice::Notify, this, &ViewerWidget::QueueNextAudioBuffer, Qt::QueuedConnection); + AudioManager::instance()->SetOutputNotifyInterval(ap.time_to_bytes(kAudioPlaybackInterval)); + connect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer); - if (audio_playback_device_->open(QIODevice::ReadWrite)) { - if (std::abs(playback_speed_) > 1) { - tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_)); - } + if (std::abs(playback_speed_) > 1) { + tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_)); + } - prequeuing_audio_ = 2; // Queue two buffers ahead of time - audio_playback_queue_time_ = GetTime(); - for (int i=0; iStopOutput(); - AudioMonitor::StopOnAll(); - playback_speed_ = 0; controls_->ShowPlayButton(); @@ -707,8 +696,11 @@ void ViewerWidget::PauseInternal() playback_queue_.clear(); playback_backup_timer_.stop(); - disconnect(audio_playback_device_.get(), &PreviewAudioDevice::Notify, this, &ViewerWidget::QueueNextAudioBuffer); - audio_playback_device_ = nullptr; + // Handle audio + AudioManager::instance()->StopOutput(); + AudioMonitor::StopOnAll(); + prequeued_audio_.clear(); + disconnect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer); qDeleteAll(audio_playback_queue_); audio_playback_queue_.clear(); if (tempo_processor_.IsOpen()) { @@ -730,7 +722,7 @@ void ViewerWidget::PushScrubbedAudio() if (params.is_valid()) { // NOTE: Hardcoded scrubbing interval (20ms) - rational interval = rational(50, 1000); + rational interval = rational(20, 1000); RenderTicketWatcher *watcher = new RenderTicketWatcher(); connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::ReceivedAudioBufferForScrubbing); @@ -837,8 +829,13 @@ void ViewerWidget::FinishPlayPreprocess() int64_t playback_start_time = GetTimestamp(); - if (audio_playback_device_) { - StartAudioOutput(); + // Start audio waveform playback + if (!prequeued_audio_.isEmpty()) { + AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), prequeued_audio_); + prequeued_audio_.clear(); + + AudioMonitor::StartWaveformOnAll(&GetConnectedNode()->audio_playback_cache()->visual(), + GetTime(), playback_speed_); } playback_timer_.Start(playback_start_time, playback_speed_, timebase_dbl()); diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index b9633b44f..c8c100d6a 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -256,11 +256,11 @@ private: QVector queue_watchers_; - std::shared_ptr audio_playback_device_; std::list audio_playback_queue_; rational audio_playback_queue_time_; PackedProcessor packed_processor_; TempoProcessor tempo_processor_; + QByteArray prequeued_audio_; static const int kAudioPlaybackInterval; static QVector instances_; @@ -308,8 +308,6 @@ private slots: void Dropped(QDropEvent* event); - void StartAudioOutput(); - void QueueNextAudioBuffer(); void ReceivedAudioBufferForPlayback();