From 35873dce0109a7987542a556eedb67b671d620f0 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 23 Sep 2021 17:08:12 -0700 Subject: [PATCH 01/21] started new cache --- app/node/output/viewer/viewer.cpp | 2 +- app/render/previewautocacher.cpp | 27 ++++++++++++++++++++++++++ app/render/previewautocacher.h | 32 +++++++++++++++++++++++++++++++ app/widget/viewer/viewer.cpp | 2 +- 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/app/node/output/viewer/viewer.cpp b/app/node/output/viewer/viewer.cpp index 0c1ebf408..d003c91d6 100644 --- a/app/node/output/viewer/viewer.cpp +++ b/app/node/output/viewer/viewer.cpp @@ -60,7 +60,7 @@ ViewerOutput::ViewerOutput(bool create_buffer_inputs, bool create_default_stream IgnoreHashingFrom(kVideoAutoCacheInput); IgnoreInvalidationsFrom(kVideoAutoCacheInput); - AddInput(kAudioAutoCacheInput, NodeValue::kBoolean, true, InputFlags(kInputFlagNotKeyframable | kInputFlagNotConnectable)); + AddInput(kAudioAutoCacheInput, NodeValue::kBoolean, false, InputFlags(kInputFlagNotKeyframable | kInputFlagNotConnectable)); IgnoreHashingFrom(kAudioAutoCacheInput); IgnoreInvalidationsFrom(kAudioAutoCacheInput); } diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index d18975da7..18d8367dd 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -887,4 +887,31 @@ void PreviewAutoCacher::SetViewerNode(ViewerOutput *viewer_node) } } +PreviewAutoCacher::PlaybackDevice::PlaybackDevice(PreviewAutoCacher *cacher, QObject *parent) : + cacher_(cacher), + current_time_(0) +{ + audio_params_ = viewer()->GetAudioParams(); +} + +bool PreviewAutoCacher::PlaybackDevice::seek(qint64 pos) +{ + // Call super function + if (QIODevice::seek(pos)) { + // Convert bytes to time + current_time_ = audio_params_.bytes_to_time(pos); + + return true; + } + + return false; +} + +qint64 PreviewAutoCacher::PlaybackDevice::size() const +{ + rational audio_length = cacher_->copied_viewer_node_->GetAudioLength(); + + return audio_params_.time_to_bytes(audio_length); +} + } diff --git a/app/render/previewautocacher.h b/app/render/previewautocacher.h index 1108714d6..04077fd14 100644 --- a/app/render/previewautocacher.h +++ b/app/render/previewautocacher.h @@ -9,6 +9,7 @@ #include "node/node.h" #include "node/output/viewer/viewer.h" #include "node/project/project.h" +#include "render/audioparams.h" #include "render/renderjobtracker.h" #include "threading/threadticketwatcher.h" @@ -71,6 +72,37 @@ public: void CancelVideoTasks(bool and_wait_for_them_to_finish = false); void CancelAudioTasks(bool and_wait_for_them_to_finish = false); + class PlaybackDevice : public QIODevice + { + public: + PlaybackDevice(PreviewAutoCacher *cacher, QObject *parent = nullptr); + + virtual ~PlaybackDevice() override; + + virtual bool isSequential() const override; + + virtual bool seek(qint64 pos) override; + + virtual qint64 size() const override; + + virtual qint64 readData(char *data, qint64 maxSize) override; + + virtual qint64 writeData(const char *, qint64) override; + + ViewerOutput *viewer() const + { + return cacher_->copied_viewer_node_; + } + + private: + PreviewAutoCacher *cacher_; + + rational current_time_; + + AudioParams audio_params_; + + }; + private: void TryRender(); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 253443795..b9cb8aa60 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -603,7 +603,7 @@ void ViewerWidget::PushScrubbedAudio() const AudioParams& params = GetConnectedNode()->audio_playback_cache()->GetParameters(); if (params.is_valid()) { - AudioPlaybackCache::PlaybackDevice* audio_src = GetConnectedNode()->audio_playback_cache()->CreatePlaybackDevice(); + PreviewAutoCacher::PlaybackDevice *audio_src = new PreviewAutoCacher::PlaybackDevice(&auto_cacher_, this); if (audio_src->open(QIODevice::ReadOnly)) { // FIXME: Hardcoded scrubbing interval (20ms) From 4c9863a6963bfd39f45ea03725668ee742ba7db4 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sat, 25 Sep 2021 12:44:58 -0700 Subject: [PATCH 02/21] implement audio device to pull directly from renderer --- app/audio/audiomanager.cpp | 8 +- app/audio/audiomanager.h | 4 +- app/audio/outputdeviceproxy.cpp | 10 +-- app/audio/outputdeviceproxy.h | 2 +- app/audio/outputmanager.cpp | 4 +- app/audio/outputmanager.h | 2 +- app/codec/samplebuffer.cpp | 6 +- app/render/CMakeLists.txt | 2 + app/render/previewaudiodevice.cpp | 111 +++++++++++++++++++++++++++ app/render/previewaudiodevice.h | 68 +++++++++++++++++ app/render/previewautocacher.cpp | 70 +++++++++-------- app/render/previewautocacher.h | 54 ++++++------- app/widget/viewer/viewer.cpp | 123 ++++++++++++++++++------------ app/widget/viewer/viewer.h | 17 +++-- 14 files changed, 336 insertions(+), 145 deletions(-) create mode 100644 app/render/previewaudiodevice.cpp create mode 100644 app/render/previewaudiodevice.h diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index 0b4f2fa57..382b6dd8d 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -94,11 +94,8 @@ void AudioManager::PushToOutput(const QByteArray &samples) emit OutputPushed(samples); } -void AudioManager::StartOutput(AudioPlaybackCache *cache, qint64 offset, int playback_speed) +void AudioManager::StartOutput(QIODevice *device, int playback_speed) { - // Create device - QIODevice* device = cache->CreatePlaybackDevice(); - // Move to output manager's thread device->moveToThread(&output_thread_); @@ -107,10 +104,7 @@ void AudioManager::StartOutput(AudioPlaybackCache *cache, qint64 offset, int pla "PullFromDevice", Qt::QueuedConnection, Q_ARG(QIODevice*, device), - Q_ARG(qint64, offset), Q_ARG(int, playback_speed)); - - emit OutputDeviceStarted(cache, offset, playback_speed); } void AudioManager::StopOutput() diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index d71b1c57d..43c4edc29 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -68,7 +68,7 @@ public: /** * @brief Start playing audio from AudioPlaybackCache */ - void StartOutput(AudioPlaybackCache* cache, qint64 offset, int playback_speed); + void StartOutput(QIODevice *device, int playback_speed); /** * @brief Stop audio output immediately @@ -93,8 +93,6 @@ signals: void OutputNotified(); - void OutputDeviceStarted(AudioPlaybackCache* cache, qint64 offset, int playback_speed); - void OutputWaveformStarted(const AudioVisualWaveform* waveform, const rational &start, int playback_speed); void AudioParamsChanged(const AudioParams& params); diff --git a/app/audio/outputdeviceproxy.cpp b/app/audio/outputdeviceproxy.cpp index f8da3cb05..712b3f556 100644 --- a/app/audio/outputdeviceproxy.cpp +++ b/app/audio/outputdeviceproxy.cpp @@ -35,24 +35,17 @@ void AudioOutputDeviceProxy::SetParameters(const AudioParams ¶ms) params_ = params; } -void AudioOutputDeviceProxy::SetDevice(QIODevice* device, qint64 offset, int playback_speed) +void AudioOutputDeviceProxy::SetDevice(QIODevice* device, int playback_speed) { - if (device_) { - delete device_; - } - device_ = device; device_->setParent(this); if (!device_->open(QFile::ReadOnly)) { qCritical() << "Failed to open IO device for audio playback"; - delete device_; device_ = nullptr; return; } - device_->seek(offset); - playback_speed_ = playback_speed; if (qAbs(playback_speed_) != 1) { @@ -64,7 +57,6 @@ void AudioOutputDeviceProxy::close() { QIODevice::close(); - delete device_; device_ = nullptr; if (tempo_processor_.IsOpen()) { diff --git a/app/audio/outputdeviceproxy.h b/app/audio/outputdeviceproxy.h index 08022b2b4..453d1b94c 100644 --- a/app/audio/outputdeviceproxy.h +++ b/app/audio/outputdeviceproxy.h @@ -39,7 +39,7 @@ public: void SetParameters(const AudioParams& params); - void SetDevice(QIODevice *device, qint64 offset, int playback_speed); + void SetDevice(QIODevice *device, int playback_speed); virtual void close() override; diff --git a/app/audio/outputmanager.cpp b/app/audio/outputmanager.cpp index 5d7298950..3e032075e 100644 --- a/app/audio/outputmanager.cpp +++ b/app/audio/outputmanager.cpp @@ -90,7 +90,7 @@ void AudioOutputManager::Close() } } -void AudioOutputManager::PullFromDevice(QIODevice *device, qint64 offset, int playback_speed) +void AudioOutputManager::PullFromDevice(QIODevice *device, int playback_speed) { if (!output_) { return; @@ -102,7 +102,7 @@ void AudioOutputManager::PullFromDevice(QIODevice *device, qint64 offset, int pl push_samples_.clear(); // Pull from the device - device_proxy_.SetDevice(device, offset, playback_speed); + device_proxy_.SetDevice(device, playback_speed); device_proxy_.open(QIODevice::ReadOnly); output_->start(&device_proxy_); } diff --git a/app/audio/outputmanager.h b/app/audio/outputmanager.h index 888a0f923..990303bd3 100644 --- a/app/audio/outputmanager.h +++ b/app/audio/outputmanager.h @@ -53,7 +53,7 @@ public slots: * This will clear any pushed samples or QIODevices currently being read and will start reading from this next time * the audio output requests data. */ - void PullFromDevice(QIODevice* device, qint64 offset, int playback_speed); + void PullFromDevice(QIODevice* device, int playback_speed); // Queued void ResetToPushMode(); diff --git a/app/codec/samplebuffer.cpp b/app/codec/samplebuffer.cpp index 4846c96a3..5a60c49a2 100644 --- a/app/codec/samplebuffer.cpp +++ b/app/codec/samplebuffer.cpp @@ -246,13 +246,9 @@ QByteArray SampleBuffer::toPackedData() const float* output_data = reinterpret_cast(packed_data.data()); - int output_index = 0; - for (int j=0;j. + +***/ + +#include "previewaudiodevice.h" + +namespace olive { + +PreviewAudioDevice::PreviewAudioDevice(QObject *parent) +{ + // These pointers are always valid + using_ = &internal_buffer_[0]; + pushing_ = &internal_buffer_[1]; + + // Default to swap being true because we'll have nothing in the main buffer at first + swap_requested_ = true; +} + +PreviewAudioDevice::~PreviewAudioDevice() +{ + close(); +} + +bool PreviewAudioDevice::isSequential() const +{ + return true; +} + +qint64 PreviewAudioDevice::readData(char *data, qint64 maxSize) +{ + if (swap_requested_) { + SwapBuffers(kFullLock); + swap_requested_ = false; + } + + // This function should NEVER touch the buffer in `pushing_` + qint64 copy_length = qMin(maxSize, qint64(using_->size())); + + if (copy_length) { + memcpy(data, using_->constData(), copy_length); + *using_ = using_->mid(copy_length); + + if (using_->isEmpty() && !SwapBuffers(kTryLock)) { + // Ask push function to swap if it can. If it can't, we'll catch it next read. + swap_requested_ = true; + } + } + + return copy_length; +} + +qint64 PreviewAudioDevice::writeData(const char *, qint64) +{ + // No writing to this device + return -1; +} + +void PreviewAudioDevice::Push(const QByteArray &b) +{ + // This function should NEVER touch the buffer in `using_` + QMutexLocker locker(&lock_); + pushing_->append(b); + + // If swap requested, do this now + if (swap_requested_) { + SwapBuffers(kDontLock); + swap_requested_ = false; + } +} + +bool PreviewAudioDevice::SwapBuffers(LockMethod m) +{ + switch (m) { + case kDontLock: + break; + case kTryLock: + if (!lock_.tryLock()) { + return false; + } + break; + case kFullLock: + lock_.lock(); + break; + } + + std::swap(using_, pushing_); + + if (m != kDontLock) { + lock_.unlock(); + } + + return true; +} + +} diff --git a/app/render/previewaudiodevice.h b/app/render/previewaudiodevice.h new file mode 100644 index 000000000..9b4179128 --- /dev/null +++ b/app/render/previewaudiodevice.h @@ -0,0 +1,68 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef PREVIEWAUDIODEVICE_H +#define PREVIEWAUDIODEVICE_H + +#include "previewautocacher.h" + +namespace olive { + +class PreviewAudioDevice : public QIODevice +{ + Q_OBJECT +public: + PreviewAudioDevice(QObject *parent = nullptr); + + virtual ~PreviewAudioDevice() override; + + void StartQueuing(); + + virtual bool isSequential() const override; + + virtual qint64 readData(char *data, qint64 maxSize) override; + + virtual qint64 writeData(const char *, qint64) override; + + void Push(const QByteArray &b); + +private: + enum LockMethod { + kDontLock, + kTryLock, + kFullLock + }; + + bool SwapBuffers(LockMethod m); + + QMutex lock_; + + QByteArray internal_buffer_[2]; + + QByteArray *using_; + QByteArray *pushing_; + + QAtomicInt swap_requested_; + +}; + +} + +#endif // PREVIEWAUDIODEVICE_H diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index 18d8367dd..945eab6f8 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -1,3 +1,23 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + #include "previewautocacher.h" #include @@ -61,6 +81,11 @@ RenderTicketPtr PreviewAutoCacher::GetSingleFrame(const rational &t, bool priori return sfr; } +RenderTicketPtr PreviewAutoCacher::GetRangeOfAudio(TimeRange range, bool prioritize) +{ + return RenderAudio(range, false, prioritize); +} + QVector PreviewAutoCacher::GenerateHashes(ViewerOutput *viewer, FrameHashCache* cache, const QVector ×) { QVector hash_data(times.size()); @@ -631,11 +656,7 @@ void PreviewAutoCacher::TryRender() r.set_out(qMin(r.out(), r.in() + 1)); // Start job - RenderTicketWatcher* watcher = new RenderTicketWatcher(); - watcher->setProperty("job", QVariant::fromValue(last_update_time_)); - connect(watcher, &RenderTicketWatcher::Finished, this, &PreviewAutoCacher::AudioRendered); - audio_tasks_.insert(watcher, r); - watcher->SetTicket(RenderManager::instance()->RenderAudio(copied_viewer_node_, r, RenderMode::kOffline, true)); + RenderAudio(r, true, false); audio_iterator_.remove(r); } @@ -658,6 +679,18 @@ RenderTicketWatcher* PreviewAutoCacher::RenderFrame(const QByteArray &hash, cons return watcher; } +RenderTicketPtr PreviewAutoCacher::RenderAudio(const TimeRange &r, bool generate_waveforms, bool prioritize) +{ + RenderTicketWatcher* watcher = new RenderTicketWatcher(); + watcher->setProperty("job", QVariant::fromValue(last_update_time_)); + connect(watcher, &RenderTicketWatcher::Finished, this, &PreviewAutoCacher::AudioRendered); + audio_tasks_.insert(watcher, r); + + RenderTicketPtr ticket = RenderManager::instance()->RenderAudio(copied_viewer_node_, r, RenderMode::kOffline, generate_waveforms, prioritize); + watcher->SetTicket(ticket); + return ticket; +} + void PreviewAutoCacher::RequeueFrames() { delayed_requeue_timer_.stop(); @@ -887,31 +920,4 @@ void PreviewAutoCacher::SetViewerNode(ViewerOutput *viewer_node) } } -PreviewAutoCacher::PlaybackDevice::PlaybackDevice(PreviewAutoCacher *cacher, QObject *parent) : - cacher_(cacher), - current_time_(0) -{ - audio_params_ = viewer()->GetAudioParams(); -} - -bool PreviewAutoCacher::PlaybackDevice::seek(qint64 pos) -{ - // Call super function - if (QIODevice::seek(pos)) { - // Convert bytes to time - current_time_ = audio_params_.bytes_to_time(pos); - - return true; - } - - return false; -} - -qint64 PreviewAutoCacher::PlaybackDevice::size() const -{ - rational audio_length = cacher_->copied_viewer_node_->GetAudioLength(); - - return audio_params_.time_to_bytes(audio_length); -} - } diff --git a/app/render/previewautocacher.h b/app/render/previewautocacher.h index 04077fd14..a4d15cf82 100644 --- a/app/render/previewautocacher.h +++ b/app/render/previewautocacher.h @@ -1,3 +1,23 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + #ifndef AUTOCACHER_H #define AUTOCACHER_H @@ -30,6 +50,8 @@ public: RenderTicketPtr GetSingleFrame(const rational& t, bool prioritize); + RenderTicketPtr GetRangeOfAudio(TimeRange range, bool prioritize); + /** * @brief Set the viewer node to auto-cache */ @@ -72,41 +94,11 @@ public: void CancelVideoTasks(bool and_wait_for_them_to_finish = false); void CancelAudioTasks(bool and_wait_for_them_to_finish = false); - class PlaybackDevice : public QIODevice - { - public: - PlaybackDevice(PreviewAutoCacher *cacher, QObject *parent = nullptr); - - virtual ~PlaybackDevice() override; - - virtual bool isSequential() const override; - - virtual bool seek(qint64 pos) override; - - virtual qint64 size() const override; - - virtual qint64 readData(char *data, qint64 maxSize) override; - - virtual qint64 writeData(const char *, qint64) override; - - ViewerOutput *viewer() const - { - return cacher_->copied_viewer_node_; - } - - private: - PreviewAutoCacher *cacher_; - - rational current_time_; - - AudioParams audio_params_; - - }; - private: void TryRender(); RenderTicketWatcher *RenderFrame(const QByteArray& hash, const rational &time, bool prioritize, bool texture_only); + RenderTicketPtr RenderAudio(const TimeRange &range, bool generate_waveforms, bool prioritize); /** * @brief Process all changes to internal NodeGraph copy diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index b9cb8aa60..f359cf5d2 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -54,8 +54,10 @@ ViewerWidget::ViewerWidget(QWidget *parent) : playback_speed_(0), color_menu_enabled_(true), time_changed_from_timer_(false), - prequeuing_(false), - active_queue_jobs_(0) + prequeuing_video_(false), + prequeuing_audio_(false), + active_queue_jobs_(0), + audio_playback_device_(nullptr) { // Set up main layout QVBoxLayout* layout = new QVBoxLayout(this); @@ -111,11 +113,6 @@ ViewerWidget::ViewerWidget(QWidget *parent) : connect(controls_, &PlaybackControls::TimeChanged, this, &ViewerWidget::SetTimeAndSignal); layout->addWidget(controls_); - // If audio is invalidated during playback, we wait some time before starting it again - audio_restart_timer_.setInterval(250); - audio_restart_timer_.setSingleShot(true); - connect(&audio_restart_timer_, &QTimer::timeout, this, &ViewerWidget::StartAudioOutput); - // FIXME: Magic number SetScale(48.0); @@ -182,8 +179,6 @@ void ViewerWidget::ConnectNodeEvent(ViewerOutput *n) connect(n, &ViewerOutput::AudioParamsChanged, this, &ViewerWidget::UpdateRendererAudioParameters); connect(n->video_frame_cache(), &FrameHashCache::Invalidated, this, &ViewerWidget::ViewerInvalidatedVideoRange); connect(n->video_frame_cache(), &FrameHashCache::Shifted, this, &ViewerWidget::ViewerShiftedRange); - connect(n->audio_playback_cache(), &AudioPlaybackCache::Invalidated, this, &ViewerWidget::AudioCacheInvalidated); - connect(n->audio_playback_cache(), &AudioPlaybackCache::Validated, this, &ViewerWidget::AudioCacheValidated); connect(n, &ViewerOutput::TextureInputChanged, this, &ViewerWidget::UpdateStack); VideoParams vp = n->GetVideoParams(); @@ -228,8 +223,6 @@ void ViewerWidget::DisconnectNodeEvent(ViewerOutput *n) disconnect(n, &ViewerOutput::AudioParamsChanged, this, &ViewerWidget::UpdateRendererAudioParameters); disconnect(n->video_frame_cache(), &FrameHashCache::Invalidated, this, &ViewerWidget::ViewerInvalidatedVideoRange); disconnect(n->video_frame_cache(), &FrameHashCache::Shifted, this, &ViewerWidget::ViewerShiftedRange); - disconnect(n->audio_playback_cache(), &AudioPlaybackCache::Invalidated, this, &ViewerWidget::AudioCacheInvalidated); - disconnect(n->audio_playback_cache(), &AudioPlaybackCache::Validated, this, &ViewerWidget::AudioCacheValidated); disconnect(n, &ViewerOutput::TextureInputChanged, this, &ViewerWidget::UpdateStack); SetDisplayImage(QVariant()); @@ -408,14 +401,55 @@ void ViewerWidget::ClearVideoAutoCacherQueue() void ViewerWidget::StartAudioOutput() { - AudioPlaybackCache* audio_cache = GetConnectedNode()->audio_playback_cache(); - if (audio_cache->GetParameters().is_valid()) { - AudioManager::instance()->SetOutputParams(audio_cache->GetParameters()); - AudioManager::instance()->StartOutput(audio_cache, - audio_cache->GetParameters().time_to_bytes_per_channel(GetTime()), - playback_speed_); - emit AudioManager::instance()->OutputWaveformStarted(&audio_cache->visual(), - GetTime(), playback_speed_); + AudioParams params = GetConnectedNode()->GetAudioParams(); + + if (params.is_valid()) { + AudioManager::instance()->SetOutputParams(params); + AudioManager::instance()->StartOutput(audio_playback_device_.get(), playback_speed_); + + qDebug() << "STUB: Nothing to send to audio monitor"; + /*emit AudioManager::instance()->OutputWaveformStarted(&audio_cache->visual(), + GetTime(), playback_speed_);*/ + } +} + +void ViewerWidget::QueueNextAudioBuffer() +{ + // NOTE: Hardcoded 2 second interval + TimeRange range(audio_playback_queue_time_, audio_playback_queue_time_ + 2); + audio_playback_queue_time_ = range.out(); + + RenderTicketWatcher *watcher = new RenderTicketWatcher(this); + connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::ReceivedAudioBufferForPlayback); + audio_playback_queue_.push_back(watcher); + watcher->SetTicket(auto_cacher_.GetRangeOfAudio(range, true)); +} + +void ViewerWidget::ReceivedAudioBufferForPlayback() +{ + while (!audio_playback_queue_.empty() && !audio_playback_queue_.front()->IsRunning()) { + RenderTicketWatcher *watcher = audio_playback_queue_.front(); + audio_playback_queue_.pop_front(); + + if (watcher->HasResult()) { + SampleBufferPtr samples = watcher->Get().value(); + if (samples && audio_playback_device_) { + qint64 t = QDateTime::currentMSecsSinceEpoch(); + QByteArray pack = samples->toPackedData(); + qDebug() << "Packing took:" << (QDateTime::currentMSecsSinceEpoch() - t); + audio_playback_device_->Push(pack); + + if (prequeuing_audio_) { + prequeuing_audio_ = false; + FinishPlayPreprocess(); + } + } + } + + // Do this in the loop so that clearing the array effectively prevents a queue + QueueNextAudioBuffer(); + + delete watcher; } } @@ -551,7 +585,7 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) prequeue_length_ = DeterminePlaybackQueueSize(); if (prequeue_length_ > 0) { - prequeuing_ = true; + prequeuing_video_ = true; // We "prioritize" the frames, which means they're pushed to the top of the render queue, // we queue in reverse so that they're still queued in order @@ -568,9 +602,10 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) } } - if (!prequeuing_) { - FinishPlayPreprocess(); - } + audio_playback_device_.reset(new PreviewAudioDevice()); + prequeuing_audio_ = true; + audio_playback_queue_time_ = GetTime(); + QueueNextAudioBuffer(); } void ViewerWidget::PauseInternal() @@ -588,12 +623,16 @@ void ViewerWidget::PauseInternal() playback_queue_.clear(); playback_backup_timer_.stop(); - audio_restart_timer_.stop(); + + audio_playback_device_.reset(nullptr); + qDeleteAll(audio_playback_queue_); + audio_playback_queue_.clear(); UpdateTextureFromNode(); } - prequeuing_ = false; + prequeuing_video_ = false; + prequeuing_audio_ = false; } void ViewerWidget::PushScrubbedAudio() @@ -603,14 +642,14 @@ void ViewerWidget::PushScrubbedAudio() const AudioParams& params = GetConnectedNode()->audio_playback_cache()->GetParameters(); if (params.is_valid()) { - PreviewAutoCacher::PlaybackDevice *audio_src = new PreviewAutoCacher::PlaybackDevice(&auto_cacher_, this); + qDebug() << "STUB: Use PAC audio function directly"; + /*PreviewAudioDevice *audio_src = new PreviewAudioDevice(&auto_cacher_, GetTime()); if (audio_src->open(QIODevice::ReadOnly)) { // FIXME: Hardcoded scrubbing interval (20ms) int size_of_sample = params.time_to_bytes(rational(20, 1000)); // Push audio - audio_src->seek(params.time_to_bytes_per_channel(GetTime())); QByteArray frame_audio = audio_src->read(size_of_sample); AudioManager::instance()->SetOutputParams(params); AudioManager::instance()->PushToOutput(frame_audio); @@ -618,7 +657,7 @@ void ViewerWidget::PushScrubbedAudio() audio_src->close(); } - delete audio_src; + delete audio_src;*/ } } } @@ -714,6 +753,11 @@ RenderTicketPtr ViewerWidget::GetFrame(const rational &t, bool prioritize) void ViewerWidget::FinishPlayPreprocess() { + // Check if we're still waiting for video or audio respectively + if (prequeuing_video_ || prequeuing_audio_) { + return; + } + int64_t playback_start_time = GetTimestamp(); StartAudioOutput(); @@ -858,7 +902,7 @@ void ViewerWidget::RendererGeneratedFrameForQueue() QVariant frame = watcher->Get(); // Ignore this signal if we've paused now - if (IsPlaying() || prequeuing_) { + if (IsPlaying() || prequeuing_video_) { rational ts = watcher->property("time").value(); playback_queue_.AppendTimewise({ts, frame}, playback_speed_); @@ -867,8 +911,8 @@ void ViewerWidget::RendererGeneratedFrameForQueue() window->queue()->AppendTimewise({ts, frame}, playback_speed_); } - if (prequeuing_ && int(playback_queue_.size()) == prequeue_length_) { - prequeuing_ = false; + if (prequeuing_video_ && int(playback_queue_.size()) == prequeue_length_) { + prequeuing_video_ = false; FinishPlayPreprocess(); } } @@ -1323,21 +1367,4 @@ void ViewerWidget::Dropped(QDropEvent *event) } } -void ViewerWidget::AudioCacheInvalidated() -{ - if (IsPlaying()) { - AudioManager::instance()->StopOutput(); - } -} - -void ViewerWidget::AudioCacheValidated() -{ - if (IsPlaying()) { - // This timer will restart audio - AudioManager::instance()->StopOutput(); - audio_restart_timer_.stop(); - audio_restart_timer_.start(); - } -} - } diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index e5f6edc82..a0e91f898 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -32,6 +32,7 @@ #include "common/rational.h" #include "node/output/viewer/viewer.h" #include "panel/scope/scope.h" +#include "render/previewaudiodevice.h" #include "render/previewautocacher.h" #include "threading/threadticketwatcher.h" #include "viewerdisplay.h" @@ -240,7 +241,8 @@ private: ViewerQueue playback_queue_; int64_t playback_queue_next_frame_; - bool prequeuing_; + bool prequeuing_video_; + bool prequeuing_audio_; QList nonqueue_watchers_; @@ -250,10 +252,12 @@ private: PreviewAutoCacher auto_cacher_; - QTimer audio_restart_timer_; - int active_queue_jobs_; + std::unique_ptr audio_playback_device_; + std::list audio_playback_queue_; + rational audio_playback_queue_time_; + static QVector instances_; private slots: @@ -299,11 +303,12 @@ private slots: void Dropped(QDropEvent* event); - void AudioCacheInvalidated(); - void AudioCacheValidated(); - void StartAudioOutput(); + void QueueNextAudioBuffer(); + + void ReceivedAudioBufferForPlayback(); + }; } From 845b4a755954a626587aeaf166757daeef522752 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sat, 25 Sep 2021 13:22:13 -0700 Subject: [PATCH 03/21] use ffmpeg for sample packing FFmpeg has a much more optimized algorithm than we do so may as well use it. --- app/audio/CMakeLists.txt | 12 ++-- app/audio/packedprocessor.cpp | 106 ++++++++++++++++++++++++++++++++++ app/audio/packedprocessor.h | 60 +++++++++++++++++++ app/audio/tempoprocessor.cpp | 4 ++ app/audio/tempoprocessor.h | 4 ++ app/codec/samplebuffer.cpp | 19 ------ app/codec/samplebuffer.h | 2 - app/widget/viewer/viewer.cpp | 11 +++- app/widget/viewer/viewer.h | 2 + 9 files changed, 191 insertions(+), 29 deletions(-) create mode 100644 app/audio/packedprocessor.cpp create mode 100644 app/audio/packedprocessor.h diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index dd5a92d48..a03a4d88e 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -16,15 +16,17 @@ set(OLIVE_SOURCES ${OLIVE_SOURCES} - audio/audiomanager.h audio/audiomanager.cpp - audio/audiovisualwaveform.h + audio/audiomanager.h audio/audiovisualwaveform.cpp - audio/outputdeviceproxy.h + audio/audiovisualwaveform.h audio/outputdeviceproxy.cpp - audio/outputmanager.h + audio/outputdeviceproxy.h audio/outputmanager.cpp - audio/tempoprocessor.h + audio/outputmanager.h + audio/packedprocessor.cpp + audio/packedprocessor.h audio/tempoprocessor.cpp + audio/tempoprocessor.h PARENT_SCOPE ) diff --git a/app/audio/packedprocessor.cpp b/app/audio/packedprocessor.cpp new file mode 100644 index 000000000..10eb9ea18 --- /dev/null +++ b/app/audio/packedprocessor.cpp @@ -0,0 +1,106 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "packedprocessor.h" + +#include "common/ffmpegutils.h" + +namespace olive { + +PackedProcessor::PackedProcessor() : + swr_ctx_(nullptr) +{ +} + +PackedProcessor::~PackedProcessor() +{ + Close(); +} + +bool PackedProcessor::Open(const AudioParams ¶ms) +{ + if (IsOpen()) { + return true; + } + + swr_ctx_ = swr_alloc_set_opts(nullptr, + params.channel_layout(), + FFmpegUtils::GetFFmpegSampleFormat(params.format(), false), + params.sample_rate(), + params.channel_layout(), + FFmpegUtils::GetFFmpegSampleFormat(params.format(), true), + params.sample_rate(), + 0, + nullptr); + + if (!swr_ctx_) { + qCritical() << "Failed to allocate resample context"; + return false; + } + + if (swr_init(swr_ctx_) < 0) { + qCritical() << "Failed to init resample context"; + swr_free(&swr_ctx_); + return false; + } + + return true; +} + +QByteArray PackedProcessor::Convert(SampleBufferPtr planar) +{ + if (!IsOpen()) { + qCritical() << "Tried to convert while closed"; + return QByteArray(); + } + + int nb_samples = planar->sample_count(); + if (nb_samples == 0) { + return QByteArray(); + } + + int nb_channels = planar->audio_params().channel_count(); + + QByteArray output(planar->audio_params().samples_to_bytes(nb_samples), Qt::Uninitialized); + uint8_t *output_data = reinterpret_cast(output.data()); + + QVector input_arrays(nb_channels); + for (int i=0; i(planar->data(i)); + } + + int ret = swr_convert(swr_ctx_, &output_data, nb_samples, input_arrays.data(), nb_samples); + if (ret < 0) { + char buf[200]; + av_strerror(ret, buf, 200); + qDebug() << "Packed processor failed with error:" << buf << ret; + } + + return output; +} + +void PackedProcessor::Close() +{ + if (swr_ctx_) { + swr_free(&swr_ctx_); + } +} + +} diff --git a/app/audio/packedprocessor.h b/app/audio/packedprocessor.h new file mode 100644 index 000000000..8b1bb59c5 --- /dev/null +++ b/app/audio/packedprocessor.h @@ -0,0 +1,60 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef PACKEDPROCESSOR_H +#define PACKEDPROCESSOR_H + +extern "C" { +#include +} + +#include "codec/samplebuffer.h" +#include "render/audioparams.h" + +namespace olive { + +class PackedProcessor +{ +public: + PackedProcessor(); + + ~PackedProcessor(); + + DISABLE_COPY_MOVE(PackedProcessor) + + bool Open(const AudioParams ¶ms); + + QByteArray Convert(SampleBufferPtr planar); + + void Close(); + + bool IsOpen() const + { + return swr_ctx_; + } + +private: + SwrContext *swr_ctx_; + +}; + +} + +#endif // PACKEDPROCESSOR_H diff --git a/app/audio/tempoprocessor.cpp b/app/audio/tempoprocessor.cpp index e45f51bec..0a0aa777e 100644 --- a/app/audio/tempoprocessor.cpp +++ b/app/audio/tempoprocessor.cpp @@ -39,7 +39,11 @@ TempoProcessor::TempoProcessor() : processed_frame_(nullptr), open_(false) { +} +TempoProcessor::~TempoProcessor() +{ + Close(); } bool TempoProcessor::IsOpen() const diff --git a/app/audio/tempoprocessor.h b/app/audio/tempoprocessor.h index cdbda48df..bac242751 100644 --- a/app/audio/tempoprocessor.h +++ b/app/audio/tempoprocessor.h @@ -42,6 +42,10 @@ class TempoProcessor public: TempoProcessor(); + ~TempoProcessor(); + + DISABLE_COPY_MOVE(TempoProcessor) + bool IsOpen() const; const double& GetSpeed() const; diff --git a/app/codec/samplebuffer.cpp b/app/codec/samplebuffer.cpp index 5a60c49a2..df00e0f7f 100644 --- a/app/codec/samplebuffer.cpp +++ b/app/codec/samplebuffer.cpp @@ -237,23 +237,4 @@ void SampleBuffer::set(int channel, const float *data, int sample_offset, int sa memcpy(&data_[channel].data()[sample_offset], data, sizeof(float) * sample_length); } -QByteArray SampleBuffer::toPackedData() const -{ - QByteArray packed_data; - - if (is_allocated()) { - packed_data.resize(audio_params_.samples_to_bytes(sample_count_per_channel_)); - - float* output_data = reinterpret_cast(packed_data.data()); - - for (int j=0;jHasResult()) { SampleBufferPtr samples = watcher->Get().value(); if (samples && audio_playback_device_) { - qint64 t = QDateTime::currentMSecsSinceEpoch(); - QByteArray pack = samples->toPackedData(); - qDebug() << "Packing took:" << (QDateTime::currentMSecsSinceEpoch() - t); + if (!packed_processor_.IsOpen()) { + packed_processor_.Open(samples->audio_params()); + } + + // Convert to packed data for audio output + QByteArray pack = packed_processor_.Convert(samples); + audio_playback_device_->Push(pack); if (prequeuing_audio_) { @@ -627,6 +631,7 @@ void ViewerWidget::PauseInternal() audio_playback_device_.reset(nullptr); qDeleteAll(audio_playback_queue_); audio_playback_queue_.clear(); + packed_processor_.Close(); UpdateTextureFromNode(); } diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index a0e91f898..7844bb555 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -28,6 +28,7 @@ #include #include +#include "audio/packedprocessor.h" #include "audiowaveformview.h" #include "common/rational.h" #include "node/output/viewer/viewer.h" @@ -257,6 +258,7 @@ private: std::unique_ptr audio_playback_device_; std::list audio_playback_queue_; rational audio_playback_queue_time_; + PackedProcessor packed_processor_; static QVector instances_; From 6f3f7e558b84dd2981f113a6540249349a3e777d Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sat, 25 Sep 2021 13:29:30 -0700 Subject: [PATCH 04/21] use shared ptr for audio device Prevents deletion before audio output has received stop signal --- app/audio/audiomanager.cpp | 4 ++-- app/audio/audiomanager.h | 2 +- app/audio/outputdeviceproxy.cpp | 3 +-- app/audio/outputdeviceproxy.h | 4 ++-- app/audio/outputmanager.cpp | 2 +- app/audio/outputmanager.h | 4 +++- app/widget/viewer/viewer.cpp | 6 +++--- app/widget/viewer/viewer.h | 2 +- 8 files changed, 14 insertions(+), 13 deletions(-) diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index 382b6dd8d..7484b7cb3 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -94,7 +94,7 @@ void AudioManager::PushToOutput(const QByteArray &samples) emit OutputPushed(samples); } -void AudioManager::StartOutput(QIODevice *device, int playback_speed) +void AudioManager::StartOutput(std::shared_ptr device, int playback_speed) { // Move to output manager's thread device->moveToThread(&output_thread_); @@ -103,7 +103,7 @@ void AudioManager::StartOutput(QIODevice *device, int playback_speed) QMetaObject::invokeMethod(output_manager_, "PullFromDevice", Qt::QueuedConnection, - Q_ARG(QIODevice*, device), + Q_ARG(std::shared_ptr, device), Q_ARG(int, playback_speed)); } diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index 43c4edc29..fee7ef462 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -68,7 +68,7 @@ public: /** * @brief Start playing audio from AudioPlaybackCache */ - void StartOutput(QIODevice *device, int playback_speed); + void StartOutput(std::shared_ptr device, int playback_speed); /** * @brief Stop audio output immediately diff --git a/app/audio/outputdeviceproxy.cpp b/app/audio/outputdeviceproxy.cpp index 712b3f556..c6087a064 100644 --- a/app/audio/outputdeviceproxy.cpp +++ b/app/audio/outputdeviceproxy.cpp @@ -35,10 +35,9 @@ void AudioOutputDeviceProxy::SetParameters(const AudioParams ¶ms) params_ = params; } -void AudioOutputDeviceProxy::SetDevice(QIODevice* device, int playback_speed) +void AudioOutputDeviceProxy::SetDevice(std::shared_ptr device, int playback_speed) { device_ = device; - device_->setParent(this); if (!device_->open(QFile::ReadOnly)) { qCritical() << "Failed to open IO device for audio playback"; diff --git a/app/audio/outputdeviceproxy.h b/app/audio/outputdeviceproxy.h index 453d1b94c..58587ea2f 100644 --- a/app/audio/outputdeviceproxy.h +++ b/app/audio/outputdeviceproxy.h @@ -39,7 +39,7 @@ public: void SetParameters(const AudioParams& params); - void SetDevice(QIODevice *device, int playback_speed); + void SetDevice(std::shared_ptr device, int playback_speed); virtual void close() override; @@ -51,7 +51,7 @@ protected: private: qint64 ReverseAwareRead(char* data, qint64 maxlen); - QIODevice* device_; + std::shared_ptr device_; TempoProcessor tempo_processor_; diff --git a/app/audio/outputmanager.cpp b/app/audio/outputmanager.cpp index 3e032075e..9986772f8 100644 --- a/app/audio/outputmanager.cpp +++ b/app/audio/outputmanager.cpp @@ -90,7 +90,7 @@ void AudioOutputManager::Close() } } -void AudioOutputManager::PullFromDevice(QIODevice *device, int playback_speed) +void AudioOutputManager::PullFromDevice(std::shared_ptr device, int playback_speed) { if (!output_) { return; diff --git a/app/audio/outputmanager.h b/app/audio/outputmanager.h index 990303bd3..9f65dc544 100644 --- a/app/audio/outputmanager.h +++ b/app/audio/outputmanager.h @@ -53,7 +53,7 @@ public slots: * This will clear any pushed samples or QIODevices currently being read and will start reading from this next time * the audio output requests data. */ - void PullFromDevice(QIODevice* device, int playback_speed); + void PullFromDevice(std::shared_ptr device, int playback_speed); // Queued void ResetToPushMode(); @@ -86,4 +86,6 @@ private slots: } +Q_DECLARE_METATYPE(std::shared_ptr) + #endif // AUDIOHYBRIDDEVICE_H diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 29036f9ea..b0b6603ba 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -405,7 +405,7 @@ void ViewerWidget::StartAudioOutput() if (params.is_valid()) { AudioManager::instance()->SetOutputParams(params); - AudioManager::instance()->StartOutput(audio_playback_device_.get(), playback_speed_); + AudioManager::instance()->StartOutput(audio_playback_device_, playback_speed_); qDebug() << "STUB: Nothing to send to audio monitor"; /*emit AudioManager::instance()->OutputWaveformStarted(&audio_cache->visual(), @@ -606,7 +606,7 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) } } - audio_playback_device_.reset(new PreviewAudioDevice()); + audio_playback_device_ = std::make_shared(); prequeuing_audio_ = true; audio_playback_queue_time_ = GetTime(); QueueNextAudioBuffer(); @@ -628,7 +628,7 @@ void ViewerWidget::PauseInternal() playback_queue_.clear(); playback_backup_timer_.stop(); - audio_playback_device_.reset(nullptr); + audio_playback_device_ = nullptr; qDeleteAll(audio_playback_queue_); audio_playback_queue_.clear(); packed_processor_.Close(); diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 7844bb555..cca17f9d6 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -255,7 +255,7 @@ private: int active_queue_jobs_; - std::unique_ptr audio_playback_device_; + std::shared_ptr audio_playback_device_; std::list audio_playback_queue_; rational audio_playback_queue_time_; PackedProcessor packed_processor_; From db873dfc69fec46814e10a15875f86146f317ab9 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sun, 26 Sep 2021 14:22:50 -0700 Subject: [PATCH 05/21] reimplemented reversing and shuttling in new audio --- app/audio/audiomanager.cpp | 5 +- app/audio/audiomanager.h | 2 +- app/audio/outputdeviceproxy.cpp | 66 +------------------ app/audio/outputdeviceproxy.h | 8 +-- app/audio/outputmanager.cpp | 4 +- app/audio/outputmanager.h | 2 +- app/widget/viewer/viewer.cpp | 108 ++++++++++++++++++++++++-------- app/widget/viewer/viewer.h | 4 ++ 8 files changed, 95 insertions(+), 104 deletions(-) diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index 7484b7cb3..4cb9358fa 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -94,7 +94,7 @@ void AudioManager::PushToOutput(const QByteArray &samples) emit OutputPushed(samples); } -void AudioManager::StartOutput(std::shared_ptr device, int playback_speed) +void AudioManager::StartOutput(std::shared_ptr device) { // Move to output manager's thread device->moveToThread(&output_thread_); @@ -103,8 +103,7 @@ void AudioManager::StartOutput(std::shared_ptr device, int playback_s QMetaObject::invokeMethod(output_manager_, "PullFromDevice", Qt::QueuedConnection, - Q_ARG(std::shared_ptr, device), - Q_ARG(int, playback_speed)); + Q_ARG(std::shared_ptr, device)); } void AudioManager::StopOutput() diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index fee7ef462..a9ba75670 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -68,7 +68,7 @@ public: /** * @brief Start playing audio from AudioPlaybackCache */ - void StartOutput(std::shared_ptr device, int playback_speed); + void StartOutput(std::shared_ptr device); /** * @brief Stop audio output immediately diff --git a/app/audio/outputdeviceproxy.cpp b/app/audio/outputdeviceproxy.cpp index c6087a064..924cc359c 100644 --- a/app/audio/outputdeviceproxy.cpp +++ b/app/audio/outputdeviceproxy.cpp @@ -35,7 +35,7 @@ void AudioOutputDeviceProxy::SetParameters(const AudioParams ¶ms) params_ = params; } -void AudioOutputDeviceProxy::SetDevice(std::shared_ptr device, int playback_speed) +void AudioOutputDeviceProxy::SetDevice(std::shared_ptr device) { device_ = device; @@ -44,12 +44,6 @@ void AudioOutputDeviceProxy::SetDevice(std::shared_ptr device, int pl device_ = nullptr; return; } - - playback_speed_ = playback_speed; - - if (qAbs(playback_speed_) != 1) { - tempo_processor_.Open(params_, qAbs(playback_speed_)); - } } void AudioOutputDeviceProxy::close() @@ -57,10 +51,6 @@ void AudioOutputDeviceProxy::close() QIODevice::close(); device_ = nullptr; - - if (tempo_processor_.IsOpen()) { - tempo_processor_.Close(); - } } qint64 AudioOutputDeviceProxy::readData(char *data, qint64 maxlen) @@ -69,26 +59,7 @@ qint64 AudioOutputDeviceProxy::readData(char *data, qint64 maxlen) return 0; } - qint64 read_count; - - if (tempo_processor_.IsOpen()) { - - while ((read_count = tempo_processor_.Pull(data, static_cast(maxlen))) == 0) { - int dev_read = static_cast(ReverseAwareRead(data, maxlen)); - - if (!dev_read) { - break; - } - - tempo_processor_.Push(data, dev_read); - } - - } else { - // If we aren't doing any tempo processing, simply passthrough the read signal - read_count = ReverseAwareRead(data, maxlen); - } - - return read_count; + return device_->read(data, maxlen); } qint64 AudioOutputDeviceProxy::writeData(const char *data, qint64 maxSize) @@ -96,38 +67,7 @@ qint64 AudioOutputDeviceProxy::writeData(const char *data, qint64 maxSize) Q_UNUSED(data) Q_UNUSED(maxSize) - return 0; -} - -qint64 AudioOutputDeviceProxy::ReverseAwareRead(char *data, qint64 maxlen) -{ - qint64 new_pos = -1; - - if (playback_speed_ < 0) { - // If we're reversing, we'll seek back by maxlen bytes before we read - qint64 len_adjusted_by_channels = maxlen / params_.channel_count(); - - new_pos = device_->pos() - len_adjusted_by_channels; - - if (new_pos < 0) { - maxlen = device_->pos() * params_.channel_count(); - - new_pos = 0; - } - - device_->seek(new_pos); - } - - qint64 read_count = device_->read(data, maxlen); - - if (playback_speed_ < 0) { - device_->seek(new_pos); - - // Reverse the samples here - AudioManager::ReverseBuffer(data, static_cast(read_count), params_.samples_to_bytes(1)); - } - - return read_count; + return -1; } } diff --git a/app/audio/outputdeviceproxy.h b/app/audio/outputdeviceproxy.h index 58587ea2f..092776a7c 100644 --- a/app/audio/outputdeviceproxy.h +++ b/app/audio/outputdeviceproxy.h @@ -39,7 +39,7 @@ public: void SetParameters(const AudioParams& params); - void SetDevice(std::shared_ptr device, int playback_speed); + void SetDevice(std::shared_ptr device); virtual void close() override; @@ -49,16 +49,10 @@ protected: virtual qint64 writeData(const char *data, qint64 maxSize) override; private: - qint64 ReverseAwareRead(char* data, qint64 maxlen); - std::shared_ptr device_; - TempoProcessor tempo_processor_; - AudioParams params_; - int playback_speed_; - }; } diff --git a/app/audio/outputmanager.cpp b/app/audio/outputmanager.cpp index 9986772f8..9182c5040 100644 --- a/app/audio/outputmanager.cpp +++ b/app/audio/outputmanager.cpp @@ -90,7 +90,7 @@ void AudioOutputManager::Close() } } -void AudioOutputManager::PullFromDevice(std::shared_ptr device, int playback_speed) +void AudioOutputManager::PullFromDevice(std::shared_ptr device) { if (!output_) { return; @@ -102,7 +102,7 @@ void AudioOutputManager::PullFromDevice(std::shared_ptr device, int p push_samples_.clear(); // Pull from the device - device_proxy_.SetDevice(device, playback_speed); + device_proxy_.SetDevice(device); device_proxy_.open(QIODevice::ReadOnly); output_->start(&device_proxy_); } diff --git a/app/audio/outputmanager.h b/app/audio/outputmanager.h index 9f65dc544..cea14474e 100644 --- a/app/audio/outputmanager.h +++ b/app/audio/outputmanager.h @@ -53,7 +53,7 @@ public slots: * This will clear any pushed samples or QIODevices currently being read and will start reading from this next time * the audio output requests data. */ - void PullFromDevice(std::shared_ptr device, int playback_speed); + void PullFromDevice(std::shared_ptr device); // Queued void ResetToPushMode(); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index b0b6603ba..01ea71517 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -192,6 +192,9 @@ void ViewerWidget::ConnectNodeEvent(ViewerOutput *n) last_length_ = 0; LengthChangedSlot(n->GetLength()); + AudioParams ap = n->GetAudioParams(); + packed_processor_.Open(ap); + ColorManager* color_manager = n->project()->color_manager(); display_widget_->ConnectColorManager(color_manager); @@ -225,6 +228,8 @@ void ViewerWidget::DisconnectNodeEvent(ViewerOutput *n) disconnect(n->video_frame_cache(), &FrameHashCache::Shifted, this, &ViewerWidget::ViewerShiftedRange); disconnect(n, &ViewerOutput::TextureInputChanged, this, &ViewerWidget::UpdateStack); + packed_processor_.Close(); + SetDisplayImage(QVariant()); ruler()->SetPlaybackCache(nullptr); @@ -405,7 +410,7 @@ void ViewerWidget::StartAudioOutput() if (params.is_valid()) { AudioManager::instance()->SetOutputParams(params); - AudioManager::instance()->StartOutput(audio_playback_device_, playback_speed_); + AudioManager::instance()->StartOutput(audio_playback_device_); qDebug() << "STUB: Nothing to send to audio monitor"; /*emit AudioManager::instance()->OutputWaveformStarted(&audio_cache->visual(), @@ -416,13 +421,27 @@ void ViewerWidget::StartAudioOutput() void ViewerWidget::QueueNextAudioBuffer() { // NOTE: Hardcoded 2 second interval - TimeRange range(audio_playback_queue_time_, audio_playback_queue_time_ + 2); - audio_playback_queue_time_ = range.out(); + rational queue_end = audio_playback_queue_time_ + (2 * playback_speed_); + + if (playback_speed_ < 0) { + // Limit to 0 if playing in reverse + queue_end = qMax(rational(0), queue_end); + } else { + // Limit to audio length if playing forwards + queue_end = qMin(GetConnectedNode()->GetAudioLength(), queue_end); + } + + if (queue_end == audio_playback_queue_time_) { + // This will queue nothing, so stop the loop here + return; + } RenderTicketWatcher *watcher = new RenderTicketWatcher(this); connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::ReceivedAudioBufferForPlayback); audio_playback_queue_.push_back(watcher); - watcher->SetTicket(auto_cacher_.GetRangeOfAudio(range, true)); + watcher->SetTicket(auto_cacher_.GetRangeOfAudio(TimeRange(audio_playback_queue_time_, queue_end), true)); + + audio_playback_queue_time_ = queue_end; } void ViewerWidget::ReceivedAudioBufferForPlayback() @@ -434,18 +453,31 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() if (watcher->HasResult()) { SampleBufferPtr samples = watcher->Get().value(); if (samples && audio_playback_device_) { - if (!packed_processor_.IsOpen()) { - packed_processor_.Open(samples->audio_params()); + // If the samples must be reversed, reverse them now + if (playback_speed_ < 0) { + samples->reverse(); } // Convert to packed data for audio output QByteArray pack = packed_processor_.Convert(samples); - audio_playback_device_->Push(pack); + // If the tempo must be adjusted, adjust now + if (tempo_processor_.IsOpen()) { + tempo_processor_.Push(pack.data(), pack.size()); + int actual = tempo_processor_.Pull(pack.data(), pack.size()); + if (actual != pack.size()) { + pack.resize(actual); + } + } - if (prequeuing_audio_) { - prequeuing_audio_ = false; - FinishPlayPreprocess(); + // TempoProcessor may have emptied the array + if (!pack.isEmpty()) { + audio_playback_device_->Push(pack); + + if (prequeuing_audio_) { + prequeuing_audio_ = false; + FinishPlayPreprocess(); + } } } } @@ -457,6 +489,28 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() } } +void ViewerWidget::ReceivedAudioBufferForScrubbing() +{ + RenderTicketWatcher *watcher = static_cast(sender()); + + if (watcher->HasResult()) { + if (SampleBufferPtr samples = watcher->Get().value()) { + /* Fade code + const int kFadeSz = qMin(200, samples->sample_count()/4); + for (int i=0; itransform_volume_for_sample(i, amt); + samples->transform_volume_for_sample(samples->sample_count() - i - 1, amt); + }*/ + + AudioManager::instance()->SetOutputParams(samples->audio_params()); + AudioManager::instance()->PushToOutput(packed_processor_.Convert(samples)); + } + } + + delete watcher; +} + void ViewerWidget::UpdateTextureFromNode() { rational time = GetTime(); @@ -606,6 +660,9 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) } } + if (std::abs(playback_speed_) > 1) { + tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_)); + } audio_playback_device_ = std::make_shared(); prequeuing_audio_ = true; audio_playback_queue_time_ = GetTime(); @@ -631,7 +688,9 @@ void ViewerWidget::PauseInternal() audio_playback_device_ = nullptr; qDeleteAll(audio_playback_queue_); audio_playback_queue_.clear(); - packed_processor_.Close(); + if (tempo_processor_.IsOpen()) { + tempo_processor_.Close(); + } UpdateTextureFromNode(); } @@ -642,27 +701,17 @@ void ViewerWidget::PauseInternal() void ViewerWidget::PushScrubbedAudio() { - if (!IsPlaying() && GetConnectedNode() && Config::Current()["AudioScrubbing"].toBool()) { + if (!IsPlaying() && GetConnectedNode() && Config::Current()[QStringLiteral("AudioScrubbing")].toBool()) { // Get audio src device from renderer const AudioParams& params = GetConnectedNode()->audio_playback_cache()->GetParameters(); if (params.is_valid()) { - qDebug() << "STUB: Use PAC audio function directly"; - /*PreviewAudioDevice *audio_src = new PreviewAudioDevice(&auto_cacher_, GetTime()); + // NOTE: Hardcoded scrubbing interval (20ms) + rational interval = rational(50, 1000); - if (audio_src->open(QIODevice::ReadOnly)) { - // FIXME: Hardcoded scrubbing interval (20ms) - int size_of_sample = params.time_to_bytes(rational(20, 1000)); - - // Push audio - QByteArray frame_audio = audio_src->read(size_of_sample); - AudioManager::instance()->SetOutputParams(params); - AudioManager::instance()->PushToOutput(frame_audio); - - audio_src->close(); - } - - delete audio_src;*/ + RenderTicketWatcher *watcher = new RenderTicketWatcher(); + connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::ReceivedAudioBufferForScrubbing); + watcher->SetTicket(auto_cacher_.GetRangeOfAudio(TimeRange(GetTime(), GetTime() + interval), true)); } } } @@ -1308,6 +1357,11 @@ void ViewerWidget::UpdateRendererVideoParameters() void ViewerWidget::UpdateRendererAudioParameters() { + packed_processor_.Close(); + + AudioParams ap = GetConnectedNode()->GetAudioParams(); + + packed_processor_.Open(ap); } void ViewerWidget::SetZoomFromMenu(QAction *action) diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index cca17f9d6..53a23dcd7 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -29,6 +29,7 @@ #include #include "audio/packedprocessor.h" +#include "audio/tempoprocessor.h" #include "audiowaveformview.h" #include "common/rational.h" #include "node/output/viewer/viewer.h" @@ -259,6 +260,7 @@ private: std::list audio_playback_queue_; rational audio_playback_queue_time_; PackedProcessor packed_processor_; + TempoProcessor tempo_processor_; static QVector instances_; @@ -311,6 +313,8 @@ private slots: void ReceivedAudioBufferForPlayback(); + void ReceivedAudioBufferForScrubbing(); + }; } From 867123ead9c3e05fde201c1137a67c40b9c5cc54 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sun, 26 Sep 2021 14:26:19 -0700 Subject: [PATCH 06/21] added note --- app/widget/viewer/viewer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 01ea71517..1b883c933 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -491,6 +491,9 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() void ViewerWidget::ReceivedAudioBufferForScrubbing() { + // NOTE: Might be good to organize a queue for this in the event that audio takes a long time to + // keep the scrubbed chunks ordered, similar to the playback_queue_ or audio_playback_queue_ + RenderTicketWatcher *watcher = static_cast(sender()); if (watcher->HasResult()) { From 79e49d0b24c837e2745a878f1bcc700de65325c1 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sun, 26 Sep 2021 23:21:21 -0700 Subject: [PATCH 07/21] viewer: improved playback behavior --- app/widget/viewer/viewer.cpp | 44 +++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 1b883c933..7f4f590fe 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -1255,6 +1255,10 @@ void ViewerWidget::PlaybackTimerUpdate() max_time = qMax(min_time, max_time - timebase()); } + rational time_to_set; + bool end_of_line = false; + bool play_after_pause = false; + if ((playback_speed_ < 0 && current_time <= min_time) || (playback_speed_ > 0 && current_time >= max_time)) { @@ -1267,34 +1271,48 @@ void ViewerWidget::PlaybackTimerUpdate() tripped_time = max_time; } + // Signal that we've reached the end of whatever range we're playing and should either pause + // or restart playback + end_of_line = true; + if (Config::Current()[QStringLiteral("Loop")].toBool()) { // If we're looping, jump to the other side of the workarea and continue - rational opposing_time = (tripped_time == min_time) ? max_time : min_time; + time_to_set = (tripped_time == min_time) ? max_time : min_time; - // Cache the current speed - int current_speed = playback_speed_; - - // Jump to the other side and keep playing at the same speed - SetTimeAndSignal(opposing_time); - PlayInternal(current_speed, play_in_to_out_only_); + // Signal to restart playback after the pause signalled by `end_of_line` + play_after_pause = true; } else { - // Pause at the boundary - SetTimeAndSignal(tripped_time); + // Pause at the boundary we tripped + time_to_set = tripped_time; } } else { - // Sets time, wrapping in this bool ensures we don't pause from setting the time - time_changed_from_timer_ = true; - SetTimeAndSignal(current_time); - time_changed_from_timer_ = false; + // Sets time normally to whatever we calculated as the "current time" + time_to_set = current_time; } + // Set the time. By wrapping in this bool, we prevent TimeChangedEvent's default behavior of + // pausing. Even if we pause it later with `end_of_line`, we prefer pausing after setting the time + // so that an audio scrub event, etc. isn't sent. + time_changed_from_timer_ = true; + SetTimeAndSignal(time_to_set); + time_changed_from_timer_ = false; + if (end_of_line) { + // Cache the current speed + int current_speed = playback_speed_; + + PauseInternal(); + if (play_after_pause) { + PlayInternal(current_speed, play_in_to_out_only_); + } + } + if (display_widget_->isVisible()) { // Updating display widget UpdateTextureFromNode(); From afe02c1033fe3e4fb94afa3e710e1584553a3b1f Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Mon, 27 Sep 2021 00:38:08 -0700 Subject: [PATCH 08/21] viewer: use function pointer Possibly slightly faster --- CMakeLists.txt | 5 +++++ app/widget/viewer/viewer.cpp | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b73ca1160..3b4540f48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,6 +122,11 @@ list(APPEND OLIVE_LIBRARIES FFMPEG::swresample ) +find_package(PortAudio REQUIRED) +list(APPEND OLIVE_LIBRARIES + portaudio +) + # Optional: Link OpenTimelineIO find_package(OpenTimelineIO) if (OpenTimelineIO_FOUND) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 7f4f590fe..a114099c8 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -246,7 +246,7 @@ void ViewerWidget::DisconnectNodeEvent(ViewerOutput *n) waveform_view_->ConnectTimelinePoints(nullptr); // Queue an UpdateStack so that when it runs, the viewer node will be fully disconnected - QMetaObject::invokeMethod(this, "UpdateStack", Qt::QueuedConnection); + QMetaObject::invokeMethod(this, &ViewerWidget::UpdateStack, Qt::QueuedConnection); } void ViewerWidget::ConnectedNodeChangeEvent(ViewerOutput *n) @@ -1394,7 +1394,7 @@ void ViewerWidget::ViewerInvalidatedVideoRange(const TimeRange &range) { // If our current frame is within this range, we need to update if (GetTime() >= range.in() && (GetTime() < range.out() || range.in() == range.out())) { - QMetaObject::invokeMethod(this, "UpdateTextureFromNode", Qt::QueuedConnection); + QMetaObject::invokeMethod(this, &ViewerWidget::UpdateTextureFromNode, Qt::QueuedConnection); } } @@ -1410,7 +1410,7 @@ void ViewerWidget::ManualSwitchToWaveform(bool e) void ViewerWidget::ViewerShiftedRange(const rational &from, const rational &to) { if (GetTime() >= qMin(from, to)) { - QMetaObject::invokeMethod(this, "UpdateTextureFromNode", Qt::QueuedConnection); + QMetaObject::invokeMethod(this, &ViewerWidget::UpdateTextureFromNode, Qt::QueuedConnection); } } From 337cb4838dab83c003e34447a28548c0095e58b4 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Mon, 27 Sep 2021 14:58:45 -0700 Subject: [PATCH 09/21] viewer: use clamp Cleaner code --- app/widget/viewer/viewer.cpp | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index a114099c8..3eb589b37 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -31,6 +31,7 @@ #include #include "audio/audiomanager.h" +#include "common/clamp.h" #include "common/power.h" #include "common/ratiodialog.h" #include "common/timecodefunctions.h" @@ -423,14 +424,8 @@ void ViewerWidget::QueueNextAudioBuffer() // NOTE: Hardcoded 2 second interval rational queue_end = audio_playback_queue_time_ + (2 * playback_speed_); - if (playback_speed_ < 0) { - // Limit to 0 if playing in reverse - queue_end = qMax(rational(0), queue_end); - } else { - // Limit to audio length if playing forwards - queue_end = qMin(GetConnectedNode()->GetAudioLength(), queue_end); - } - + // Clamp queue end by zero and the audio length + queue_end = clamp(queue_end, rational(0), GetConnectedNode()->GetAudioLength()); if (queue_end == audio_playback_queue_time_) { // This will queue nothing, so stop the loop here return; From c29939b52371d848a1590f4edb98f3837a33eecc Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 21:35:24 -0700 Subject: [PATCH 10/21] use audio waveform constants to determine min/max zoom levels --- app/audio/audiovisualwaveform.cpp | 16 +++++++--------- app/audio/audiovisualwaveform.h | 4 ++++ app/widget/nodeparamview/nodeparamview.cpp | 2 -- app/widget/timebased/timescaledobject.cpp | 6 ++---- app/widget/timebased/timescaledobject.h | 3 +-- app/widget/timelinewidget/tool/zoom.cpp | 2 +- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/app/audio/audiovisualwaveform.cpp b/app/audio/audiovisualwaveform.cpp index ab590fe41..5130fe46a 100644 --- a/app/audio/audiovisualwaveform.cpp +++ b/app/audio/audiovisualwaveform.cpp @@ -27,13 +27,12 @@ namespace olive { +const rational AudioVisualWaveform::kMinimumSampleRate = rational(1, 8); +const rational AudioVisualWaveform::kMaximumSampleRate = 1024; + AudioVisualWaveform::AudioVisualWaveform() : channels_(0) { - // Must be a power of 2 - static const rational kMinimumSampleRate = rational(1, 8); - static const rational kMaximumSampleRate = 1024; - for (rational i=kMinimumSampleRate; i<=kMaximumSampleRate; i*=2) { mipmapped_data_.insert({i, Sample()}); } @@ -431,15 +430,14 @@ int AudioVisualWaveform::time_to_samples(const double &time, double sample_rate) std::map::const_iterator AudioVisualWaveform::GetMipmapForScale(double scale) const { // Find largest mipmap for this scale (or the largest if we don't find one sufficient) - auto using_mipmap = mipmapped_data_.cend(); - using_mipmap--; for (auto it=mipmapped_data_.cbegin(); it!=mipmapped_data_.cend(); it++) { if (it->first.toDouble() >= scale) { - using_mipmap = it; - break; + return it; } } - return using_mipmap; + + // We don't have a mipmap large enough for this scale, so just return the largest we have + return std::prev(mipmapped_data_.cend()); } void AudioVisualWaveform::ExpandMinMax(AudioVisualWaveform::SamplePerChannel &sum, float value) diff --git a/app/audio/audiovisualwaveform.h b/app/audio/audiovisualwaveform.h index 33cce0fdd..ce74f9b4a 100644 --- a/app/audio/audiovisualwaveform.h +++ b/app/audio/audiovisualwaveform.h @@ -107,6 +107,10 @@ public: static void DrawWaveform(QPainter* painter, const QRect &rect, const double &scale, const AudioVisualWaveform& samples, const rational &start_time); + // Must be a power of 2 + static const rational kMinimumSampleRate; + static const rational kMaximumSampleRate; + private: static void ExpandMinMax(SamplePerChannel &sum, float value); diff --git a/app/widget/nodeparamview/nodeparamview.cpp b/app/widget/nodeparamview/nodeparamview.cpp index a97c76b3a..9d3e77000 100644 --- a/app/widget/nodeparamview/nodeparamview.cpp +++ b/app/widget/nodeparamview/nodeparamview.cpp @@ -128,8 +128,6 @@ NodeParamView::NodeParamView(QWidget *parent) : // Set a default scale - FIXME: Hardcoded SetScale(120); - SetMaximumScale(TimeBasedView::kMaximumScale); - // Pickup on widget focus changes connect(qApp, &QApplication::focusChanged, diff --git a/app/widget/timebased/timescaledobject.cpp b/app/widget/timebased/timescaledobject.cpp index 93da7c171..85b154a48 100644 --- a/app/widget/timebased/timescaledobject.cpp +++ b/app/widget/timebased/timescaledobject.cpp @@ -23,19 +23,17 @@ #include #include +#include "audio/audiovisualwaveform.h" #include "common/clamp.h" namespace olive { -// Keep this aligned with the kMaximumSampleRate in AudioVisualWaveform -const double TimeScaledObject::kMaximumScale = 1024; - const int TimeScaledObject::kCalculateDimensionsPadding = 10; TimeScaledObject::TimeScaledObject() : scale_(1.0), min_scale_(0), - max_scale_(kMaximumScale) + max_scale_(AudioVisualWaveform::kMaximumSampleRate.toDouble()) { } diff --git a/app/widget/timebased/timescaledobject.h b/app/widget/timebased/timescaledobject.h index cf734fb60..8c858c8fc 100644 --- a/app/widget/timebased/timescaledobject.h +++ b/app/widget/timebased/timescaledobject.h @@ -44,6 +44,7 @@ public: static rational SceneToTime(const double &x, const double& x_scale, const rational& timebase, bool round = false); const double& GetScale() const; + const double &GetMaximumScale() const { return max_scale_; } void SetScale(const double& scale); @@ -54,8 +55,6 @@ public: double TimeToScene(const rational& time) const; rational SceneToTime(const double &x, bool round = false) const; - static const double kMaximumScale; - protected: virtual void TimebaseChangedEvent(const rational&){} diff --git a/app/widget/timelinewidget/tool/zoom.cpp b/app/widget/timelinewidget/tool/zoom.cpp index 60f85594f..9b6731fe2 100644 --- a/app/widget/timelinewidget/tool/zoom.cpp +++ b/app/widget/timelinewidget/tool/zoom.cpp @@ -68,7 +68,7 @@ void ZoomTool::MouseRelease(TimelineViewMouseEvent *event) // Normalize scale to 1.0 scale double scene_width = (scene_right - scene_left) / parent()->GetScale(); - double new_scale = qMin(TimeBasedView::kMaximumScale, static_cast(reference_view->viewport()->width()) / scene_width); + double new_scale = qMin(parent()->GetFirstTimelineView()->GetMaximumScale(), static_cast(reference_view->viewport()->width()) / scene_width); parent()->SetScale(new_scale); From 40a1bba0e62f95bf9caddd4d863c702bb28a3037 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 21:37:52 -0700 Subject: [PATCH 11/21] cache: fix bug where small zoom levels wouldn't be mipmapped --- app/render/previewautocacher.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index 45c5483b1..d8ee8a129 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -652,8 +652,9 @@ void PreviewAutoCacher::TryRender() // Copy first range in list TimeRange r = audio_iterator_.first(); - // Limit to 1 second (FIXME: Hardcoded) - r.set_out(qMin(r.out(), r.in() + 1)); + // Limit to the minimum sample rate supported by AudioVisualWaveform - we use this value so that + // whatever chunk we render can be summed down to the smallest mipmap whole + r.set_out(qMin(r.out(), r.in() + AudioVisualWaveform::kMinimumSampleRate.flipped())); // Start job RenderAudio(r, true, false); From 90da62900c45a064b3fca3703946dcfeecca154f Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 21:38:25 -0700 Subject: [PATCH 12/21] viewer: improved realtime audio playback --- app/widget/viewer/viewer.cpp | 18 ++++++++++++++---- app/widget/viewer/viewer.h | 2 ++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 3eb589b37..99463d529 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -47,6 +47,7 @@ namespace olive { #define super TimeBasedWidget QVector ViewerWidget::instances_; +const int ViewerWidget::kAudioPlaybackInterval = 2; const int kMaxPreQueueSize = 8; @@ -128,6 +129,11 @@ ViewerWidget::ViewerWidget(QWidget *parent) : instances_.append(this); setAcceptDrops(true); + + audio_queue_next_timer_ = new QTimer(this); + audio_queue_next_timer_->setInterval(kAudioPlaybackInterval * 1000); + audio_queue_next_timer_->setSingleShot(true); + connect(audio_queue_next_timer_, &QTimer::timeout, this, &ViewerWidget::QueueNextAudioBuffer); } ViewerWidget::~ViewerWidget() @@ -422,7 +428,7 @@ void ViewerWidget::StartAudioOutput() void ViewerWidget::QueueNextAudioBuffer() { // NOTE: Hardcoded 2 second interval - rational queue_end = audio_playback_queue_time_ + (2 * playback_speed_); + rational queue_end = audio_playback_queue_time_ + (kAudioPlaybackInterval * playback_speed_); // Clamp queue end by zero and the audio length queue_end = clamp(queue_end, rational(0), GetConnectedNode()->GetAudioLength()); @@ -441,7 +447,7 @@ void ViewerWidget::QueueNextAudioBuffer() void ViewerWidget::ReceivedAudioBufferForPlayback() { - while (!audio_playback_queue_.empty() && !audio_playback_queue_.front()->IsRunning()) { + while (!audio_playback_queue_.empty() && audio_playback_queue_.front()->HasResult()) { RenderTicketWatcher *watcher = audio_playback_queue_.front(); audio_playback_queue_.pop_front(); @@ -478,7 +484,8 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() } // Do this in the loop so that clearing the array effectively prevents a queue - QueueNextAudioBuffer(); + audio_queue_next_timer_->stop(); + audio_queue_next_timer_->start(); delete watcher; } @@ -664,7 +671,9 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) audio_playback_device_ = std::make_shared(); prequeuing_audio_ = true; audio_playback_queue_time_ = GetTime(); - QueueNextAudioBuffer(); + for (int i=0; i<2; i++) { + QueueNextAudioBuffer(); + } } void ViewerWidget::PauseInternal() @@ -689,6 +698,7 @@ void ViewerWidget::PauseInternal() if (tempo_processor_.IsOpen()) { tempo_processor_.Close(); } + audio_queue_next_timer_->stop(); UpdateTextureFromNode(); } diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 53a23dcd7..8ff8ddacd 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -261,6 +261,8 @@ private: rational audio_playback_queue_time_; PackedProcessor packed_processor_; TempoProcessor tempo_processor_; + static const int kAudioPlaybackInterval; + QTimer *audio_queue_next_timer_; static QVector instances_; From 6df9735e4962d7c5d5f4f12d548fab68de8df9bc Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 21:39:06 -0700 Subject: [PATCH 13/21] cache: separated waveform and pcm writing stages --- app/render/audioplaybackcache.cpp | 24 +++++++++++++++--------- app/render/audioplaybackcache.h | 4 +++- app/render/previewautocacher.cpp | 21 +++++++++++++++------ app/render/previewautocacher.h | 2 ++ 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/app/render/audioplaybackcache.cpp b/app/render/audioplaybackcache.cpp index 93b3c592a..995658d26 100644 --- a/app/render/audioplaybackcache.cpp +++ b/app/render/audioplaybackcache.cpp @@ -58,7 +58,7 @@ void AudioPlaybackCache::SetParameters(const AudioParams ¶ms) emit ParametersChanged(); } -void AudioPlaybackCache::WritePCM(const TimeRange &range, const TimeRangeList &valid_ranges, SampleBufferPtr samples, const AudioVisualWaveform *waveform) +void AudioPlaybackCache::WritePCM(const TimeRange &range, const TimeRangeList &valid_ranges, SampleBufferPtr samples) { // Ensure if we have enough segments to write this data, creating more if not qint64 length_diff = params_.time_to_bytes_per_channel(range.out()) - playlist_.GetLength(); @@ -143,13 +143,6 @@ void AudioPlaybackCache::WritePCM(const TimeRange &range, const TimeRangeList &v // Each segment is contiguous, so this out will be the next segment's in this_segment_in = this_segment_out; } - - // Write visual - if (waveform) { - visual_.OverwriteSums(*waveform, r.in(), r.in() - range.in(), r.length()); - } else { - visual_.OverwriteSilence(r.in(), r.length()); - } } foreach (const TimeRange& v, ranges_we_validated) { @@ -157,11 +150,24 @@ void AudioPlaybackCache::WritePCM(const TimeRange &range, const TimeRangeList &v } } +void AudioPlaybackCache::WriteWaveform(const TimeRange &range, const TimeRangeList &valid_ranges, const AudioVisualWaveform *waveform) +{ + // Write each valid range to the segments + foreach (const TimeRange& r, valid_ranges) { + // Write visual + if (waveform) { + visual_.OverwriteSums(*waveform, r.in(), r.in() - range.in(), r.length()); + } else { + visual_.OverwriteSilence(r.in(), r.length()); + } + } +} + void AudioPlaybackCache::WriteSilence(const TimeRange &range) { // WritePCM will automatically fill non-existent bytes with silence, so we just have to send // it an empty sample buffer - WritePCM(range, {range}, nullptr, nullptr); + WritePCM(range, {range}, nullptr); } void AudioPlaybackCache::ShiftEvent(const rational &from_in_time, const rational &to_in_time) diff --git a/app/render/audioplaybackcache.h b/app/render/audioplaybackcache.h index ee697808d..8394975a9 100644 --- a/app/render/audioplaybackcache.h +++ b/app/render/audioplaybackcache.h @@ -66,7 +66,9 @@ public: void SetParameters(const AudioParams& params); - void WritePCM(const TimeRange &range, const TimeRangeList &valid_ranges, SampleBufferPtr samples, const AudioVisualWaveform *waveform); + void WritePCM(const TimeRange &range, const TimeRangeList &valid_ranges, SampleBufferPtr samples); + + void WriteWaveform(const TimeRange &range, const TimeRangeList &valid_ranges, const AudioVisualWaveform *waveform); void WriteSilence(const TimeRange &range); diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index d8ee8a129..daf18aac7 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -32,6 +32,10 @@ namespace olive { +// We may want to make this configurable at some point, so for now this constant is used as a +// placeholder for where that configarable variable would be used. +const bool PreviewAutoCacher::kRealTimeWaveformsEnabled = true; + PreviewAutoCacher::PreviewAutoCacher() : viewer_node_(nullptr), use_custom_range_(false), @@ -141,7 +145,9 @@ void PreviewAutoCacher::AudioInvalidated(const TimeRange &range) // cancelled, so some areas may end up unrendered forever // ClearAudioQueue(); - if (viewer_node_->GetAudioAutoCacheEnabled()) { + // If we're auto-caching audio or require realtime waveforms, we'll have to render this + if (viewer_node_->GetAudioAutoCacheEnabled() || kRealTimeWaveformsEnabled) { + // We still render for the sake of waveforms audio_job_tracker_.insert(range, graph_changed_time_); // Start jobs to re-render the audio at this range, split into 2 second chunks @@ -203,11 +209,14 @@ void PreviewAutoCacher::AudioRendered() AudioVisualWaveform waveform = watcher->GetTicket()->property("waveform").value(); - // WritePCM is tolerant to its buffer being null, it will just write silence instead - viewer_node_->audio_playback_cache()->WritePCM(range, - valid_ranges, - watcher->Get().value(), - &waveform); + if (viewer_node_->GetAudioAutoCacheEnabled()) { + // WritePCM is tolerant to its buffer being null, it will just write silence instead + viewer_node_->audio_playback_cache()->WritePCM(range, + valid_ranges, + watcher->Get().value()); + } + + viewer_node_->audio_playback_cache()->WriteWaveform(range, valid_ranges, &waveform); // Detect if this audio was incomplete because it was waiting on a conform to finish if (watcher->GetTicket()->property("incomplete").toBool()) { diff --git a/app/render/previewautocacher.h b/app/render/previewautocacher.h index a4d15cf82..150a45e48 100644 --- a/app/render/previewautocacher.h +++ b/app/render/previewautocacher.h @@ -192,6 +192,8 @@ private: TimeRangeListFrameIterator hash_iterator_; TimeRangeList audio_iterator_; + static const bool kRealTimeWaveformsEnabled; + private slots: /** * @brief Handler for when the NodeGraph reports a video change over a certain time range From 68033b88285706ba0eeb3598f1c7c58c925d04cd Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 21:39:17 -0700 Subject: [PATCH 14/21] viewer: restore original audio monitor behavior --- app/widget/viewer/viewer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 99463d529..3c3c48263 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -419,9 +419,8 @@ void ViewerWidget::StartAudioOutput() AudioManager::instance()->SetOutputParams(params); AudioManager::instance()->StartOutput(audio_playback_device_); - qDebug() << "STUB: Nothing to send to audio monitor"; - /*emit AudioManager::instance()->OutputWaveformStarted(&audio_cache->visual(), - GetTime(), playback_speed_);*/ + emit AudioManager::instance()->OutputWaveformStarted(&GetConnectedNode()->audio_playback_cache()->visual(), + GetTime(), playback_speed_); } } From e4ee364e77e61f6f9ca977d9ce662c93aa8ac8c3 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 21:39:37 -0700 Subject: [PATCH 15/21] viewer: remove debug line This has served its purpose, and we have a better UI message now. --- app/widget/viewer/viewer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 3c3c48263..acf40fb30 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -571,7 +571,7 @@ void ViewerWidget::UpdateTextureFromNode() // Only show warning if frame actually exists if (frame_exists_at_time && !frame_might_be_still) { - qWarning() << "Playback queue failed to keep up"; + //qWarning() << "Playback queue failed to keep up"; } } From 59d9307adbc2ffc92b5ad4610d52eefc06ed210b Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 21:39:56 -0700 Subject: [PATCH 16/21] cache; removed unnecessary comment --- app/render/previewautocacher.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index daf18aac7..d06b8b499 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -147,7 +147,6 @@ void PreviewAutoCacher::AudioInvalidated(const TimeRange &range) // If we're auto-caching audio or require realtime waveforms, we'll have to render this if (viewer_node_->GetAudioAutoCacheEnabled() || kRealTimeWaveformsEnabled) { - // We still render for the sake of waveforms audio_job_tracker_.insert(range, graph_changed_time_); // Start jobs to re-render the audio at this range, split into 2 second chunks From 909ac230f225fbcb5ac807364a6a76cb2fdb9a9b Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 22:39:24 -0700 Subject: [PATCH 17/21] cmake: we'll re-add this later --- CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b4540f48..b73ca1160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,11 +122,6 @@ list(APPEND OLIVE_LIBRARIES FFMPEG::swresample ) -find_package(PortAudio REQUIRED) -list(APPEND OLIVE_LIBRARIES - portaudio -) - # Optional: Link OpenTimelineIO find_package(OpenTimelineIO) if (OpenTimelineIO_FOUND) From d5c3735f94fbb37f37202dec192d748017be9d04 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 23:14:07 -0700 Subject: [PATCH 18/21] cache: improve performance --- app/render/previewautocacher.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index d06b8b499..2884b226c 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -130,8 +130,8 @@ void PreviewAutoCacher::VideoInvalidated(const TimeRange &range) // want to dedicate all our rendering power to realtime feedback for the user CancelVideoTasks(); - // If a slider is not being dragged, queue up to hash these frames - if (!NodeInputDragger::IsInputBeingDragged()) { + // If auto-cache is enabled and a slider is not being dragged, queue up to hash these frames + if (viewer_node_->GetVideoAutoCacheEnabled() && !NodeInputDragger::IsInputBeingDragged()) { invalidated_video_.insert(range); video_job_tracker_.insert(range, graph_changed_time_); From 64980a6891a0de5fa077cf5fed8845aa512fef7c Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 28 Sep 2021 23:45:20 -0700 Subject: [PATCH 19/21] Revert "cache: improve performance" This reverts commit d5c3735f94fbb37f37202dec192d748017be9d04. --- app/render/previewautocacher.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index 2884b226c..d06b8b499 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -130,8 +130,8 @@ void PreviewAutoCacher::VideoInvalidated(const TimeRange &range) // want to dedicate all our rendering power to realtime feedback for the user CancelVideoTasks(); - // If auto-cache is enabled and a slider is not being dragged, queue up to hash these frames - if (viewer_node_->GetVideoAutoCacheEnabled() && !NodeInputDragger::IsInputBeingDragged()) { + // If a slider is not being dragged, queue up to hash these frames + if (!NodeInputDragger::IsInputBeingDragged()) { invalidated_video_.insert(range); video_job_tracker_.insert(range, graph_changed_time_); From a7777402328a93ae7bae97499f8c560b8ea8acef Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Wed, 29 Sep 2021 00:15:05 -0700 Subject: [PATCH 20/21] audiovisualwaveform: avoid index out of bounds --- app/audio/audiovisualwaveform.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/audio/audiovisualwaveform.cpp b/app/audio/audiovisualwaveform.cpp index 5130fe46a..1ee61425b 100644 --- a/app/audio/audiovisualwaveform.cpp +++ b/app/audio/audiovisualwaveform.cpp @@ -277,7 +277,19 @@ AudioVisualWaveform::Sample AudioVisualWaveform::GetSummaryFromTime(const ration int start_sample = time_to_samples(start, rate_dbl); int sample_length = time_to_samples(length, rate_dbl); - return ReSumSamples(&using_mipmap->second.constData()[start_sample], sample_length, channels_); + const QVector &mipmap_data = using_mipmap->second; + + // Determine if the array actually has this sample + sample_length = qMin(sample_length, mipmap_data.size() - start_sample); + + // Based on the above `min`, if sample length <= 0, that means start_sample >= the size of the + // array and nothing can be returned. + if (sample_length > 0) { + return ReSumSamples(&mipmap_data.constData()[start_sample], sample_length, channels_); + } + + // Return null samples + return AudioVisualWaveform::Sample(channel_count(), {0, 0}); } AudioVisualWaveform::Sample AudioVisualWaveform::SumSamples(const float *samples, int nb_samples, int nb_channels) From 2ea2b0a51ea31067e9b9bde9185a729067f386d4 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Wed, 29 Sep 2021 00:17:30 -0700 Subject: [PATCH 21/21] Revert "Revert "cache: improve performance"" This reverts commit 64980a6891a0de5fa077cf5fed8845aa512fef7c. --- app/render/previewautocacher.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index d06b8b499..2884b226c 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -130,8 +130,8 @@ void PreviewAutoCacher::VideoInvalidated(const TimeRange &range) // want to dedicate all our rendering power to realtime feedback for the user CancelVideoTasks(); - // If a slider is not being dragged, queue up to hash these frames - if (!NodeInputDragger::IsInputBeingDragged()) { + // If auto-cache is enabled and a slider is not being dragged, queue up to hash these frames + if (viewer_node_->GetVideoAutoCacheEnabled() && !NodeInputDragger::IsInputBeingDragged()) { invalidated_video_.insert(range); video_job_tracker_.insert(range, graph_changed_time_);