From 558773538d8f3a13b49d912563bb396d1260ef6f Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 18 May 2020 17:16:28 +1000 Subject: [PATCH] cache: fixed audio with new system --- app/node/output/track/tracklist.cpp | 1 - app/render/audioplaybackcache.cpp | 1 - app/render/backend/renderbackend.cpp | 124 ++++++++++++++++++++------- app/render/backend/renderbackend.h | 31 +++++-- app/render/backend/renderworker.cpp | 16 +--- app/task/cache/cache.cpp | 1 + app/widget/viewer/viewer.cpp | 10 +-- 7 files changed, 121 insertions(+), 63 deletions(-) diff --git a/app/node/output/track/tracklist.cpp b/app/node/output/track/tracklist.cpp index 0c7feb627..1bf9b4246 100644 --- a/app/node/output/track/tracklist.cpp +++ b/app/node/output/track/tracklist.cpp @@ -192,7 +192,6 @@ void TrackList::TrackConnected(NodeEdgePtr edge) connect(connected_track, &TrackOutput::TrackLengthChanged, this, &TrackList::UpdateTotalLength); connect(connected_track, &TrackOutput::TrackHeightChanged, this, &TrackList::TrackHeightChangedSlot); - qDebug() << "Setting track type" << connected_track << type_; connected_track->set_track_type(type_); emit TrackListChanged(); diff --git a/app/render/audioplaybackcache.cpp b/app/render/audioplaybackcache.cpp index 4be298478..a48d3c3eb 100644 --- a/app/render/audioplaybackcache.cpp +++ b/app/render/audioplaybackcache.cpp @@ -60,7 +60,6 @@ void AudioPlaybackCache::SetParameters(const AudioRenderingParams ¶ms) void AudioPlaybackCache::WritePCM(const TimeRange &range, SampleBufferPtr samples) { QFile f(filename_); - qDebug() << "Writing PCM to" << filename_; if (f.open(QFile::ReadWrite)) { qint64 start_offset = params_.time_to_bytes(range.in()); qint64 max_len = params_.time_to_bytes(range.out()); diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index e2dcc7675..11a3a41d4 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -34,8 +34,7 @@ OLIVE_NAMESPACE_ENTER RenderBackend::RenderBackend(QObject *parent) : QObject(parent), viewer_node_(nullptr), - video_instance_queuer_(0), - audio_instance_queuer_(0), + audio_enabled_(true), divider_(1), render_mode_(RenderMode::kOnline), pix_fmt_(PixelFormat::PIX_FMT_RGBA32F), @@ -62,9 +61,10 @@ void RenderBackend::SetViewerNode(ViewerOutput *viewer_node) CancelQueue(); // Delete all of our copied nodes - foreach (RenderWorker* instance, video_instance_pool_) { - instance->Close(); - } + hash_pool_.Close(); + video_pool_.Close(); + audio_pool_.Close(); + queued_audio_.clear(); disconnect(viewer_node_, &ViewerOutput::GraphChangedFrom, @@ -82,32 +82,42 @@ void RenderBackend::SetViewerNode(ViewerOutput *viewer_node) if (viewer_node_) { // Initiate instances with new node - foreach (RenderWorker* instance, video_instance_pool_) { - instance->Init(viewer_node_); - } + hash_pool_.Init(viewer_node_); + video_pool_.Init(viewer_node_); + audio_pool_.Init(viewer_node_); connect(viewer_node_, &ViewerOutput::GraphChangedFrom, this, &RenderBackend::NodeGraphChanged); - connect(viewer_node_->audio_playback_cache(), - &AudioPlaybackCache::Invalidated, - this, - &RenderBackend::AudioInvalidated); + if (audio_enabled_) { + // Listen for audio invalidation signals + connect(viewer_node_->audio_playback_cache(), + &AudioPlaybackCache::Invalidated, + this, + &RenderBackend::AudioInvalidated); + + // Start caching audio + foreach (const TimeRange& r, viewer_node_->audio_playback_cache()->GetInvalidatedRanges()) { + AudioInvalidated(r); + } + } } } void RenderBackend::CancelQueue() { // FIXME: Implement something better than this... - video_thread_pool_.waitForDone(); + video_pool_.threads.waitForDone(); + audio_pool_.threads.waitForDone(); + hash_pool_.threads.waitForDone(); } QFuture RenderBackend::Hash(const rational &time, bool block_for_update) { - return QtConcurrent::run(&video_thread_pool_, - GetInstanceFromPool(video_instance_pool_, video_thread_pool_, video_instance_queuer_), + return QtConcurrent::run(&hash_pool_.threads, + GetInstanceFromPool(hash_pool_), &RenderWorker::Hash, time, block_for_update); @@ -116,11 +126,11 @@ QFuture RenderBackend::Hash(const rational &time, bool block_for_upd QFuture RenderBackend::RenderFrame(const rational &time, bool clear_queue, bool block_for_update) { if (clear_queue) { - video_thread_pool_.clear(); + video_pool_.threads.clear(); } - return QtConcurrent::run(&video_thread_pool_, - GetInstanceFromPool(video_instance_pool_, video_thread_pool_, video_instance_queuer_), + return QtConcurrent::run(&video_pool_.threads, + GetInstanceFromPool(video_pool_), &RenderWorker::RenderFrame, time, block_for_update); @@ -151,11 +161,23 @@ void RenderBackend::SetVideoDownloadMatrix(const QMatrix4x4 &mat) video_download_matrix_ = mat; } +void RenderBackend::SetAudioEnabled(bool e) +{ + audio_enabled_ = e; +} + +void RenderBackend::WorkerStartedRenderingAudio(const TimeRange &r) +{ + queued_audio_lock_.lock(); + queued_audio_.RemoveTimeRange(r); + queued_audio_lock_.unlock(); +} + void RenderBackend::NodeGraphChanged(NodeInput *source) { - foreach (RenderWorker* worker, video_instance_pool_) { - worker->Queue(source); - } + video_pool_.Queue(source); + audio_pool_.Queue(source); + hash_pool_.Queue(source); } void RenderBackend::UpdateInstance(RenderWorker *instance) @@ -171,8 +193,9 @@ void RenderBackend::Close() { CancelQueue(); - qDeleteAll(video_instance_pool_); - video_instance_pool_.clear(); + video_pool_.Destroy(); + audio_pool_.Destroy(); + hash_pool_.Destroy(); } VideoRenderingParams RenderBackend::video_params() const @@ -185,11 +208,11 @@ AudioRenderingParams RenderBackend::audio_params() const return AudioRenderingParams(viewer_node_->audio_params(), sample_fmt_); } -RenderWorker *RenderBackend::GetInstanceFromPool(QVector &worker_pool, QThreadPool &thread_pool, int &instance_queuer) +RenderWorker *RenderBackend::GetInstanceFromPool(RenderPool& pool) { RenderWorker* instance = nullptr; - foreach (RenderWorker* worker, worker_pool) { + foreach (RenderWorker* worker, pool.instances) { if (worker->IsAvailable()) { instance = worker; break; @@ -197,10 +220,10 @@ RenderWorker *RenderBackend::GetInstanceFromPool(QVector &worker } if (!instance) { - if (worker_pool.size() < thread_pool.maxThreadCount()) { + if (pool.instances.size() < pool.threads.maxThreadCount()) { // Can create another instance instance = CreateNewWorker(); - worker_pool.append(instance); + pool.instances.append(instance); if (viewer_node_) { instance->Init(viewer_node_); @@ -212,8 +235,8 @@ RenderWorker *RenderBackend::GetInstanceFromPool(QVector &worker connect(instance, &RenderWorker::AudioConformUnavailable, this, &RenderBackend::AudioConformUnavailable, Qt::QueuedConnection); } else { - instance = worker_pool.at(instance_queuer % worker_pool.size()); - instance_queuer++; + instance = pool.instances.at(pool.queuer % pool.instances.size()); + pool.queuer++; } } @@ -337,13 +360,23 @@ void RenderBackend::AudioInvalidated(const TimeRange& r) } } + // FIXME: Split into chunks for multithreading + + { + QMutexLocker locker(&queued_audio_lock_); + if (queued_audio_.ContainsTimeRange(r)) { + return; + } + queued_audio_.InsertTimeRange(r); + } + QFutureWatcher* watcher = new QFutureWatcher(); connect(watcher, &QFutureWatcher::finished, this, &RenderBackend::AudioRendered); audio_jobs_.insert(watcher, r); - watcher->setFuture(QtConcurrent::run(&video_thread_pool_, - GetInstanceFromPool(audio_instance_pool_, audio_thread_pool_, audio_instance_queuer_), + watcher->setFuture(QtConcurrent::run(&audio_pool_.threads, + GetInstanceFromPool(audio_pool_), &RenderWorker::RenderAudio, r)); } @@ -356,10 +389,8 @@ void RenderBackend::AudioRendered() TimeRange r = audio_jobs_.take(watcher); if (watcher->result()) { - qDebug() << "Received" << watcher->result()->sample_count_per_channel() << "samples"; viewer_node_->audio_playback_cache()->WritePCM(r, watcher->result()); } else { - qDebug() << "Received null"; viewer_node_->audio_playback_cache()->WriteSilence(r); } } @@ -379,4 +410,31 @@ bool RenderBackend::ConformWaitInfo::operator==(const RenderBackend::ConformWait && rhs.affected_range == affected_range; } +void RenderBackend::RenderPool::Init(ViewerOutput* v) +{ + foreach (RenderWorker* instance, instances) { + instance->Init(v); + } +} + +void RenderBackend::RenderPool::Queue(NodeInput *input) +{ + foreach (RenderWorker* worker, instances) { + worker->Queue(input); + } +} + +void RenderBackend::RenderPool::Destroy() +{ + qDeleteAll(instances); + instances.clear(); +} + +void RenderBackend::RenderPool::Close() +{ + foreach (RenderWorker* worker, instances) { + worker->Close(); + } +} + OLIVE_NAMESPACE_EXIT diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index fd3918e8a..0e6b79bc9 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -64,6 +64,10 @@ public: void SetVideoDownloadMatrix(const QMatrix4x4& mat); + void SetAudioEnabled(bool e); + + void WorkerStartedRenderingAudio(const TimeRange& r); + public slots: void NodeGraphChanged(NodeInput *source); @@ -79,22 +83,31 @@ protected: AudioRenderingParams audio_params() const; private: - RenderWorker *GetInstanceFromPool(QVector& worker_pool, - QThreadPool& thread_pool, - int& instance_queuer); + struct RenderPool { + QVector instances; + int queuer; + QThreadPool threads; + + void Init(ViewerOutput *v); + void Queue(NodeInput* input); + void Close(); + void Destroy(); + }; + + RenderWorker *GetInstanceFromPool(RenderPool &pool); ViewerOutput* viewer_node_; RenderCancelDialog* cancel_dialog_; - QVector video_instance_pool_; - QVector audio_instance_pool_; + RenderPool video_pool_; + RenderPool audio_pool_; + RenderPool hash_pool_; - QThreadPool video_thread_pool_; - QThreadPool audio_thread_pool_; + QMutex queued_audio_lock_; + TimeRangeList queued_audio_; - int video_instance_queuer_; - int audio_instance_queuer_; + bool audio_enabled_; // VIDEO MEMBERS int divider_; diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index ba81d8c96..11ace410d 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -107,17 +107,12 @@ SampleBufferPtr RenderWorker::RenderAudio(const TimeRange &range) QMutexLocker locker(&lock_); + parent_->WorkerStartedRenderingAudio(range); + UpdateData(true); audio_render_time_ = range; - qDebug() << "Audio deps:"; - - QList d = viewer_->samples_input()->GetDependencies(); - foreach (Node* n, d) { - qDebug() << " " << n; - } - NodeValueTable table = ProcessInput(viewer_->samples_input(), range); QVariant samples = table.Get(NodeParam::kSamples); @@ -314,8 +309,6 @@ void RenderWorker::ProcessNodeEvent(const Node *node, const TimeRange &range, No void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable *table) { - qDebug() << "Stream type" << stream->type(); - if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) { ImageStreamPtr video_stream = std::static_pointer_cast(stream); @@ -355,7 +348,6 @@ void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &inp } else if (stream->type() == Stream::kAudio) { - qDebug() << "Hello!"; table->Push(GetDataFromStream(stream, input_time)); } @@ -369,18 +361,14 @@ NodeValue RenderWorker::GetDataFromStream(StreamPtr stream, const TimeRange &inp if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) { return FrameToTexture(decoder, stream, input_time); } else if (stream->type() == Stream::kAudio) { - qDebug() << "Decoding audio!"; if (decoder->HasConformedVersion(audio_params())) { - qDebug() << " Retrieving audio!"; SampleBufferPtr frame = decoder->RetrieveAudio(input_time.in(), input_time.length(), audio_params()); if (frame) { - qDebug() << " Returning audio!"; return NodeValue(NodeParam::kSamples, QVariant::fromValue(frame)); } } else { - qDebug() << " Conform doesn't exist! AAAAA"; emit AudioConformUnavailable(decoder->stream(), audio_render_time_, input_time.out(), audio_params()); } diff --git a/app/task/cache/cache.cpp b/app/task/cache/cache.cpp index 60d8a551a..cc1eab71b 100644 --- a/app/task/cache/cache.cpp +++ b/app/task/cache/cache.cpp @@ -63,6 +63,7 @@ void CacheTask::Action() RenderMode::Mode mode = RenderMode::kOffline; PixelFormat::Format format = PixelFormat::instance()->GetConfiguredFormatForMode(mode); + backend.SetAudioEnabled(false); backend.SetViewerNode(viewer_); backend.SetPixelFormat(format); backend.SetMode(mode); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index e7edea9cc..bdbed11e1 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -433,6 +433,11 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) playback_speed_ = speed; play_in_to_out_only_ = in_to_out_only; + int64_t start_time = ruler()->GetTime(); + + playback_queue_next_frame_ = start_time; + FillPlaybackQueue(); + QString audio_fn = GetConnectedNode()->audio_playback_cache()->GetCacheFilename(); if (!audio_fn.isEmpty()) { AudioManager::instance()->SetOutputParams(GetConnectedNode()->audio_playback_cache()->GetParameters()); @@ -441,11 +446,6 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) playback_speed_); } - int64_t start_time = ruler()->GetTime(); - - playback_queue_next_frame_ = start_time; - FillPlaybackQueue(); - playback_timer_.Start(start_time, playback_speed_, timebase_dbl()); foreach (ViewerWindow* window, windows_) {