From 07a53d45f05706b53b2cc49dbfbd236c864d8d07 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Wed, 5 Aug 2020 22:32:24 +1000 Subject: [PATCH] renderer: updated audio waveform generated for new cache overhaul --- app/core.cpp | 1 + app/render/backend/renderbackend.cpp | 66 ++++++++++++++------- app/render/backend/renderbackend.h | 31 +++------- app/render/backend/renderticket.cpp | 3 +- app/render/backend/renderticket.h | 15 +++++ app/render/backend/renderworker.cpp | 88 +++++++++++----------------- app/render/backend/renderworker.h | 19 ++---- app/widget/viewer/viewer.cpp | 1 + 8 files changed, 112 insertions(+), 112 deletions(-) diff --git a/app/core.cpp b/app/core.cpp index 25a0e6d07..1b6b8c74a 100644 --- a/app/core.cpp +++ b/app/core.cpp @@ -173,6 +173,7 @@ void Core::DeclareTypesForQt() qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); + qRegisterMetaType(); } void Core::Start() diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index 75a609846..19fcb06f7 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -41,7 +41,7 @@ RenderBackend::RenderBackend(QObject *parent) : viewer_node_(nullptr), autocache_enabled_(false), autocache_paused_(false), - preview_job_time_(0), + generate_audio_previews_(false), render_mode_(RenderMode::kOnline), autocache_has_changed_(false), use_custom_autocache_range_(false) @@ -374,6 +374,7 @@ void RenderBackend::RunNextJob() for (int i=0;iSetAudioParams(audio_params_); worker->SetVideoDownloadMatrix(video_download_matrix_); worker->SetRenderMode(render_mode_); - if (preview_job_time_) { - worker->EnablePreviewGeneration(viewer_node_->audio_playback_cache(), preview_job_time_); - } + worker->SetPreviewGenerationEnabled(generate_audio_previews_); worker->SetCopyMap(©_map_); + // Move ticket from queue to running list RenderTicketPtr ticket = render_queue_.front(); render_queue_.pop_front(); - running_tickets_.push_back(ticket); + + // Create watcher to remove from running list RenderTicketWatcher* watcher = new RenderTicketWatcher(); connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::TicketFinished); watcher->SetTicket(ticket); + // Set job time to now + ticket->SetJobTime(); + switch (ticket->GetType()) { case RenderTicket::kTypeHash: QtConcurrent::run(&thread_pool_, @@ -458,6 +462,28 @@ void RenderBackend::TicketFinished() running_tickets_.remove(ticket); } +void RenderBackend::WorkerGeneratedWaveform(RenderTicketPtr ticket, TrackOutput *track, AudioVisualWaveform samples, TimeRange range) +{ + qDebug() << "Hello?"; + + QList valid_ranges = viewer_node_->audio_playback_cache()->GetValidRanges(range, + ticket->GetJobTime()); + if (!valid_ranges.isEmpty()) { + // Generate visual waveform in this background thread + track->waveform_lock()->lock(); + + track->waveform().set_channel_count(audio_params_.channel_count()); + + foreach (const TimeRange& r, valid_ranges) { + track->waveform().OverwriteSums(samples, r.in(), r.in() - range.in(), r.length()); + } + + track->waveform_lock()->unlock(); + + emit track->PreviewChanged(); + } +} + void RenderBackend::AutoCacheVideoInvalidated(const TimeRange &range) { ClearVideoQueue(); @@ -465,7 +491,7 @@ void RenderBackend::AutoCacheVideoInvalidated(const TimeRange &range) // Hash these frames since that should be relatively quick. RenderTicketWatcher* watcher = new RenderTicketWatcher(); QVector frames = viewer_node_->video_frame_cache()->GetFrameListFromTimeRange({range}); - autocache_hash_tasks_.insert(watcher, {frames, QDateTime::currentMSecsSinceEpoch()}); + autocache_hash_tasks_.insert(watcher, frames); connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheHashesGenerated); watcher->SetTicket(Hash(frames)); } @@ -474,7 +500,7 @@ void RenderBackend::AutoCacheAudioInvalidated(const TimeRange &range) { // Start a task to re-render the audio at this range RenderTicketWatcher* watcher = new RenderTicketWatcher(); - autocache_audio_tasks_.insert(watcher, {range, QDateTime::currentMSecsSinceEpoch()}); + autocache_audio_tasks_.insert(watcher, range); connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheAudioRendered); watcher->SetTicket(RenderAudio(range, true)); } @@ -509,17 +535,15 @@ void RenderBackend::AutoCacheHashesGenerated() if (autocache_hash_tasks_.contains(watcher)) { if (!watcher->WasCancelled()) { - const HashJobInfo& info = autocache_hash_tasks_.value(watcher); - QFutureWatcher* hw = new QFutureWatcher(); connect(hw, &QFutureWatcher::finished, this, &RenderBackend::AutoCacheHashesProcessed); autocache_hash_process_tasks_.append(hw); hw->setFuture(QtConcurrent::run(this, &RenderBackend::SetHashes, viewer_node_->video_frame_cache(), - info.times, + autocache_hash_tasks_.value(watcher), watcher->Get().value >(), - info.job_time)); + watcher->GetTicket()->GetJobTime())); } autocache_hash_tasks_.remove(watcher); @@ -547,11 +571,9 @@ void RenderBackend::AutoCacheAudioRendered() if (autocache_audio_tasks_.contains(watcher)) { if (!watcher->WasCancelled()) { - const AudioJobInfo& job_info = autocache_audio_tasks_.value(watcher); - - viewer_node_->audio_playback_cache()->WritePCM(job_info.range, + viewer_node_->audio_playback_cache()->WritePCM(autocache_audio_tasks_.value(watcher), watcher->Get().value(), - job_info.job_time); + watcher->GetTicket()->GetJobTime()); } autocache_audio_tasks_.remove(watcher); @@ -566,14 +588,14 @@ void RenderBackend::AutoCacheVideoRendered() if (autocache_video_tasks_.contains(watcher)) { if (!watcher->WasCancelled()) { - const VideoJobInfo& info = autocache_video_tasks_.value(watcher); + const QByteArray& hash = autocache_video_tasks_.value(watcher); // Download frame in another thread QFutureWatcher* w = new QFutureWatcher(); - autocache_video_download_tasks_.insert(w, info); + autocache_video_download_tasks_.insert(w, hash); connect(w, &QFutureWatcher::finished, this, &RenderBackend::AutoCacheVideoDownloaded); w->setFuture(QtConcurrent::run(FrameHashCache::SaveCacheFrame, - info.hash, + hash, watcher->Get().value())); } @@ -590,11 +612,11 @@ void RenderBackend::AutoCacheVideoDownloaded() if (autocache_video_download_tasks_.contains(watcher)) { if (!watcher->isCanceled()) { if (watcher->result()) { - const VideoJobInfo& info = autocache_video_download_tasks_.value(watcher); + const QByteArray& hash = autocache_video_download_tasks_.value(watcher); - currently_caching_hashes_.removeOne(info.hash); + currently_caching_hashes_.removeOne(hash); - viewer_node_->video_frame_cache()->ValidateFramesWithHash(info.hash); + viewer_node_->video_frame_cache()->ValidateFramesWithHash(hash); } else { qCritical() << "Failed to download video frame"; } @@ -792,7 +814,7 @@ void RenderBackend::AutoCacheRequeueFrames() RenderTicketWatcher* watcher = new RenderTicketWatcher(); connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheVideoRendered); - autocache_video_tasks_.insert(watcher, {hash, QDateTime::currentMSecsSinceEpoch()}); + autocache_video_tasks_.insert(watcher, hash); watcher->SetTicket(RenderFrame(t, false, hash)); } diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index 1f0b5879d..aaaeab98f 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -95,9 +95,9 @@ public: render_mode_ = e; } - void EnablePreviewGeneration(qint64 job_time) + void SetPreviewGenerationEnabled(bool e) { - preview_job_time_ = job_time; + generate_audio_previews_ = e; } void ProcessUpdateQueue(); @@ -185,7 +185,7 @@ private: bool autocache_enabled_; bool autocache_paused_; - qint64 preview_job_time_; + bool generate_audio_previews_; RenderMode::Mode render_mode_; @@ -202,30 +202,15 @@ private: static QThreadPool thread_pool_; void SetActiveInstance(); - struct HashJobInfo { - QVector times; - qint64 job_time; - }; - - struct AudioJobInfo { - TimeRange range; - qint64 job_time; - }; - - struct VideoJobInfo { - QByteArray hash; - qint64 job_time; - }; - - QMap autocache_hash_tasks_; + QMap > autocache_hash_tasks_; QList*> autocache_hash_process_tasks_; - QMap autocache_audio_tasks_; + QMap autocache_audio_tasks_; - QMap autocache_video_tasks_; + QMap autocache_video_tasks_; - QMap*, VideoJobInfo> autocache_video_download_tasks_; + QMap*, QByteArray> autocache_video_download_tasks_; QVector currently_caching_hashes_; @@ -236,6 +221,8 @@ private slots: void TicketFinished(); + void WorkerGeneratedWaveform(OLIVE_NAMESPACE::RenderTicketPtr ticket, OLIVE_NAMESPACE::TrackOutput* track, OLIVE_NAMESPACE::AudioVisualWaveform samples, OLIVE_NAMESPACE::TimeRange range); + void AutoCacheVideoInvalidated(const OLIVE_NAMESPACE::TimeRange &range); void AutoCacheAudioInvalidated(const OLIVE_NAMESPACE::TimeRange &range); diff --git a/app/render/backend/renderticket.cpp b/app/render/backend/renderticket.cpp index 1e719a5f6..027683cd1 100644 --- a/app/render/backend/renderticket.cpp +++ b/app/render/backend/renderticket.cpp @@ -26,7 +26,8 @@ RenderTicket::RenderTicket(Type type, const QVariant &time) : finished_(false), cancelled_(false), time_(time), - type_(type) + type_(type), + job_time_(0) { } diff --git a/app/render/backend/renderticket.h b/app/render/backend/renderticket.h index 22fe46c8d..f4fca0c6b 100644 --- a/app/render/backend/renderticket.h +++ b/app/render/backend/renderticket.h @@ -21,6 +21,7 @@ #ifndef RENDERTICKET_H #define RENDERTICKET_H +#include #include #include @@ -42,6 +43,16 @@ public: RenderTicket(Type type, const QVariant& time); + qint64 GetJobTime() const + { + return job_time_; + } + + void SetJobTime() + { + job_time_ = QDateTime::currentMSecsSinceEpoch(); + } + const QVariant& GetTime() const { return time_; @@ -87,10 +98,14 @@ private: Type type_; + qint64 job_time_; + }; using RenderTicketPtr = std::shared_ptr; OLIVE_NAMESPACE_EXIT +Q_DECLARE_METATYPE(OLIVE_NAMESPACE::RenderTicketPtr) + #endif // RENDERTICKET_H diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index 31194a105..8eb34ef8f 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -34,14 +34,15 @@ OLIVE_NAMESPACE_ENTER RenderWorker::RenderWorker(RenderBackend* parent) : parent_(parent), available_(true), - audio_mode_is_preview_(false), - preview_cache_(nullptr), + generate_audio_previews_(false), render_mode_(RenderMode::kOnline) { } void RenderWorker::Hash(RenderTicketPtr ticket, ViewerOutput *viewer, const QVector ×) { + ticket_ = ticket; + QVector hashes(times.size()); for (int i=0;itexture_input(), TimeRange(time, time + video_params_.time_base())); @@ -109,6 +112,8 @@ void RenderWorker::RenderFrame(RenderTicketPtr ticket, ViewerOutput* viewer, con void RenderWorker::RenderAudio(RenderTicketPtr ticket, ViewerOutput* viewer, const TimeRange &range) { + ticket_ = ticket; + NodeValueTable table = ProcessInput(viewer->samples_input(), range); QVariant samples = table.Get(NodeParam::kSamples); @@ -168,38 +173,26 @@ NodeValueTable RenderWorker::GenerateBlockTable(const TrackOutput *track, const NodeValueTable::Merge({merged_table, table}); } - if (preview_cache_) { + if (generate_audio_previews_) { // Find original track object TrackOutput* original_track = nullptr; - QList valid_ranges = preview_cache_->GetValidRanges(range, preview_job_time_); - if (!valid_ranges.isEmpty()) { - QHash::const_iterator i; - for (i=copy_map_->constBegin(); i!=copy_map_->constEnd(); i++) { - if (i.value() == track) { - original_track = static_cast(i.key()); - break; - } + // Have to do a manual loop since our track is const and QHash won't take it + QHash::const_iterator i; + for (i=copy_map_->constBegin(); i!=copy_map_->constEnd(); i++) { + if (i.value() == track) { + original_track = static_cast(i.key()); + break; } + } - // Generate visual waveform in this background thread - if (original_track) { - AudioVisualWaveform visual_waveform; - visual_waveform.set_channel_count(audio_params_.channel_count()); - visual_waveform.OverwriteSamples(block_range_buffer, audio_params_.sample_rate()); + if (original_track) { + // Generate a visual waveform and send it back to the main thread + AudioVisualWaveform visual_waveform; + visual_waveform.set_channel_count(audio_params_.channel_count()); + visual_waveform.OverwriteSamples(block_range_buffer, audio_params_.sample_rate()); - original_track->waveform_lock()->lock(); - - original_track->waveform().set_channel_count(audio_params_.channel_count()); - - foreach (const TimeRange& r, valid_ranges) { - original_track->waveform().OverwriteSums(visual_waveform, r.in(), r.in() - range.in(), r.length()); - } - - original_track->waveform_lock()->unlock(); - - emit original_track->PreviewChanged(); - } + emit WaveformGenerated(ticket_, original_track, visual_waveform, range); } } @@ -386,37 +379,24 @@ QVariant RenderWorker::ProcessAudioFootage(StreamPtr stream, const TimeRange &in // See if we have a conformed version of this audio if (!decoder->HasConformedVersion(audio_params())) { - // If not, check what audio mode we're in - if (audio_mode_is_preview_) { + // If not, the audio needs to be conformed + // For online rendering/export, it's a waste of time to render the audio until we have + // all we need, so we try to handle the conform ourselves + AudioStreamPtr as = std::static_pointer_cast(stream); - // For preview, we report the conform is missing and finish the render without it - // temporarily. The backend that picks up this signal will recache this section once the - // conform is available. - emit AudioConformUnavailable(decoder->stream(), - audio_render_time_, - input_time.out(), - audio_params()); + // Check if any other threads are conforming this audio + if (as->try_start_conforming(audio_params())) { + + // If not, conform it ourselves + decoder->ConformAudio(&IsCancelled(), audio_params()); } else { - // For online rendering/export, it's a waste of time to render the audio until we have - // all we need, so we try to handle the conform ourselves - AudioStreamPtr as = std::static_pointer_cast(stream); + // If another thread is conforming already, hackily try to wait until it's done. + do { + QThread::msleep(1000); + } while (!as->has_conformed_version(audio_params()) && !IsCancelled()); - // Check if any other threads are conforming this audio - if (as->try_start_conforming(audio_params())) { - - // If not, conform it ourselves - decoder->ConformAudio(&IsCancelled(), audio_params()); - - } else { - - // If another thread is conforming already, hackily try to wait until it's done. - do { - QThread::msleep(1000); - } while (!as->has_conformed_version(audio_params()) && !IsCancelled()); - - } } } diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index 65655a2b2..44b945b5a 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -63,11 +63,6 @@ public: video_download_matrix_ = mat; } - void SetAudioModeIsPreview(bool audio_mode_is_preview) - { - audio_mode_is_preview_ = audio_mode_is_preview; - } - void SetCopyMap(QHash* copy_map) { copy_map_ = copy_map; @@ -78,10 +73,9 @@ public: render_mode_ = mode; } - void EnablePreviewGeneration(AudioPlaybackCache* cache, qint64 job_time) + void SetPreviewGenerationEnabled(bool e) { - preview_cache_ = cache; - preview_job_time_ = job_time; + generate_audio_previews_ = e; } void Hash(RenderTicketPtr ticket, ViewerOutput* viewer, const QVector& times); @@ -143,7 +137,7 @@ signals: void FinishedJob(); - void WaveformGenerated(OLIVE_NAMESPACE::TrackOutput* track, OLIVE_NAMESPACE::AudioVisualWaveform samples, OLIVE_NAMESPACE::TimeRange start); + void WaveformGenerated(OLIVE_NAMESPACE::RenderTicketPtr ticket, OLIVE_NAMESPACE::TrackOutput* track, OLIVE_NAMESPACE::AudioVisualWaveform samples, OLIVE_NAMESPACE::TimeRange range); private: DecoderPtr ResolveDecoderFromInput(StreamPtr stream); @@ -152,6 +146,8 @@ private: RenderBackend* parent_; + RenderTicketPtr ticket_; + VideoParams video_params_; AudioParams audio_params_; @@ -173,10 +169,7 @@ private: TimeRange audio_render_time_; bool available_; - bool audio_mode_is_preview_; - - AudioPlaybackCache* preview_cache_; - qint64 preview_job_time_; + bool generate_audio_previews_; QHash* copy_map_; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 82d70cc15..9ae33e60c 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -110,6 +110,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) : renderer_ = new OpenGLBackend(this); renderer_->SetAutoCacheEnabled(true); renderer_->SetRenderMode(RenderMode::kOffline); + renderer_->SetPreviewGenerationEnabled(true); // Ensures that seeking on the waveform view updates the time as expected connect(waveform_view_, &AudioWaveformView::TimeChanged, this, &ViewerWidget::SetTimeAndSignal);