From 25e94112f8723266799b5d35ab0211e747db7ea7 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Tue, 26 Jul 2022 14:49:19 -0700 Subject: [PATCH] viewer: implement lookahead to ensure footage is ready in advance --- app/codec/decoder.cpp | 6 ++ app/codec/decoder.h | 2 + app/render/previewautocacher.cpp | 13 ++-- app/render/previewautocacher.h | 8 +-- app/render/rendermanager.cpp | 14 +++- app/render/rendermanager.h | 6 +- app/render/renderprocessor.cpp | 107 ++++++++++++++++--------------- app/widget/viewer/viewer.cpp | 36 ++++++++++- app/widget/viewer/viewer.h | 7 ++ 9 files changed, 137 insertions(+), 62 deletions(-) diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index fd845737e..c98cb5c2a 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -43,6 +43,12 @@ Decoder::Decoder() : UpdateLastAccessed(); } +void Decoder::IncrementAccessTime(qint64 t) +{ + QMutexLocker locker(&mutex_); + last_accessed_ += t; +} + bool Decoder::Open(const CodecStream &stream) { QMutexLocker locker(&mutex_); diff --git a/app/codec/decoder.h b/app/codec/decoder.h index 08ccf6e32..665f21cc8 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -87,6 +87,8 @@ public: virtual bool SupportsVideo(){return false;} virtual bool SupportsAudio(){return false;} + void IncrementAccessTime(qint64 t); + class CodecStream { public: diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index d5707c845..e876ddafa 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -61,7 +61,7 @@ PreviewAutoCacher::~PreviewAutoCacher() SetViewerNode(nullptr); } -RenderTicketPtr PreviewAutoCacher::GetSingleFrame(const rational &t) +RenderTicketPtr PreviewAutoCacher::GetSingleFrame(const rational &t, bool dry) { // If we have a single frame render queued (but not yet sent to the RenderManager), cancel it now CancelQueuedSingleFrameRender(); @@ -70,6 +70,7 @@ RenderTicketPtr PreviewAutoCacher::GetSingleFrame(const rational &t) auto sfr = std::make_shared(); sfr->Start(); sfr->setProperty("time", QVariant::fromValue(t)); + sfr->setProperty("dry", dry); // Queue it and try to render single_frame_render_ = sfr; @@ -566,7 +567,8 @@ void PreviewAutoCacher::TryRender() if (single_frame_render_) { // Check if already caching this RenderTicketWatcher *watcher = RenderFrame(single_frame_render_->property("time").value(), - nullptr); + nullptr, + single_frame_render_->property("dry").toBool()); video_immediate_passthroughs_[watcher].append(single_frame_render_); single_frame_render_ = nullptr; @@ -583,7 +585,7 @@ void PreviewAutoCacher::TryRender() // We want this hash, if we're not already rendering, start render now if (!render_task) { // Don't render any hash more than once - RenderFrame(t, viewer_node_->video_frame_cache()); + RenderFrame(t, viewer_node_->video_frame_cache(), false); } emit SignalCacheProxyTaskProgress(double(queued_frame_iterator_.frame_index()) / double(queued_frame_iterator_.size())); @@ -609,7 +611,7 @@ void PreviewAutoCacher::TryRender() } } -RenderTicketWatcher* PreviewAutoCacher::RenderFrame(Node *node, const rational& time, FrameHashCache *cache) +RenderTicketWatcher* PreviewAutoCacher::RenderFrame(Node *node, const rational& time, FrameHashCache *cache, bool dry) { RenderTicketWatcher* watcher = new RenderTicketWatcher(); watcher->setProperty("job", QVariant::fromValue(last_update_time_)); @@ -623,7 +625,8 @@ RenderTicketWatcher* PreviewAutoCacher::RenderFrame(Node *node, const rational& time, RenderMode::kOffline, cache, - RenderManager::kTexture)); + dry ? RenderManager::kNull : RenderManager::kTexture)); + return watcher; } diff --git a/app/render/previewautocacher.h b/app/render/previewautocacher.h index c9343021f..0ce429673 100644 --- a/app/render/previewautocacher.h +++ b/app/render/previewautocacher.h @@ -49,7 +49,7 @@ public: virtual ~PreviewAutoCacher() override; - RenderTicketPtr GetSingleFrame(const rational& t); + RenderTicketPtr GetSingleFrame(const rational& t, bool dry = false); RenderTicketPtr GetRangeOfAudio(TimeRange range); @@ -98,10 +98,10 @@ signals: private: void TryRender(); - RenderTicketWatcher *RenderFrame(Node *node, const rational &time, FrameHashCache *cache); - RenderTicketWatcher *RenderFrame(const rational &time, FrameHashCache *cache) + RenderTicketWatcher *RenderFrame(Node *node, const rational &time, FrameHashCache *cache, bool dry); + RenderTicketWatcher *RenderFrame(const rational &time, FrameHashCache *cache, bool dry) { - return RenderFrame(copied_viewer_node_->GetConnectedTextureOutput(), time, cache); + return RenderFrame(copied_viewer_node_->GetConnectedTextureOutput(), time, cache, dry); } RenderTicketPtr RenderAudio(Node *node, const TimeRange &range, bool generate_waveforms); diff --git a/app/render/rendermanager.cpp b/app/render/rendermanager.cpp index 202820be5..1df5590b3 100644 --- a/app/render/rendermanager.cpp +++ b/app/render/rendermanager.cpp @@ -35,6 +35,7 @@ namespace olive { RenderManager* RenderManager::instance_ = nullptr; +const rational RenderManager::kDryRunInterval = rational(10); RenderManager::RenderManager(QObject *parent) : backend_(kOpenGL), @@ -52,9 +53,11 @@ RenderManager::RenderManager(QObject *parent) : if (context_) { video_thread_ = new RenderThread(context_, decoder_cache_, shader_cache_, this); + dry_run_thread_ = new RenderThread(nullptr, decoder_cache_, shader_cache_, this); audio_thread_ = new RenderThread(nullptr, decoder_cache_, shader_cache_, this); video_thread_->start(QThread::IdlePriority); + dry_run_thread_->start(QThread::IdlePriority); audio_thread_->start(QThread::IdlePriority); } @@ -73,6 +76,9 @@ RenderManager::~RenderManager() video_thread_->quit(); video_thread_->wait(); + dry_run_thread_->quit(); + dry_run_thread_->wait(); + context_->PostDestroy(); delete context_; @@ -129,7 +135,11 @@ RenderTicketPtr RenderManager::RenderFrame(Node *node, ColorManager* color_manag ticket->setProperty("cacheuuid", QVariant::fromValue(cache->GetUuid())); } - video_thread_->AddTicket(ticket); + if (return_type == ReturnType::kNull) { + dry_run_thread_->AddTicket(ticket); + } else { + video_thread_->AddTicket(ticket); + } return ticket; } @@ -157,6 +167,8 @@ bool RenderManager::RemoveTicket(RenderTicketPtr ticket) return true; } else if (audio_thread_->RemoveTicket(ticket)) { return true; + } else if (dry_run_thread_->RemoveTicket(ticket)) { + return true; } else { return false; } diff --git a/app/render/rendermanager.h b/app/render/rendermanager.h index a75c58aaf..891e2fffc 100644 --- a/app/render/rendermanager.h +++ b/app/render/rendermanager.h @@ -97,9 +97,12 @@ public: enum ReturnType { kTexture, - kFrame + kFrame, + kNull }; + static const rational kDryRunInterval; + /** * @brief Asynchronously generate a frame at a given time * @@ -168,6 +171,7 @@ private: QTimer *decoder_clear_timer_; RenderThread *video_thread_; + RenderThread *dry_run_thread_; RenderThread *audio_thread_; private slots: diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index 08a4fbdbc..0a6d78078 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -151,53 +151,57 @@ void RenderProcessor::Run() TexturePtr texture = GenerateTexture(time, frame_length); - if (GetCacheVideoParams().interlacing() != VideoParams::kInterlaceNone) { - // Get next between frame and interlace it - TexturePtr top = texture; - TexturePtr bottom = GenerateTexture(time + frame_length, frame_length); - - if (GetCacheVideoParams().interlacing() == VideoParams::kInterlacedBottomFirst) { - std::swap(top, bottom); - } - - texture = render_ctx_->InterlaceTexture(top, bottom, GetCacheVideoParams()); - } - - if (HeardCancel()) { - // Finish cancelled ticket with nothing since we can't guarantee the frame we generated - // is actually "complete + if (!render_ctx_) { ticket_->Finish(); } else { - RenderManager::ReturnType return_type = RenderManager::ReturnType(ticket_->property("return").toInt()); + if (GetCacheVideoParams().interlacing() != VideoParams::kInterlaceNone) { + // Get next between frame and interlace it + TexturePtr top = texture; + TexturePtr bottom = GenerateTexture(time + frame_length, frame_length); - FramePtr frame; - QString cache = ticket_->property("cache").toString(); - - if (return_type == RenderManager::kFrame || !cache.isEmpty()) { - // Convert to CPU frame - frame = GenerateFrame(texture, time); - - // Save to cache if requested - if (!cache.isEmpty()) { - rational timebase = ticket_->property("cachetimebase").value(); - QUuid uuid = ticket_->property("cacheuuid").value(); - bool cache_result = FrameHashCache::SaveCacheFrame(cache, uuid, time, timebase, frame); - ticket_->setProperty("cached", cache_result); + if (GetCacheVideoParams().interlacing() == VideoParams::kInterlacedBottomFirst) { + std::swap(top, bottom); } + + texture = render_ctx_->InterlaceTexture(top, bottom, GetCacheVideoParams()); } - if (return_type == RenderManager::kTexture) { - // Return GPU texture - if (!texture) { - texture = render_ctx_->CreateTexture(GetCacheVideoParams()); - render_ctx_->ClearDestination(texture.get()); + if (HeardCancel()) { + // Finish cancelled ticket with nothing since we can't guarantee the frame we generated + // is actually "complete + ticket_->Finish(); + } else { + RenderManager::ReturnType return_type = RenderManager::ReturnType(ticket_->property("return").toInt()); + + FramePtr frame; + QString cache = ticket_->property("cache").toString(); + + if (return_type == RenderManager::kFrame || !cache.isEmpty()) { + // Convert to CPU frame + frame = GenerateFrame(texture, time); + + // Save to cache if requested + if (!cache.isEmpty()) { + rational timebase = ticket_->property("cachetimebase").value(); + QUuid uuid = ticket_->property("cacheuuid").value(); + bool cache_result = FrameHashCache::SaveCacheFrame(cache, uuid, time, timebase, frame); + ticket_->setProperty("cached", cache_result); + } } - render_ctx_->Flush(); + if (return_type == RenderManager::kTexture) { + // Return GPU texture + if (!texture) { + texture = render_ctx_->CreateTexture(GetCacheVideoParams()); + render_ctx_->ClearDestination(texture.get()); + } - ticket_->Finish(QVariant::fromValue(texture)); - } else { - ticket_->Finish(QVariant::fromValue(frame)); + render_ctx_->Flush(); + + ticket_->Finish(QVariant::fromValue(texture)); + } else { + ticket_->Finish(QVariant::fromValue(frame)); + } } } break; @@ -265,6 +269,11 @@ DecoderPtr RenderProcessor::ResolveDecoderFromInput(const QString& decoder_id, c << "::" << stream.stream(); return nullptr; } + + if (!render_ctx_) { + // Assume dry run and increment access time + decoder.decoder->IncrementAccessTime(RenderManager::kDryRunInterval.toDouble() * 1000); + } } return decoder.decoder; @@ -404,10 +413,6 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim void RenderProcessor::ProcessVideoFootage(TexturePtr destination, const FootageJob &stream, const rational &input_time) { - if (!render_ctx_) { - return; - } - if (ticket_->property("type").value() != RenderManager::kTypeVideo) { // Video cannot contribute to audio, so we do nothing here return; @@ -440,21 +445,23 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination, const FootageJ break; case VideoParams::kVideoTypeImageSequence: { - // Since image sequences involve multiple files, we don't engage the decoder cache - decoder = Decoder::CreateFromID(decoder_id); + if (render_ctx_) { + // Since image sequences involve multiple files, we don't engage the decoder cache + decoder = Decoder::CreateFromID(decoder_id); - QString frame_filename; + QString frame_filename; - int64_t frame_number = stream_data.get_time_in_timebase_units(input_time); - frame_filename = Decoder::TransformImageSequenceFileName(stream.filename(), frame_number); + int64_t frame_number = stream_data.get_time_in_timebase_units(input_time); + frame_filename = Decoder::TransformImageSequenceFileName(stream.filename(), frame_number); - // Decoder will close automatically since it's a stream_ptr - decoder->Open(Decoder::CodecStream(frame_filename, stream_data.stream_index(), GetCurrentBlock())); + // Decoder will close automatically since it's a stream_ptr + decoder->Open(Decoder::CodecStream(frame_filename, stream_data.stream_index(), GetCurrentBlock())); + } break; } } - if (decoder) { + if (decoder && render_ctx_) { Decoder::RetrieveVideoParams p; p.divider = stream.video_params().divider(); p.maximum_format = destination->format(); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 18d086386..c5635cc86 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -60,7 +60,7 @@ QVector ViewerWidget::instances_; // changing values. 1/4 second seems to be a good middleground. const rational ViewerWidget::kAudioPlaybackInterval = rational(1, 4); -const rational kVideoPlaybackInterval = rational(2); +const rational kVideoPlaybackInterval = rational(1, 2); ViewerWidget::ViewerWidget(QWidget *parent) : super(false, true, parent), @@ -560,6 +560,35 @@ void ViewerWidget::ShowSubtitleProperties() } } +void ViewerWidget::DryRunFinished() +{ + RenderTicketWatcher *w = static_cast(sender()); + + if (dry_run_watchers_.contains(w)) { + RequestNextDryRun(); + } + + delete w; +} + +void ViewerWidget::RequestNextDryRun() +{ + if (IsPlaying()) { + rational next_time = Timecode::timestamp_to_time(dry_run_next_frame_, timebase()); + if (FrameExistsAtTime(next_time)) { + if (next_time > GetTime() + RenderManager::kDryRunInterval) { + QTimer::singleShot(timebase().toDouble() / playback_speed_, this, &ViewerWidget::RequestNextDryRun); + } else { + RenderTicketWatcher *watcher = new RenderTicketWatcher(this); + connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::DryRunFinished); + watcher->SetTicket(auto_cacher_.GetSingleFrame(next_time, true)); + dry_run_next_frame_ += playback_speed_; + dry_run_watchers_.append(watcher); + } + } + } +} + void ViewerWidget::CloseAudioProcessor() { audio_processor_.Close(); @@ -828,6 +857,9 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) for (int i=0; i playback_devices_; bool prequeuing_video_; @@ -294,6 +295,8 @@ private: WaveformMode waveform_mode_; + QVector dry_run_watchers_; + private slots: void PlaybackTimerUpdate(); @@ -354,6 +357,10 @@ private slots: void ShowSubtitleProperties(); + void DryRunFinished(); + + void RequestNextDryRun(); + }; }