From 8c4c6ba2a5d61d808183e6290fed64a7f89d68b4 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 9 May 2020 04:40:12 +1000 Subject: [PATCH] viewer: queue frames during playback Rather than have to decode EXRs and display them in the span of one frame, decode into memory ahead of time so that they're ready for upload. Has the potential to speed up playback by about 5-6x. --- app/widget/viewer/viewer.cpp | 119 ++++++++++++++++++++++++++-- app/widget/viewer/viewer.h | 19 ++++- app/widget/viewer/viewerdisplay.cpp | 60 +------------- app/widget/viewer/viewerdisplay.h | 5 -- 4 files changed, 133 insertions(+), 70 deletions(-) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 5d8cf5e62..34fd47a6f 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -345,15 +345,31 @@ void ViewerWidget::SetGizmos(Node *node) void ViewerWidget::UpdateTextureFromNode(const rational& time) { - if (!GetConnectedNode() || time >= GetConnectedNode()->Length()) { - main_gl_widget()->SetImage(QString()); - video_renderer_->UpdateLastRequestedTime(time); - } else { - QString frame_fn = video_renderer_->GetCachedFrame(time); + { + QMutexLocker locker(&playback_frame_queue_lock_); + while (!playback_frame_queue_.isEmpty()) { + PlaybackFrame pf = playback_frame_queue_.takeFirst(); + + if (pf.timestamp == time) { + // Frame was in queue, no need to decode anything + main_gl_widget()->SetImageFromLoadBuffer(pf.frame.get()); + QtConcurrent::run(this, &ViewerWidget::FillPlaybackQueue); + return; + } + } + } + + // Frame was not in queue, will require decoding + if (FrameExistsAtTime(time)) { + QString frame_fn = GetCachedFilenameFromTime(time); if (!frame_fn.isEmpty()) { - main_gl_widget()->SetImage(frame_fn); + FramePtr f = DecodeCachedImage(frame_fn); + main_gl_widget()->SetImageFromLoadBuffer(f.get()); } + } else { + main_gl_widget()->SetImageFromLoadBuffer(nullptr); + video_renderer_->UpdateLastRequestedTime(time); } } @@ -372,11 +388,14 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) QString audio_fn = audio_renderer_->CachePathName(); if (!audio_fn.isEmpty()) { AudioManager::instance()->SetOutputParams(audio_renderer_->params()); - AudioManager::instance()->StartOutput(audio_fn, audio_renderer_->params().time_to_bytes(GetTime()), playback_speed_); + AudioManager::instance()->StartOutput(audio_fn, + audio_renderer_->params().time_to_bytes(GetTime()), + playback_speed_); } start_msec_ = QDateTime::currentMSecsSinceEpoch(); start_timestamp_ = ruler()->GetTime(); + playback_frame_queue_next_frame_ = start_timestamp_; controls_->ShowPauseButton(); @@ -385,6 +404,8 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) if (playback_is_audio_only_) { connect(AudioManager::instance(), &AudioManager::OutputNotified, this, &ViewerWidget::PlaybackTimerUpdate); } else { + FillPlaybackQueue(); + connect(main_gl_widget(), &ViewerDisplayWidget::frameSwapped, this, &ViewerWidget::PlaybackTimerUpdate); } } @@ -442,6 +463,86 @@ void ViewerWidget::SetColorTransform(const ColorTransform &transform, ViewerDisp sender->SetColorTransform(transform); } +void ViewerWidget::FillPlaybackQueue() +{ + playback_frame_queue_lock_.lock(); + while (playback_frame_queue_.size() < 8) { + // Load frame from cache + FramePtr frame = nullptr; + + rational rtime = Timecode::timestamp_to_time(playback_frame_queue_next_frame_, + timebase()); + + QString frame_fn = GetCachedFilenameFromTime(rtime); + + if (!frame_fn.isEmpty()) { + frame = DecodeCachedImage(frame_fn); + } + + playback_frame_queue_.append({rtime, frame}); + + if (!playback_speed_) { + break; + } + + playback_frame_queue_next_frame_ += playback_speed_; + } + playback_frame_queue_lock_.unlock(); +} + +QString ViewerWidget::GetCachedFilenameFromTime(const rational &time) +{ + if (FrameExistsAtTime(time)) { + return video_renderer_->GetCachedFrame(time); + } else { + return QString(); + } +} + +bool ViewerWidget::FrameExistsAtTime(const rational &time) +{ + return GetConnectedNode() && time < GetConnectedNode()->Length(); +} + +FramePtr ViewerWidget::DecodeCachedImage(const QString &fn) +{ + FramePtr frame = nullptr; + + if (!fn.isEmpty() && QFileInfo::exists(fn)) { + auto input = OIIO::ImageInput::open(fn.toStdString()); + + if (input) { + + PixelFormat::Format image_format = PixelFormat::OIIOFormatToOliveFormat(input->spec().format, + input->spec().nchannels == kRGBAChannels); + + frame = Frame::Create(); + + frame->set_video_params(VideoRenderingParams(input->spec().width, + input->spec().height, + image_format)); + + frame->allocate(); + + input->read_image(input->spec().format, + frame->data(), + OIIO::AutoStride, + frame->linesize_bytes()); + + input->close(); + +#if OIIO_VERSION < 10903 + OIIO::ImageInput::destroy(input); +#endif + + } else { + qWarning() << "OIIO Error:" << OIIO::geterror().c_str(); + } + } + + return frame; +} + void ViewerWidget::UpdateStack() { if (GetConnectedNode() @@ -710,6 +811,10 @@ void ViewerWidget::Pause() } else { disconnect(main_gl_widget(), &ViewerDisplayWidget::frameSwapped, this, &ViewerWidget::PlaybackTimerUpdate); } + + playback_frame_queue_lock_.lock(); + playback_frame_queue_.clear(); + playback_frame_queue_lock_.unlock(); } } diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 164f77122..26abc3985 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -172,6 +172,14 @@ private: void SetColorTransform(const ColorTransform& transform, ViewerDisplayWidget* sender); + void FillPlaybackQueue(); + + QString GetCachedFilenameFromTime(const rational& time); + + bool FrameExistsAtTime(const rational& time); + + FramePtr DecodeCachedImage(const QString& fn); + QStackedWidget* stack_; ViewerSizer* sizer_; @@ -179,7 +187,7 @@ private: qint64 start_msec_; int64_t start_timestamp_; - int playback_speed_; + QAtomicInt playback_speed_; qint64 frame_cache_job_time_; @@ -205,6 +213,15 @@ private: ViewerDisplayWidget* context_menu_widget_; + struct PlaybackFrame { + rational timestamp; + FramePtr frame; + }; + + QMutex playback_frame_queue_lock_; + QLinkedList playback_frame_queue_; + int64_t playback_frame_queue_next_frame_; + private slots: void PlaybackTimerUpdate(); diff --git a/app/widget/viewer/viewerdisplay.cpp b/app/widget/viewer/viewerdisplay.cpp index 3d86eb5d6..68584b836 100644 --- a/app/widget/viewer/viewerdisplay.cpp +++ b/app/widget/viewer/viewerdisplay.cpp @@ -30,6 +30,7 @@ #include #include "common/define.h" +#include "common/functiontimer.h" #include "gizmotraverser.h" #include "render/backend/opengl/openglrenderfunctions.h" #include "render/backend/opengl/openglshader.h" @@ -61,63 +62,6 @@ void ViewerDisplayWidget::SetMatrix(const QMatrix4x4 &mat) update(); } -void ViewerDisplayWidget::SetImage(const QString &fn) -{ - has_image_ = false; - - if (!fn.isEmpty() && QFileInfo::exists(fn)) { - auto input = OIIO::ImageInput::open(fn.toStdString()); - - if (input) { - - PixelFormat::Format image_format = PixelFormat::OIIOFormatToOliveFormat(input->spec().format, - input->spec().nchannels == kRGBAChannels); - - // Ensure the following texture operations are done in our context (in case we're in a separate window for instance) - makeCurrent(); - - if (!texture_.IsCreated() - || texture_.width() != input->spec().width - || texture_.height() != input->spec().height - || texture_.format() != image_format) { - load_buffer_.destroy(); - texture_.Destroy(); - - load_buffer_.set_video_params(VideoRenderingParams(input->spec().width, input->spec().height, image_format)); - load_buffer_.allocate(); - - texture_.Create(context(), VideoRenderingParams(input->spec().width, input->spec().height, image_format)); - } - - input->read_image(input->spec().format, load_buffer_.data(), OIIO::AutoStride, load_buffer_.linesize_bytes()); - input->close(); - - texture_.Upload(&load_buffer_); - - doneCurrent(); - - emit LoadedBuffer(&load_buffer_); - - has_image_ = true; - -#if OIIO_VERSION < 10903 - OIIO::ImageInput::destroy(input); -#endif - - } else { - qWarning() << "OIIO Error:" << OIIO::geterror().c_str(); - } - } - - update(); - - if (has_image_) { - emit LoadedBuffer(&load_buffer_); - } else { - emit LoadedBuffer(nullptr); - } -} - void ViewerDisplayWidget::SetSignalCursorColorEnabled(bool e) { signal_cursor_color_ = e; @@ -144,6 +88,8 @@ void ViewerDisplayWidget::SetImageFromLoadBuffer(Frame *in_buffer) } update(); + + emit LoadedBuffer(in_buffer); } void ViewerDisplayWidget::ConnectSibling(ViewerDisplayWidget *sibling) diff --git a/app/widget/viewer/viewerdisplay.h b/app/widget/viewer/viewerdisplay.h index 996758718..307c350a6 100644 --- a/app/widget/viewer/viewerdisplay.h +++ b/app/widget/viewer/viewerdisplay.h @@ -66,11 +66,6 @@ public: virtual ~ViewerDisplayWidget() override; - /** - * @brief Set an image to load and display on screen - */ - void SetImage(const QString& fn); - const QMatrix4x4& GetMatrix(); void ConnectSibling(ViewerDisplayWidget* sibling);