From ca26f023f2dbf87aa60411d05861fe9d775094c5 Mon Sep 17 00:00:00 2001
From: itsmattkc <34096995+itsmattkc@users.noreply.github.com>
Date: Mon, 30 May 2022 20:55:59 -0700
Subject: [PATCH] viewer: improved frame queuing

---
 app/codec/ffmpeg/ffmpegdecoder.cpp |  8 ++++++
 app/widget/viewer/viewer.cpp       | 43 ++++++++++++++++++++++++------
 app/widget/viewer/viewer.h         |  3 ++-
 3 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp
index 0e8a51ef2..977f1c251 100644
--- a/app/codec/ffmpeg/ffmpegdecoder.cpp
+++ b/app/codec/ffmpeg/ffmpegdecoder.cpp
@@ -145,6 +145,10 @@ bool FFmpegDecoder::OpenInternal()
 TexturePtr FFmpegDecoder::RetrieveVideoInternal(Renderer *renderer, const rational &timecode, const RetrieveVideoParams &params, const QAtomicInt *cancelled)
 {
   if (AVFramePtr f = RetrieveFrame(timecode, cancelled)) {
+    if (cancelled && *cancelled) {
+      return nullptr;
+    }
+
     if (InitScaler(f.get(), params)) {
       VideoParams vp(instance_.avstream()->codecpar->width,
                      instance_.avstream()->codecpar->height,
@@ -790,6 +794,10 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational& time, const QAtomicInt *
     // Pull from the decoder
     ret = instance_.GetFrame(working_packet_, filtered.get());
 
+    if (cancelled && *cancelled) {
+      break;
+    }
+
     // Handle any errors that aren't EOF (EOF is handled later on)
     if (ret < 0 && ret != AVERROR_EOF) {
       qCritical() << "Failed to retrieve frame:" << ret;
diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp
index 41681b009..f807b2d19 100644
--- a/app/widget/viewer/viewer.cpp
+++ b/app/widget/viewer/viewer.cpp
@@ -60,6 +60,8 @@ QVector<ViewerWidget*> ViewerWidget::instances_;
 //       changing values. 1/4 second seems to be a good middleground.
 const rational ViewerWidget::kAudioPlaybackInterval = rational(1, 4);
 
+const rational kVideoPlaybackInterval = rational(1, 2);
+
 ViewerWidget::ViewerWidget(QWidget *parent) :
   super(false, true, parent),
   playback_speed_(0),
@@ -69,6 +71,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) :
   prequeuing_audio_(0),
   record_armed_(false),
   recording_(false),
+  first_requeue_watcher_(nullptr),
   enable_audio_scrubbing_(true)
 {
   // Set up main layout
@@ -635,12 +638,20 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing()
 
 void ViewerWidget::QueueStarved()
 {
-  static const int kMaximumWaitTime = 250;
+  static const int kMaximumWaitTimeMs = 250;
+  static const rational kMaximumWaitTime(kMaximumWaitTimeMs, 1000);
   qint64 now = QDateTime::currentMSecsSinceEpoch();
 
   if (!queue_starved_start_) {
     queue_starved_start_ = now;
-  } else if (now > queue_starved_start_ + kMaximumWaitTime) {
+  } else if (now > queue_starved_start_ + kMaximumWaitTimeMs) {
+    if (first_requeue_watcher_) {
+      if (GetTime() + kMaximumWaitTime < first_requeue_watcher_->property("time").value<rational>()) {
+        // We still have time
+        return;
+      }
+    }
+
     ForceRequeueFromCurrentTime();
     queue_starved_start_ = 0;
   }
@@ -653,11 +664,19 @@ void ViewerWidget::QueueNoLongerStarved()
 
 void ViewerWidget::ForceRequeueFromCurrentTime()
 {
+  // Allow half a second for requeue to complete
+  static const rational kRequeueWaitTime(1);
+
   ClearVideoAutoCacherQueue();
+  queue_watchers_.clear();
   int queue = DeterminePlaybackQueueSize();
-  playback_queue_next_frame_ = GetTimestamp() + playback_speed_;
-  for (int i=queue_watchers_.size(); i<queue; i++) {
-    RequestNextFrameForQueue();
+  playback_queue_next_frame_ = GetTimestamp() + playback_speed_ * Timecode::time_to_timestamp(kRequeueWaitTime, timebase(), Timecode::kFloor);;
+  first_requeue_watcher_ = nullptr;
+  for (int i=0; i<queue; i++) {
+    RenderTicketWatcher *watcher = RequestNextFrameForQueue();
+    if (!first_requeue_watcher_) {
+      first_requeue_watcher_ = watcher;
+    }
   }
 }
 
@@ -899,8 +918,10 @@ void ViewerWidget::SetDisplayImage(QVariant frame)
   }
 }
 
-void ViewerWidget::RequestNextFrameForQueue(RenderTicketPriority priority, bool increment)
+RenderTicketWatcher *ViewerWidget::RequestNextFrameForQueue(RenderTicketPriority priority, bool increment)
 {
+  RenderTicketWatcher *watcher = nullptr;
+
   rational next_time = Timecode::timestamp_to_time(playback_queue_next_frame_,
                                                    timebase());
 
@@ -909,12 +930,14 @@ void ViewerWidget::RequestNextFrameForQueue(RenderTicketPriority priority, bool
       playback_queue_next_frame_ += playback_speed_;
     }
 
-    RenderTicketWatcher* watcher = new RenderTicketWatcher();
+    watcher = new RenderTicketWatcher();
     watcher->setProperty("time", QVariant::fromValue(next_time));
     connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::RendererGeneratedFrameForQueue);
     queue_watchers_.append(watcher);
     watcher->SetTicket(GetFrame(next_time, priority));
   }
+
+  return watcher;
 }
 
 RenderTicketPtr ViewerWidget::GetFrame(const rational &t, RenderTicketPriority priority)
@@ -986,7 +1009,7 @@ int ViewerWidget::DeterminePlaybackQueueSize()
   int remaining_frames = (end_ts - GetTimestamp()) / playback_speed_;
 
   // Generate maximum queue
-  int max_frames = qCeil(kAudioPlaybackInterval.toDouble() / timebase().toDouble());
+  int max_frames = qCeil(kVideoPlaybackInterval.toDouble() / timebase().toDouble());
 
   return qMin(max_frames, remaining_frames);
 }
@@ -1095,6 +1118,10 @@ void ViewerWidget::RendererGeneratedFrameForQueue()
     }
   }
 
+  if (first_requeue_watcher_ == watcher) {
+    first_requeue_watcher_ = nullptr;
+  }
+
   delete watcher;
 }
 
diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h
index b19e8f1f5..6c58ed194 100644
--- a/app/widget/viewer/viewer.h
+++ b/app/widget/viewer/viewer.h
@@ -195,7 +195,7 @@ private:
 
   void SetDisplayImage(QVariant frame);
 
-  void RequestNextFrameForQueue(RenderTicketPriority priority = RenderTicketPriority::kNormal, bool increment = true);
+  RenderTicketWatcher *RequestNextFrameForQueue(RenderTicketPriority priority = RenderTicketPriority::kNormal, bool increment = true);
 
   RenderTicketPtr GetFrame(const rational& t, RenderTicketPriority priority);
 
@@ -280,6 +280,7 @@ private:
   QString recording_filename_;
 
   qint64 queue_starved_start_;
+  RenderTicketWatcher *first_requeue_watcher_;
 
   bool enable_audio_scrubbing_;