diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp
index a2ebad125..bfd53c4f0 100644
--- a/app/codec/ffmpeg/ffmpegdecoder.cpp
+++ b/app/codec/ffmpeg/ffmpegdecoder.cpp
@@ -838,7 +838,7 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational& time, const QAtomicInt *
     } else {
 
       // Cut down to thread count - 1 before we acquire a new frame
-      if (cached_frames_.size() == size_t(MaximumQueueSize())) {
+      if (cached_frames_.size() > size_t(MaximumQueueSize())) {
         RemoveFirstFrame();
       }
 
@@ -1041,7 +1041,11 @@ void FFmpegDecoder::RemoveFirstFrame()
 
 int FFmpegDecoder::MaximumQueueSize()
 {
-  return QThread::idealThreadCount();
+  // Fairly arbitrary size. This used to need to be the number of current threads to ensure any
+  // thread that arrived would have its frame available, but if we only have one render thread,
+  // that's no longer a concern. Now, this value could technically be 1, but some memory cache
+  // may be useful for reversing. This value may be tweaked over time.
+  return 2;
 }
 
 FFmpegDecoder::Instance::Instance() :
diff --git a/app/render/rendermanager.cpp b/app/render/rendermanager.cpp
index 80ccdaf70..6281a380b 100644
--- a/app/render/rendermanager.cpp
+++ b/app/render/rendermanager.cpp
@@ -38,8 +38,9 @@ namespace olive {
 RenderManager* RenderManager::instance_ = nullptr;
 
 RenderManager::RenderManager(QObject *parent) :
-  ThreadPool(0, parent),
-  backend_(kOpenGL)
+  ThreadPool(1, parent),
+  backend_(kOpenGL),
+  aggressive_gc_(0)
 {
   Renderer* graphics_renderer = nullptr;
 
@@ -60,10 +61,10 @@ RenderManager::RenderManager(QObject *parent) :
     decoder_cache_ = nullptr;
   }
 
-  QTimer *decoder_clear_timer = new QTimer(this);
-  decoder_clear_timer->setInterval(kDecoderMaximumInactivity);
-  connect(decoder_clear_timer, &QTimer::timeout, this, &RenderManager::ClearOldDecoders);
-  decoder_clear_timer->start();
+  decoder_clear_timer_ = new QTimer(this);
+  decoder_clear_timer_->setInterval(kDecoderMaximumInactivity);
+  connect(decoder_clear_timer_, &QTimer::timeout, this, &RenderManager::ClearOldDecoders);
+  decoder_clear_timer_->start();
 }
 
 RenderManager::~RenderManager()
@@ -162,6 +163,17 @@ void RenderManager::RunTicket(RenderTicketPtr ticket) const
   RenderProcessor::Process(ticket, context_, decoder_cache_, shader_cache_);
 }
 
+void RenderManager::SetAggressiveGarbageCollection(bool enabled)
+{
+  aggressive_gc_ += enabled ? 1 : -1;
+
+  if (aggressive_gc_ > 0) {
+    decoder_clear_timer_->setInterval(kDecoderMaximumInactivityAggressive);
+  } else {
+    decoder_clear_timer_->setInterval(kDecoderMaximumInactivity);
+  }
+}
+
 void RenderManager::ClearOldDecoders()
 {
   QMutexLocker locker(decoder_cache_->mutex());
diff --git a/app/render/rendermanager.h b/app/render/rendermanager.h
index 24d29e34a..6452f4f0b 100644
--- a/app/render/rendermanager.h
+++ b/app/render/rendermanager.h
@@ -113,6 +113,9 @@ public:
     return QThread::idealThreadCount();
   }
 
+public slots:
+  void SetAggressiveGarbageCollection(bool enabled);
+
 signals:
 
 private:
@@ -130,7 +133,12 @@ private:
 
   ShaderCache* shader_cache_;
 
-  static constexpr auto kDecoderMaximumInactivity = 10000;
+  static constexpr auto kDecoderMaximumInactivityAggressive = 1000;
+  static constexpr auto kDecoderMaximumInactivity = 5000;
+
+  int aggressive_gc_;
+
+  QTimer *decoder_clear_timer_;
 
 private slots:
   void ClearOldDecoders();
diff --git a/app/task/render/render.cpp b/app/task/render/render.cpp
index 2968f6d5a..23fa2e387 100644
--- a/app/task/render/render.cpp
+++ b/app/task/render/render.cpp
@@ -44,6 +44,8 @@ bool RenderTask::Render(ColorManager* manager,
                         const QMatrix4x4 &force_matrix, VideoParams::Format force_format,
                         ColorProcessorPtr force_color_output)
 {
+  QMetaObject::invokeMethod(RenderManager::instance(), "SetAggressiveGarbageCollection", Q_ARG(bool, true));
+
   // Run watchers in another thread so they can accept signals even while this thread is blocked
   QThread watcher_thread;
   watcher_thread.start();
@@ -236,6 +238,8 @@ bool RenderTask::Render(ColorManager* manager,
   watcher_thread.quit();
   watcher_thread.wait();
 
+  QMetaObject::invokeMethod(RenderManager::instance(), "SetAggressiveGarbageCollection", Q_ARG(bool, false));
+
   return result;
 }
 
diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp
index 801512e60..e0f01c404 100644
--- a/app/widget/viewer/viewer.cpp
+++ b/app/widget/viewer/viewer.cpp
@@ -765,6 +765,8 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only)
     viewer->auto_cacher_.SetAudioPaused(true);
   }
 
+  RenderManager::instance()->SetAggressiveGarbageCollection(true);
+
   // Disarm recording if armed
   if (record_armed_) {
     DisarmRecording();
@@ -861,6 +863,8 @@ void ViewerWidget::PauseInternal()
     }
 
     UpdateTextureFromNode();
+
+    RenderManager::instance()->SetAggressiveGarbageCollection(false);
   }
 
   prequeuing_video_ = false;
@@ -1108,7 +1112,9 @@ void ViewerWidget::RendererGeneratedFrameForQueue()
             prequeuing_video_ = false;
             FinishPlayPreprocess();
           } else {
-            RequestNextFrameForQueue();
+            // This call was mostly necessary to keep the threads busy between prequeue and playback.
+            // If we only have a single render thread, it's no longer necessary.
+            //RequestNextFrameForQueue();
           }
         }
       }