diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp
index d124240b4..d87466814 100644
--- a/app/codec/ffmpeg/ffmpegdecoder.cpp
+++ b/app/codec/ffmpeg/ffmpegdecoder.cpp
@@ -107,9 +107,11 @@ bool FFmpegDecoder::Open()
     FFmpegFramePoolValue& frame_pool = frame_pool_map_[key];
 
     if (!frame_pool.pool) {
-      // FIXME: Hardcoded value. It seems to work fine, but is there a possibility we should make
-      //        this a dynamic value somehow or a configurable value?
-      frame_pool.pool = new FFmpegFramePool(32);
+      // Frames are allocated as threads * threads, to scale from each thread sharing one set
+      // to all of them working individually
+      int thread_count = QThread::idealThreadCount();
+      int max_memory_frame_count = thread_count * thread_count;
+      frame_pool.pool = new FFmpegFramePool(max_memory_frame_count);
     }
     frame_pool.handles++;
 
@@ -1114,6 +1116,9 @@ FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t&
         break;
       }
 
+      // Cut down to thread count - 1 before we acquire a new frame
+      TruncateCacheRangeToFrames(QThread::idealThreadCount() -1);
+
       FFmpegFramePool::ElementPtr cached = frame_pool_->Get();
 
       if (!cached) {
@@ -1154,14 +1159,6 @@ FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t&
         previous = cached_frames_.last();
       }
 
-      // Clear early frames
-      int removed = TruncateCacheRangeTo(second_ts_);
-
-      if (removed == 0 && MemoryPoolLimitReached()) {
-        // Relinquish a frame if we have to conserve memory
-        RemoveFirstFrame();
-      }
-
       // Append this frame and signal to other threads that a new frame has arrived
       cached_frames_.append(cached);
 
@@ -1349,7 +1346,7 @@ void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t)
   }
 }
 
-int FFmpegDecoderInstance::TruncateCacheRangeTo(const qint64 &t)
+int FFmpegDecoderInstance::TruncateCacheRangeToTime(const qint64 &t)
 {
   int counter = 0;
 
@@ -1362,6 +1359,19 @@ int FFmpegDecoderInstance::TruncateCacheRangeTo(const qint64 &t)
   return counter;
 }
 
+int FFmpegDecoderInstance::TruncateCacheRangeToFrames(int nb_frames)
+{
+  int counter = 0;
+
+  // We keep one frame in memory as an identifier for what pts the decoder is up to
+  while (cached_frames_.size() > nb_frames) {
+    RemoveFirstFrame();
+    counter++;
+  }
+
+  return counter;
+}
+
 void FFmpegDecoderInstance::RemoveFirstFrame()
 {
   cached_frames_.removeFirst();
diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h
index e8a5ed296..7f909461b 100644
--- a/app/codec/ffmpeg/ffmpegdecoder.h
+++ b/app/codec/ffmpeg/ffmpegdecoder.h
@@ -62,7 +62,8 @@ public:
   FFmpegFramePool::ElementPtr GetFrameFromCache(const int64_t& t) const;
 
   void RemoveFramesBefore(const qint64& t);
-  int TruncateCacheRangeTo(const qint64& t);
+  int TruncateCacheRangeToTime(const qint64& t);
+  int TruncateCacheRangeToFrames(int nb_frames);
   void RemoveFirstFrame();
 
   AVFormatContext* fmt_ctx() const