diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp
index 038f063b7..d80b11775 100644
--- a/app/codec/ffmpeg/ffmpegdecoder.cpp
+++ b/app/codec/ffmpeg/ffmpegdecoder.cpp
@@ -98,7 +98,7 @@ bool FFmpegDecoder::Open()
         frame_pool->SetParams(our_instance_->stream()->codecpar->width,
                               our_instance_->stream()->codecpar->height,
                               static_cast<AVPixelFormat>(our_instance_->stream()->codecpar->format));
-        frame_pool->Allocate(128);
+        frame_pool->Allocate(256);
         frame_pool_map_.insert(stream().get(), frame_pool);
       }
 
@@ -1015,6 +1015,10 @@ FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t&
         previous = cached_frames_.last();
       }
 
+      // Clear early frames
+      // FIXME: Hardcoded value (only stores a maximum of 2 seconds in the cache at any time)
+      TruncateCacheRangeTo(2*second_ts_);
+
       // Append this frame and signal to other threads that a new frame has arrived
       cached_frames_.append(cached);
 
@@ -1164,6 +1168,15 @@ void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t)
   }
 }
 
+void FFmpegDecoderInstance::TruncateCacheRangeTo(const qint64 &t)
+{
+  // We keep one frame in memory as an identifier for what pts the decoder is up to
+  while (cached_frames_.size() > 1 && (RangeEnd() - RangeStart()) > t) {
+    cached_frames_.removeFirst();
+    cache_at_zero_ = false;
+  }
+}
+
 rational FFmpegDecoderInstance::sample_aspect_ratio() const
 {
   return av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr);
diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h
index c5ebaa39c..37558d59e 100644
--- a/app/codec/ffmpeg/ffmpegdecoder.h
+++ b/app/codec/ffmpeg/ffmpegdecoder.h
@@ -62,6 +62,7 @@ public:
   FFmpegFramePool::ElementPtr GetFrameFromCache(const int64_t& t) const;
 
   void RemoveFramesBefore(const qint64& t);
+  void TruncateCacheRangeTo(const qint64& t);
 
   rational sample_aspect_ratio() const;
   AVStream* stream() const;