renderer: re-use already created textures if the images are known not to change

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -49,6 +49,7 @@ FFmpegDecoder::FFmpegDecoder() :
   cache_at_eof_(false),
   opts_(nullptr)
 {
+  // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints
   clear_timer_.setInterval(250);
   clear_timer_.moveToThread(qApp->thread());
   connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent);

app/render/backend/audio/audioworker.cpp

@@ -5,11 +5,22 @@ AudioWorker::AudioWorker(DecoderCache* decoder_cache, QObject *parent) :
 {
 }
 
-void AudioWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table)
+void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table)
 {
-  Q_UNUSED(stream)
+  if (stream->type() != Stream::kAudio) {
+    return;
+  }
 
-  table->Push(NodeParam::kSamples, frame->ToByteArray());
+  if (decoder->HasConformedVersion(audio_params())) {
+    FramePtr frame = nullptr;
+    frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params());
+
+    if (frame) {
+      table->Push(NodeParam::kSamples, frame->ToByteArray());
+    }
+  } else {
+    emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params());
+  }
 }
 
 void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params_in, NodeValueTable *output_params)

app/render/backend/audio/audioworker.h

@@ -9,7 +9,7 @@ public:
   AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr);
 
 protected:
-  virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override;
+  virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override;
 
   virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, const NodeValueDatabase& input_params, NodeValueTable* output_params) override;
 

app/render/backend/audiorenderworker.cpp

@@ -23,16 +23,6 @@ void AudioRenderWorker::CloseInternal()
   // Nothing to init yet
 }
 
-FramePtr AudioRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range)
-{
-  if (decoder->HasConformedVersion(audio_params_)) {
-    return decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params_);
-  } else {
-    emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params_);
-    return nullptr;
-  }
-}
-
 NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const TimeRange &range)
 {
   QList<Block*> active_blocks = track->BlocksAtTimeRange(range);

app/render/backend/audiorenderworker.h

@@ -19,8 +19,6 @@ protected:
 
   virtual void CloseInternal() override;
 
-  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) override;
-
   virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) override;
 
   const AudioRenderingParams& audio_params() const;

app/render/backend/opengl/openglproxy.cpp

@@ -43,7 +43,7 @@ bool OpenGLProxy::Init()
   return true;
 }
 
-void OpenGLProxy::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table)
+void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table)
 {
   // Ensure stream is video or image type
   if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) {
@@ -55,86 +55,107 @@ void OpenGLProxy::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable
   // Set up OCIO context
   QString colorspace_match = QStringLiteral("%1:%2").arg(video_stream->footage()->project()->ocio_config(), video_stream->colorspace());
 
-  OpenGLColorProcessorPtr color_processor = std::static_pointer_cast<OpenGLColorProcessor>(color_cache_.Get(colorspace_match));
+  OpenGLTextureCache::ReferencePtr footage_tex_ref = nullptr;
 
-  if (!color_processor) {
-    color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager()->GetConfig(),
-                                                         video_stream->colorspace(),
-                                                         OCIO::ROLE_SCENE_LINEAR);
-    color_cache_.Add(colorspace_match, color_processor);
-  }
+  if (stream->type() == Stream::kImage && still_image_cache_.Has(stream.get())) {
+    CachedStill cs = still_image_cache_.Get(stream.get());
 
-  ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode());
-
-  // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU
-  if (ocio_method == ColorManager::kOCIOAccurate) {
-    // If alpha is associated, disassociate for the color transform
-    if (video_stream->premultiplied_alpha()) {
-      ColorManager::DisassociateAlpha(frame);
-    }
-
-    // Convert frame to float for OCIO
-    frame = PixelService::ConvertPixelFormat(frame, PixelFormat::PIX_FMT_RGBA32F);
-
-    // Perform color transform
-    color_processor->ConvertFrame(frame);
-
-    // Associate alpha
-    if (video_stream->premultiplied_alpha()) {
-      ColorManager::ReassociateAlpha(frame);
+    if (cs.colorspace == colorspace_match && cs.alpha_is_associated == video_stream->premultiplied_alpha()) {
+      footage_tex_ref = cs.texture;
     } else {
-      ColorManager::AssociateAlpha(frame);
+      still_image_cache_.Remove(stream.get());
     }
   }
 
-  VideoRenderingParams footage_params(frame->width(), frame->height(), stream->timebase(), frame->format(), video_params_.mode());
+  // Since this is a still image, we could likely optimize this
+  if (!footage_tex_ref) {
+    OpenGLColorProcessorPtr color_processor = std::static_pointer_cast<OpenGLColorProcessor>(color_cache_.Get(colorspace_match));
 
-  OpenGLTextureCache::ReferencePtr footage_tex_ref = texture_cache_.Get(ctx_, footage_params, frame->data());
-
-  if (ocio_method == ColorManager::kOCIOFast) {
-    if (!color_processor->IsEnabled()) {
-      color_processor->Enable(ctx_, video_stream->premultiplied_alpha());
+    if (!color_processor) {
+      color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager()->GetConfig(),
+                                                           video_stream->colorspace(),
+                                                           OCIO::ROLE_SCENE_LINEAR);
+      color_cache_.Add(colorspace_match, color_processor);
     }
 
-    // Check frame aspect ratio
-    if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) {
-      int new_width = frame->width();
-      int new_height = frame->height();
+    ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode());
 
-      // Scale the frame in a way that does not reduce the resolution
-      if (frame->sample_aspect_ratio() > 1) {
-        // Make wider
-        new_width = qRound(static_cast<double>(new_width) * frame->sample_aspect_ratio().toDouble());
-      } else {
-        // Make taller
-        new_height = qRound(static_cast<double>(new_height) / frame->sample_aspect_ratio().toDouble());
+    FramePtr frame = decoder->RetrieveVideo(range.in());;
+
+    // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU
+    if (ocio_method == ColorManager::kOCIOAccurate) {
+      // If alpha is associated, disassociate for the color transform
+      if (video_stream->premultiplied_alpha()) {
+        ColorManager::DisassociateAlpha(frame);
       }
 
-      footage_params = VideoRenderingParams(new_width,
-                                            new_height,
-                                            footage_params.time_base(),
-                                            footage_params.format(),
-                                            footage_params.mode());
+      // Convert frame to float for OCIO
+      frame = PixelService::ConvertPixelFormat(frame, PixelFormat::PIX_FMT_RGBA32F);
+
+      // Perform color transform
+      color_processor->ConvertFrame(frame);
+
+      // Associate alpha
+      if (video_stream->premultiplied_alpha()) {
+        ColorManager::ReassociateAlpha(frame);
+      } else {
+        ColorManager::AssociateAlpha(frame);
+      }
     }
 
-    // Create destination texture
-    OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, footage_params);
+    VideoRenderingParams footage_params(frame->width(), frame->height(), stream->timebase(), frame->format(), video_params_.mode());
 
-    buffer_.Attach(associated_tex_ref->texture(), true);
-    buffer_.Bind();
-    footage_tex_ref->texture()->Bind();
+    footage_tex_ref = texture_cache_.Get(ctx_, footage_params, frame->data());
 
-    // Set viewport for texture size
-    functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height());
+    if (ocio_method == ColorManager::kOCIOFast) {
+      if (!color_processor->IsEnabled()) {
+        color_processor->Enable(ctx_, video_stream->premultiplied_alpha());
+      }
 
-    // Blit old texture to new texture through OCIO shader
-    color_processor->ProcessOpenGL();
+      // Check frame aspect ratio
+      if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) {
+        int new_width = frame->width();
+        int new_height = frame->height();
 
-    footage_tex_ref->texture()->Release();
-    buffer_.Release();
-    buffer_.Detach();
+        // Scale the frame in a way that does not reduce the resolution
+        if (frame->sample_aspect_ratio() > 1) {
+          // Make wider
+          new_width = qRound(static_cast<double>(new_width) * frame->sample_aspect_ratio().toDouble());
+        } else {
+          // Make taller
+          new_height = qRound(static_cast<double>(new_height) / frame->sample_aspect_ratio().toDouble());
+        }
 
-    footage_tex_ref = associated_tex_ref;
+        footage_params = VideoRenderingParams(new_width,
+                                              new_height,
+                                              footage_params.time_base(),
+                                              footage_params.format(),
+                                              footage_params.mode());
+      }
+
+      // Create destination texture
+      OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, footage_params);
+
+      buffer_.Attach(associated_tex_ref->texture(), true);
+      buffer_.Bind();
+      footage_tex_ref->texture()->Bind();
+
+      // Set viewport for texture size
+      functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height());
+
+      // Blit old texture to new texture through OCIO shader
+      color_processor->ProcessOpenGL();
+
+      footage_tex_ref->texture()->Release();
+      buffer_.Release();
+      buffer_.Detach();
+
+      footage_tex_ref = associated_tex_ref;
+    }
+
+    if (stream->type() == Stream::kImage) {
+      still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha()});
+    }
   }
 
   table->Push(NodeParam::kTexture, QVariant::fromValue(footage_tex_ref));

app/render/backend/opengl/openglproxy.h

@@ -41,7 +41,7 @@ public:
 
   void Close();
 
-  void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table);
+  void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table);
 
   void RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params);
 
@@ -65,6 +65,14 @@ private:
 
   OpenGLTextureCache texture_cache_;
 
+  struct CachedStill {
+    OpenGLTextureCache::ReferencePtr texture;
+    QString colorspace;
+    bool alpha_is_associated;
+  };
+
+  RenderCache<Stream*, CachedStill> still_image_cache_;
+
 private slots:
   void FinishInit();
 

app/render/backend/opengl/openglworker.cpp

@@ -14,9 +14,9 @@ OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, DecoderCache* dec
 {
 }
 
-void OpenGLWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table)
+void OpenGLWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable *table)
 {
-  emit RequestFrameToValue(stream, frame, table);
+  emit RequestFrameToValue(decoder, stream, range, table);
 }
 
 void OpenGLWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable *output_params)

app/render/backend/opengl/openglworker.h

@@ -16,14 +16,14 @@ public:
                QObject* parent = nullptr);
 
 signals:
-  void RequestFrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table);
+  void RequestFrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table);
 
   void RequestRunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params);
 
   void RequestTextureToBuffer(const QVariant& texture, QByteArray& buffer);
 
 protected:
-  virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override;
+  virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override;
 
   virtual void RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params) override;
 

app/render/backend/rendercache.h

@@ -18,6 +18,8 @@ public:
 
   bool Has(K key) const {return values_.contains(key);}
 
+  void Remove(K key) {values_.remove(key);}
+
 private:
   QHash<K, V> values_;
 

app/render/backend/renderworker.cpp

@@ -135,6 +135,8 @@ const NodeDependency &RenderWorker::CurrentPath() const
   return path_;
 }
 
+
+#include "common/functiontimer.h"
 NodeValueDatabase RenderWorker::GenerateDatabase(const Node* node, const TimeRange &range)
 {
   NodeValueDatabase database;
@@ -163,11 +165,7 @@ NodeValueDatabase RenderWorker::GenerateDatabase(const Node* node, const TimeRan
             Decoder::RetrieveState state = decoder->GetRetrieveState(input_time.out());
 
             if (state == Decoder::kReady) {
-              FramePtr frame = RetrieveFromDecoder(decoder, input_time);
-
-              if (frame) {
-                FrameToValue(stream, frame, &table);
-              }
+              FrameToValue(decoder, stream, input_time, &table);
             } else {
               ReportUnavailableFootage(stream, state, input_time.out());
             }

app/render/backend/renderworker.h

@@ -41,9 +41,7 @@ protected:
   StreamPtr ResolveStreamFromInput(NodeInput* input);
   DecoderPtr ResolveDecoderFromInput(StreamPtr stream);
 
-  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) = 0;
-
-  virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) = 0;
+  virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) = 0;
 
   NodeValueTable ProcessNode(const NodeDependency &dep);
 

app/render/backend/videorenderworker.cpp

@@ -87,11 +87,6 @@ NodeValueTable VideoRenderWorker::RenderInternal(const NodeDependency& path, con
   return value;
 }
 
-FramePtr VideoRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range)
-{
-  return decoder->RetrieveVideo(range.in());
-}
-
 void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, const Node* n, const rational& time)
 {
   // Resolve BlockList

app/render/backend/videorenderworker.h

@@ -71,8 +71,6 @@ protected:
 
   virtual NodeValueTable RenderInternal(const NodeDependency& CurrentPath, const qint64& job_time) override;
 
-  virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) override;
-
   virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) override;
 
   virtual void ReportUnavailableFootage(StreamPtr stream, Decoder::RetrieveState state, const rational& stream_time) override;