reimplemented color management and alpha association in new renderer system

Color management is now as functional as it was before.

app/render/backend/audio/audioworker.cpp

@@ -5,7 +5,8 @@ AudioWorker::AudioWorker(DecoderCache *decoder_cache, QObject *parent) :
 {
 }
 
-void AudioWorker::FrameToValue(FramePtr frame, NodeValueTable *table)
+void AudioWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table)
 {
+  Q_UNUSED(stream)
   table->Push(NodeParam::kSamples, frame->ToByteArray());
 }

app/render/backend/audio/audioworker.h

@@ -9,7 +9,7 @@ public:
   AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr);
 
 protected:
-  virtual void FrameToValue(FramePtr frame, NodeValueTable* table) override;
+  virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override;
 
 private:
 

app/render/backend/opengl/CMakeLists.txt

@@ -20,6 +20,8 @@ set(OLIVE_SOURCES
   render/backend/opengl/functions.cpp
   render/backend/opengl/openglbackend.h
   render/backend/opengl/openglbackend.cpp
+  render/backend/opengl/openglcolorprocessor.h
+  render/backend/opengl/openglcolorprocessor.cpp
   render/backend/opengl/openglframebuffer.h
   render/backend/opengl/openglframebuffer.cpp
   render/backend/opengl/openglshader.h

app/render/backend/opengl/openglcolorprocessor.cpp

@@ -0,0 +1,66 @@
+#include "openglcolorprocessor.h"
+
+#include <QOpenGLContext>
+#include <QOpenGLFunctions>
+
+#include "functions.h"
+
+void OpenGLColorProcessor::Enable(QOpenGLContext *context, bool alpha_is_associated)
+{
+  if (IsEnabled()) {
+    return;
+  }
+
+  context_ = context;
+
+  pipeline_ = OpenGLShader::CreateOCIO(context_,
+                                       ocio_lut_,
+                                       GetProcessor(),
+                                       alpha_is_associated);
+}
+
+bool OpenGLColorProcessor::IsEnabled() const
+{
+  return ocio_lut_;
+}
+
+OpenGLShaderPtr OpenGLColorProcessor::pipeline() const
+{
+  return pipeline_;
+}
+
+void OpenGLColorProcessor::ProcessOpenGL()
+{
+  olive::gl::OCIOBlit(pipeline_, ocio_lut_);
+}
+
+OpenGLColorProcessor::OpenGLColorProcessor(const QString &source_space, const QString &dest_space) :
+  ColorProcessor(source_space, dest_space),
+  ocio_lut_(0)
+{
+}
+
+OpenGLColorProcessor::OpenGLColorProcessor(const QString &source_space, QString display, QString view, const QString &look) :
+  ColorProcessor(source_space, display, view, look),
+  ocio_lut_(0)
+{
+}
+
+OpenGLColorProcessor::~OpenGLColorProcessor()
+{
+  if (IsEnabled()) {
+    // Clean up OCIO LUT texture and shader
+    context_->functions()->glDeleteTextures(1, &ocio_lut_);
+    pipeline_ = nullptr;
+  }
+}
+
+OpenGLColorProcessorPtr OpenGLColorProcessor::CreateOpenGL(const QString &source_space, const QString &dest_space)
+{
+  return std::make_shared<OpenGLColorProcessor>(source_space, dest_space);
+}
+
+OpenGLColorProcessorPtr OpenGLColorProcessor::CreateOpenGL(const QString &source_space, const QString &display, const QString &view, const QString &look)
+{
+  return std::make_shared<OpenGLColorProcessor>(source_space, display, view, look);
+}

app/render/backend/opengl/openglcolorprocessor.h

@@ -0,0 +1,45 @@
+#ifndef OPENGLCOLORPROCESSOR_H
+#define OPENGLCOLORPROCESSOR_H
+
+#include "openglshader.h"
+#include "render/colorprocessor.h"
+
+class OpenGLColorProcessor;
+using OpenGLColorProcessorPtr = std::shared_ptr<OpenGLColorProcessor>;
+
+class OpenGLColorProcessor : public ColorProcessor
+{
+public:
+  OpenGLColorProcessor(const QString &source_space, const QString &dest_space);
+
+  OpenGLColorProcessor(const QString& source_space,
+                       QString display,
+                       QString view,
+                       const QString& look);
+
+  ~OpenGLColorProcessor();
+
+  static OpenGLColorProcessorPtr CreateOpenGL(const QString& source_space, const QString& dest_space);
+
+  static OpenGLColorProcessorPtr CreateOpenGL(const QString& source_space,
+                                              const QString& display,
+                                              const QString& view,
+                                              const QString& look);
+
+  void Enable(QOpenGLContext* context, bool alpha_is_associated);
+  bool IsEnabled() const;
+
+  OpenGLShaderPtr pipeline() const;
+
+  void ProcessOpenGL();
+
+private:
+  QOpenGLContext* context_;
+
+  GLuint ocio_lut_;
+
+  OpenGLShaderPtr pipeline_;
+
+};
+
+#endif // OPENGLCOLORPROCESSOR_H

app/render/backend/opengl/openglworker.cpp

@@ -1,11 +1,14 @@
 #include "openglworker.h"
 
+#include "core.h"
 #include "functions.h"
 #include "node/node.h"
+#include "openglcolorprocessor.h"
+#include "render/colormanager.h"
 #include "render/pixelservice.h"
 
-OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, VideoRenderFrameCache *frame_cache, QObject *parent) :
-  VideoRenderWorker(decoder_cache, frame_cache, parent),
+OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, ColorProcessorCache *color_cache, VideoRenderFrameCache *frame_cache, QObject *parent) :
+  VideoRenderWorker(decoder_cache, color_cache, frame_cache, parent),
   share_ctx_(share_ctx),
   ctx_(nullptr),
   functions_(nullptr),
@@ -47,18 +50,79 @@ bool OpenGLWorker::InitInternal()
   return true;
 }
 
-void OpenGLWorker::FrameToValue(FramePtr frame, NodeValueTable *table)
+void OpenGLWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table)
 {
+  // Set up OCIO context
+  OpenGLColorProcessorPtr color_processor = std::static_pointer_cast<OpenGLColorProcessor>(color_cache()->Get(stream.get()));
+
+  // Ensure stream is video or image type
+  if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) {
+    return;
+  }
+
+  ImageStreamPtr video_stream = std::static_pointer_cast<ImageStream>(stream);
+
+  if (!color_processor) {
+    QString input_colorspace = video_stream->colorspace();
+    if (input_colorspace.isEmpty()) {
+      // FIXME: Should use Stream->Footage to find the Project* since that's a direct chain
+      input_colorspace = olive::core.GetActiveProject()->default_input_colorspace();
+    }
+
+    color_processor = OpenGLColorProcessor::CreateOpenGL(input_colorspace,
+                                                         OCIO::ROLE_SCENE_LINEAR);
+    color_cache()->Add(stream.get(), color_processor);
+  }
+
+  // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU
+  if (video_params().mode() == olive::kOnline) {
+    // If alpha is associated, disassociate for the color transform
+    if (video_stream->premultiplied_alpha()) {
+      ColorManager::DisassociateAlpha(frame);
+    }
+
+    // Convert frame to float for OCIO
+    frame = PixelService::ConvertPixelFormat(frame, olive::PIX_FMT_RGBA32F);
+
+    // Perform color transform
+    color_processor->ConvertFrame(frame);
+
+    // Associate alpha
+    if (video_stream->premultiplied_alpha()) {
+      ColorManager::ReassociateAlpha(frame);
+    } else {
+      ColorManager::AssociateAlpha(frame);
+    }
+  }
+
   OpenGLTexturePtr footage_tex = std::make_shared<OpenGLTexture>();
   footage_tex->Create(ctx_, frame);
 
-  // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU
-  //if (video_params().mode() == olive::kOnline) {
-    // Convert frame to float for
-    //frame = PixelService::ConvertPixelFormat(frame, olive::PIX_FMT_RGBA32F);
-  //}
+  if (video_params().mode() == olive::kOffline) {
+    if (!color_processor->IsEnabled()) {
+      color_processor->Enable(ctx_, video_stream->premultiplied_alpha());
+    }
 
-  // FIXME: Alpha association and color management
+    // Create destination texture
+    OpenGLTexturePtr associated_tex = std::make_shared<OpenGLTexture>();
+    associated_tex->Create(ctx_, footage_tex->width(), footage_tex->height(), footage_tex->format());
+
+    // Set viewport for texture size
+    functions_->glViewport(0, 0, footage_tex->width(), footage_tex->height());
+
+    buffer_.Attach(associated_tex);
+    buffer_.Bind();
+    footage_tex->Bind();
+
+    // Blit old texture to new texture through OCIO shader
+    color_processor->ProcessOpenGL();
+
+    footage_tex->Release();
+    buffer_.Release();
+    buffer_.Detach();
+
+    footage_tex = associated_tex;
+  }
 
   table->Push(NodeParam::kTexture, QVariant::fromValue(footage_tex));
 }
@@ -66,7 +130,6 @@ void OpenGLWorker::FrameToValue(FramePtr frame, NodeValueTable *table)
 void OpenGLWorker::CloseInternal()
 {
   buffer_.Destroy();
-
   functions_ = nullptr;
   delete ctx_;
 }
@@ -184,7 +247,10 @@ void OpenGLWorker::RunNodeAccelerated(Node *node, const NodeValueDatabase *input
     }
   }
 
-  //qDebug() << "    Blitting with shader!";
+  // Ensure viewport is correct
+  functions_->glViewport(0, 0, video_params().effective_width(), video_params().effective_height());
+
+  // Blit this texture through this shader
   olive::gl::Blit(shader);
 
   // Release any textures we bound before

app/render/backend/opengl/openglworker.h

@@ -14,6 +14,7 @@ public:
   OpenGLWorker(QOpenGLContext* share_ctx,
                OpenGLShaderCache* shader_cache,
                DecoderCache* decoder_cache,
+               ColorProcessorCache *color_cache,
                VideoRenderFrameCache* frame_cache,
                QObject* parent = nullptr);
 
@@ -45,7 +46,7 @@ protected:
 
   virtual void CloseInternal() override;
 
-  virtual void FrameToValue(FramePtr frame, NodeValueTable* table) override;
+  virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override;
 
   virtual void RunNodeAccelerated(Node *node, const NodeValueDatabase *input_params, NodeValueTable* output_params) override;
 

app/render/backend/renderworker.cpp

@@ -98,17 +98,16 @@ StreamPtr RenderWorker::ResolveStreamFromInput(NodeInput *input)
   return input->get_value_at_time(0).value<StreamPtr>();
 }
 
-DecoderPtr RenderWorker::ResolveDecoderFromInput(NodeInput *input)
+DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream)
 {
   // Access a map of Node inputs and decoder instances and retrieve a frame!
-  StreamPtr stream = ResolveStreamFromInput(input);
-  DecoderPtr decoder = decoder_cache()->GetDecoder(stream.get());
+  DecoderPtr decoder = decoder_cache()->Get(stream.get());
 
   if (decoder == nullptr && stream != nullptr) {
     // Create a new Decoder here
     DecoderPtr decoder = Decoder::CreateFromID(stream->footage()->decoder());
     decoder->set_stream(stream);
-    decoder_cache()->AddDecoder(stream.get(), decoder);
+    decoder_cache()->Add(stream.get(), decoder);
   }
 
   return decoder;
@@ -146,13 +145,17 @@ NodeValueTable RenderWorker::ProcessNodeNormally(const NodeDependency& dep)
 
       // Exception for Footage types where we actually retrieve some Footage data from a decoder
       if (input->data_type() == NodeParam::kFootage) {
-        DecoderPtr decoder = ResolveDecoderFromInput(input);
+        StreamPtr stream = ResolveStreamFromInput(input);
 
-        if (decoder) {
-          FramePtr frame = RetrieveFromDecoder(decoder, input_time);
+        if (stream) {
+          DecoderPtr decoder = ResolveDecoderFromInput(stream);
 
-          if (frame) {
-            FrameToValue(frame, &table);
+          if (decoder) {
+            FramePtr frame = RetrieveFromDecoder(decoder, input_time);
+
+            if (frame) {
+              FrameToValue(stream, frame, &table);
+            }
           }
         }
       }

app/render/backend/renderworker.h

@@ -46,11 +46,11 @@ protected:
   virtual void RunNodeAccelerated(Node *node, const NodeValueDatabase *input_params, NodeValueTable* output_params);
 
   StreamPtr ResolveStreamFromInput(NodeInput* input);
-  DecoderPtr ResolveDecoderFromInput(NodeInput* input);
+  DecoderPtr ResolveDecoderFromInput(StreamPtr stream);
 
   virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) = 0;
 
-  virtual void FrameToValue(FramePtr frame, NodeValueTable* table) = 0;
+  virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) = 0;
 
   NodeValueTable ProcessNodeNormally(const NodeDependency &dep);
 

app/render/backend/videorenderbackend.cpp

@@ -175,7 +175,7 @@ void VideoRenderBackend::CacheIDChangedEvent(const QString &id)
 void VideoRenderBackend::ConnectWorkerToThis(RenderWorker *processor)
 {
   connect(processor, SIGNAL(CompletedFrame(NodeDependency, QByteArray, NodeValueTable)), this, SLOT(ThreadCompletedFrame(NodeDependency, QByteArray, NodeValueTable)));
-  connect(processor, SIGNAL(HashAlreadyBeingCached()), this, SLOT(ThreadSkippedFrame()));
+  connect(processor, SIGNAL(HashAlreadyBeingCached(NodeDependency, QByteArray)), this, SLOT(ThreadSkippedFrame(NodeDependency, QByteArray)));
   connect(processor, SIGNAL(CompletedDownload(NodeDependency, QByteArray)), this, SLOT(ThreadCompletedDownload(NodeDependency, QByteArray)));
   connect(processor, SIGNAL(HashAlreadyExists(NodeDependency, QByteArray)), this, SLOT(ThreadHashAlreadyExists(NodeDependency, QByteArray)));
 }
@@ -185,6 +185,11 @@ VideoRenderFrameCache *VideoRenderBackend::frame_cache()
   return &frame_cache_;
 }
 
+ColorProcessorCache *VideoRenderBackend::color_cache()
+{
+  return &color_cache_;
+}
+
 const char *VideoRenderBackend::GetCachedFrame(const rational &time)
 {
   last_time_requested_ = time;
@@ -232,3 +237,9 @@ NodeInput *VideoRenderBackend::GetDependentInput()
 {
   return viewer_node()->texture_input();
 }
+
+void VideoRenderBackend::CompletedFrame()
+{
+  caching_ = false;
+  CacheNext();
+}

app/render/backend/videorenderbackend.h

@@ -23,6 +23,7 @@
 
 #include <QLinkedList>
 
+#include "colorprocessorcache.h"
 #include "node/output/viewer/viewer.h"
 #include "renderbackend.h"
 #include "render/pixelformat.h"
@@ -81,6 +82,8 @@ protected:
 
   VideoRenderFrameCache* frame_cache();
 
+  ColorProcessorCache* color_cache();
+
   const VideoRenderingParams& params() const;
 
   /**
@@ -92,6 +95,8 @@ protected:
 
   virtual void ConnectWorkerToThis(RenderWorker* processor) override;
 
+  void CompletedFrame();
+
 signals:
   void CachedFrameReady(const rational& time, QVariant value);
   void CachedTimeReady(const rational& time);
@@ -103,6 +108,8 @@ private:
 
   VideoRenderFrameCache frame_cache_;
 
+  ColorProcessorCache color_cache_;
+
   rational last_time_requested_;
 
 private slots:

app/render/backend/videorenderworker.cpp

@@ -4,9 +4,10 @@
 #include "node/node.h"
 #include "render/pixelservice.h"
 
-VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, VideoRenderFrameCache *frame_cache, QObject *parent) :
+VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, ColorProcessorCache *color_cache, VideoRenderFrameCache *frame_cache, QObject *parent) :
   RenderWorker(decoder_cache, parent),
-  frame_cache_(frame_cache)
+  frame_cache_(frame_cache),
+  color_cache_(color_cache)
 {
 
 }
@@ -36,7 +37,7 @@ NodeValueTable VideoRenderWorker::RenderInternal(const NodeDependency& path)
     emit CompletedFrame(path, hash, value);
   } else {
     // Another thread must be caching this already, nothing to be done
-    emit HashAlreadyBeingCached();
+    emit HashAlreadyBeingCached(path, hash);
   }
 
   return value;
@@ -85,7 +86,7 @@ void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, c
         // We have one exception for FOOTAGE types, since we resolve the footage into a frame in the renderer
         if (input->data_type() == NodeParam::kFootage) {
           StreamPtr stream = ResolveStreamFromInput(input);
-          DecoderPtr decoder = ResolveDecoderFromInput(input);
+          DecoderPtr decoder = ResolveDecoderFromInput(stream);
 
           if (decoder != nullptr) {
             // Add footage details to hash
@@ -99,10 +100,19 @@ void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, c
             // Footage stream
             hash->addData(QString::number(stream->index()).toUtf8());
 
-            // Footage timestamp
-            hash->addData(QString::number(decoder->GetTimestampFromTime(time)).toUtf8());
+            if (stream->type() == Stream::kImage || stream->type() == Stream::kVideo) {
+              ImageStreamPtr video_stream = std::static_pointer_cast<ImageStream>(stream);
 
-            // FIXME: Add colorspace and alpha assoc
+              // Footage timestamp
+              hash->addData(QString::number(decoder->GetTimestampFromTime(time)).toUtf8());
+
+              // Current colorspace
+              // FIXME: Handle empty colorspace...
+              hash->addData(video_stream->colorspace().toUtf8());
+
+              // Alpha associated setting
+              hash->addData(QString::number(video_stream->premultiplied_alpha()).toUtf8());
+            }
           }
         }
       }
@@ -171,3 +181,8 @@ NodeValueTable VideoRenderWorker::RenderBlock(TrackOutput *track, const TimeRang
 
   return table;
 }
+
+ColorProcessorCache *VideoRenderWorker::color_cache() const
+{
+  return color_cache_;
+}

app/render/backend/videorenderworker.h

@@ -3,6 +3,7 @@
 
 #include <QCryptographicHash>
 
+#include "colorprocessorcache.h"
 #include "node/dependency.h"
 #include "render/videoparams.h"
 #include "renderworker.h"
@@ -11,7 +12,7 @@
 class VideoRenderWorker : public RenderWorker {
   Q_OBJECT
 public:
-  VideoRenderWorker(DecoderCache* decoder_cache, VideoRenderFrameCache* frame_cache, QObject* parent = nullptr);
+  VideoRenderWorker(DecoderCache* decoder_cache, ColorProcessorCache* color_cache, VideoRenderFrameCache* frame_cache, QObject* parent = nullptr);
 
   void SetParameters(const VideoRenderingParams& video_params);
 
@@ -23,7 +24,7 @@ signals:
 
   void CompletedDownload(NodeDependency path, QByteArray hash);
 
-  void HashAlreadyBeingCached();
+  void HashAlreadyBeingCached(NodeDependency path, QByteArray hash);
 
   void HashAlreadyExists(NodeDependency path, QByteArray hash);
 
@@ -44,6 +45,8 @@ protected:
 
   virtual NodeValueTable RenderBlock(TrackOutput *track, const TimeRange& range) override;
 
+  ColorProcessorCache* color_cache() const;
+
 private:
   void ProcessNode();
 
@@ -53,6 +56,8 @@ private:
 
   VideoRenderFrameCache* frame_cache_;
 
+  ColorProcessorCache* color_cache_;
+
   QByteArray download_buffer_;
 
 private slots:

app/render/colorprocessor.h

@@ -4,8 +4,10 @@
 #include <OpenColorIO/OpenColorIO.h>
 namespace OCIO = OCIO_NAMESPACE::v1;
 
+#include "common/constructors.h"
 #include "decoder/frame.h"
 
+
 class ColorProcessor;
 using ColorProcessorPtr = std::shared_ptr<ColorProcessor>;
 
@@ -19,6 +21,8 @@ public:
                  QString view,
                  const QString& look);
 
+  DISABLE_COPY_MOVE(ColorProcessor)
+
   static ColorProcessorPtr Create(const QString& source_space, const QString& dest_space);
 
   static ColorProcessorPtr Create(const QString& source_space,