diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index 70a21836f..f48f43f50 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -5,7 +5,8 @@ AudioWorker::AudioWorker(DecoderCache *decoder_cache, QObject *parent) : { } -void AudioWorker::FrameToValue(FramePtr frame, NodeValueTable *table) +void AudioWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table) { + Q_UNUSED(stream) table->Push(NodeParam::kSamples, frame->ToByteArray()); } diff --git a/app/render/backend/audio/audioworker.h b/app/render/backend/audio/audioworker.h index a9598d06b..686888dee 100644 --- a/app/render/backend/audio/audioworker.h +++ b/app/render/backend/audio/audioworker.h @@ -9,7 +9,7 @@ public: AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); protected: - virtual void FrameToValue(FramePtr frame, NodeValueTable* table) override; + virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override; private: diff --git a/app/render/backend/opengl/CMakeLists.txt b/app/render/backend/opengl/CMakeLists.txt index 9bc6fd91e..631879dcf 100644 --- a/app/render/backend/opengl/CMakeLists.txt +++ b/app/render/backend/opengl/CMakeLists.txt @@ -20,6 +20,8 @@ set(OLIVE_SOURCES render/backend/opengl/functions.cpp render/backend/opengl/openglbackend.h render/backend/opengl/openglbackend.cpp + render/backend/opengl/openglcolorprocessor.h + render/backend/opengl/openglcolorprocessor.cpp render/backend/opengl/openglframebuffer.h render/backend/opengl/openglframebuffer.cpp render/backend/opengl/openglshader.h diff --git a/app/render/backend/opengl/openglcolorprocessor.cpp b/app/render/backend/opengl/openglcolorprocessor.cpp new file mode 100644 index 000000000..052f301b1 --- /dev/null +++ b/app/render/backend/opengl/openglcolorprocessor.cpp @@ -0,0 +1,66 @@ +#include "openglcolorprocessor.h" + +#include +#include + +#include "functions.h" + +void OpenGLColorProcessor::Enable(QOpenGLContext *context, bool alpha_is_associated) +{ + if (IsEnabled()) { + return; + } + + context_ = context; + + pipeline_ = OpenGLShader::CreateOCIO(context_, + ocio_lut_, + GetProcessor(), + alpha_is_associated); +} + +bool OpenGLColorProcessor::IsEnabled() const +{ + return ocio_lut_; +} + +OpenGLShaderPtr OpenGLColorProcessor::pipeline() const +{ + return pipeline_; +} + +void OpenGLColorProcessor::ProcessOpenGL() +{ + olive::gl::OCIOBlit(pipeline_, ocio_lut_); +} + +OpenGLColorProcessor::OpenGLColorProcessor(const QString &source_space, const QString &dest_space) : + ColorProcessor(source_space, dest_space), + ocio_lut_(0) +{ +} + +OpenGLColorProcessor::OpenGLColorProcessor(const QString &source_space, QString display, QString view, const QString &look) : + ColorProcessor(source_space, display, view, look), + ocio_lut_(0) +{ +} + +OpenGLColorProcessor::~OpenGLColorProcessor() +{ + if (IsEnabled()) { + // Clean up OCIO LUT texture and shader + context_->functions()->glDeleteTextures(1, &ocio_lut_); + pipeline_ = nullptr; + } +} + +OpenGLColorProcessorPtr OpenGLColorProcessor::CreateOpenGL(const QString &source_space, const QString &dest_space) +{ + return std::make_shared(source_space, dest_space); +} + +OpenGLColorProcessorPtr OpenGLColorProcessor::CreateOpenGL(const QString &source_space, const QString &display, const QString &view, const QString &look) +{ + return std::make_shared(source_space, display, view, look); +} diff --git a/app/render/backend/opengl/openglcolorprocessor.h b/app/render/backend/opengl/openglcolorprocessor.h new file mode 100644 index 000000000..9ab7d955f --- /dev/null +++ b/app/render/backend/opengl/openglcolorprocessor.h @@ -0,0 +1,45 @@ +#ifndef OPENGLCOLORPROCESSOR_H +#define OPENGLCOLORPROCESSOR_H + +#include "openglshader.h" +#include "render/colorprocessor.h" + +class OpenGLColorProcessor; +using OpenGLColorProcessorPtr = std::shared_ptr; + +class OpenGLColorProcessor : public ColorProcessor +{ +public: + OpenGLColorProcessor(const QString &source_space, const QString &dest_space); + + OpenGLColorProcessor(const QString& source_space, + QString display, + QString view, + const QString& look); + + ~OpenGLColorProcessor(); + + static OpenGLColorProcessorPtr CreateOpenGL(const QString& source_space, const QString& dest_space); + + static OpenGLColorProcessorPtr CreateOpenGL(const QString& source_space, + const QString& display, + const QString& view, + const QString& look); + + void Enable(QOpenGLContext* context, bool alpha_is_associated); + bool IsEnabled() const; + + OpenGLShaderPtr pipeline() const; + + void ProcessOpenGL(); + +private: + QOpenGLContext* context_; + + GLuint ocio_lut_; + + OpenGLShaderPtr pipeline_; + +}; + +#endif // OPENGLCOLORPROCESSOR_H diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index 9f95b0abd..dda783874 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -1,11 +1,14 @@ #include "openglworker.h" +#include "core.h" #include "functions.h" #include "node/node.h" +#include "openglcolorprocessor.h" +#include "render/colormanager.h" #include "render/pixelservice.h" -OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, VideoRenderFrameCache *frame_cache, QObject *parent) : - VideoRenderWorker(decoder_cache, frame_cache, parent), +OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, ColorProcessorCache *color_cache, VideoRenderFrameCache *frame_cache, QObject *parent) : + VideoRenderWorker(decoder_cache, color_cache, frame_cache, parent), share_ctx_(share_ctx), ctx_(nullptr), functions_(nullptr), @@ -47,18 +50,79 @@ bool OpenGLWorker::InitInternal() return true; } -void OpenGLWorker::FrameToValue(FramePtr frame, NodeValueTable *table) +void OpenGLWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table) { + // Set up OCIO context + OpenGLColorProcessorPtr color_processor = std::static_pointer_cast(color_cache()->Get(stream.get())); + + // Ensure stream is video or image type + if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) { + return; + } + + ImageStreamPtr video_stream = std::static_pointer_cast(stream); + + if (!color_processor) { + QString input_colorspace = video_stream->colorspace(); + if (input_colorspace.isEmpty()) { + // FIXME: Should use Stream->Footage to find the Project* since that's a direct chain + input_colorspace = olive::core.GetActiveProject()->default_input_colorspace(); + } + + color_processor = OpenGLColorProcessor::CreateOpenGL(input_colorspace, + OCIO::ROLE_SCENE_LINEAR); + color_cache()->Add(stream.get(), color_processor); + } + + // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU + if (video_params().mode() == olive::kOnline) { + // If alpha is associated, disassociate for the color transform + if (video_stream->premultiplied_alpha()) { + ColorManager::DisassociateAlpha(frame); + } + + // Convert frame to float for OCIO + frame = PixelService::ConvertPixelFormat(frame, olive::PIX_FMT_RGBA32F); + + // Perform color transform + color_processor->ConvertFrame(frame); + + // Associate alpha + if (video_stream->premultiplied_alpha()) { + ColorManager::ReassociateAlpha(frame); + } else { + ColorManager::AssociateAlpha(frame); + } + } + OpenGLTexturePtr footage_tex = std::make_shared(); footage_tex->Create(ctx_, frame); - // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU - //if (video_params().mode() == olive::kOnline) { - // Convert frame to float for - //frame = PixelService::ConvertPixelFormat(frame, olive::PIX_FMT_RGBA32F); - //} + if (video_params().mode() == olive::kOffline) { + if (!color_processor->IsEnabled()) { + color_processor->Enable(ctx_, video_stream->premultiplied_alpha()); + } - // FIXME: Alpha association and color management + // Create destination texture + OpenGLTexturePtr associated_tex = std::make_shared(); + associated_tex->Create(ctx_, footage_tex->width(), footage_tex->height(), footage_tex->format()); + + // Set viewport for texture size + functions_->glViewport(0, 0, footage_tex->width(), footage_tex->height()); + + buffer_.Attach(associated_tex); + buffer_.Bind(); + footage_tex->Bind(); + + // Blit old texture to new texture through OCIO shader + color_processor->ProcessOpenGL(); + + footage_tex->Release(); + buffer_.Release(); + buffer_.Detach(); + + footage_tex = associated_tex; + } table->Push(NodeParam::kTexture, QVariant::fromValue(footage_tex)); } @@ -66,7 +130,6 @@ void OpenGLWorker::FrameToValue(FramePtr frame, NodeValueTable *table) void OpenGLWorker::CloseInternal() { buffer_.Destroy(); - functions_ = nullptr; delete ctx_; } @@ -184,7 +247,10 @@ void OpenGLWorker::RunNodeAccelerated(Node *node, const NodeValueDatabase *input } } - //qDebug() << " Blitting with shader!"; + // Ensure viewport is correct + functions_->glViewport(0, 0, video_params().effective_width(), video_params().effective_height()); + + // Blit this texture through this shader olive::gl::Blit(shader); // Release any textures we bound before diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index a509ed018..e0dfe013f 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -14,6 +14,7 @@ public: OpenGLWorker(QOpenGLContext* share_ctx, OpenGLShaderCache* shader_cache, DecoderCache* decoder_cache, + ColorProcessorCache *color_cache, VideoRenderFrameCache* frame_cache, QObject* parent = nullptr); @@ -45,7 +46,7 @@ protected: virtual void CloseInternal() override; - virtual void FrameToValue(FramePtr frame, NodeValueTable* table) override; + virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override; virtual void RunNodeAccelerated(Node *node, const NodeValueDatabase *input_params, NodeValueTable* output_params) override; diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index c83c0ef71..43da6b949 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -98,17 +98,16 @@ StreamPtr RenderWorker::ResolveStreamFromInput(NodeInput *input) return input->get_value_at_time(0).value(); } -DecoderPtr RenderWorker::ResolveDecoderFromInput(NodeInput *input) +DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream) { // Access a map of Node inputs and decoder instances and retrieve a frame! - StreamPtr stream = ResolveStreamFromInput(input); - DecoderPtr decoder = decoder_cache()->GetDecoder(stream.get()); + DecoderPtr decoder = decoder_cache()->Get(stream.get()); if (decoder == nullptr && stream != nullptr) { // Create a new Decoder here DecoderPtr decoder = Decoder::CreateFromID(stream->footage()->decoder()); decoder->set_stream(stream); - decoder_cache()->AddDecoder(stream.get(), decoder); + decoder_cache()->Add(stream.get(), decoder); } return decoder; @@ -146,13 +145,17 @@ NodeValueTable RenderWorker::ProcessNodeNormally(const NodeDependency& dep) // Exception for Footage types where we actually retrieve some Footage data from a decoder if (input->data_type() == NodeParam::kFootage) { - DecoderPtr decoder = ResolveDecoderFromInput(input); + StreamPtr stream = ResolveStreamFromInput(input); - if (decoder) { - FramePtr frame = RetrieveFromDecoder(decoder, input_time); + if (stream) { + DecoderPtr decoder = ResolveDecoderFromInput(stream); - if (frame) { - FrameToValue(frame, &table); + if (decoder) { + FramePtr frame = RetrieveFromDecoder(decoder, input_time); + + if (frame) { + FrameToValue(stream, frame, &table); + } } } } diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index c023d38eb..1fcafe3c9 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -46,11 +46,11 @@ protected: virtual void RunNodeAccelerated(Node *node, const NodeValueDatabase *input_params, NodeValueTable* output_params); StreamPtr ResolveStreamFromInput(NodeInput* input); - DecoderPtr ResolveDecoderFromInput(NodeInput* input); + DecoderPtr ResolveDecoderFromInput(StreamPtr stream); virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) = 0; - virtual void FrameToValue(FramePtr frame, NodeValueTable* table) = 0; + virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) = 0; NodeValueTable ProcessNodeNormally(const NodeDependency &dep); diff --git a/app/render/backend/videorenderbackend.cpp b/app/render/backend/videorenderbackend.cpp index 2d803bcb9..5379140b5 100644 --- a/app/render/backend/videorenderbackend.cpp +++ b/app/render/backend/videorenderbackend.cpp @@ -175,7 +175,7 @@ void VideoRenderBackend::CacheIDChangedEvent(const QString &id) void VideoRenderBackend::ConnectWorkerToThis(RenderWorker *processor) { connect(processor, SIGNAL(CompletedFrame(NodeDependency, QByteArray, NodeValueTable)), this, SLOT(ThreadCompletedFrame(NodeDependency, QByteArray, NodeValueTable))); - connect(processor, SIGNAL(HashAlreadyBeingCached()), this, SLOT(ThreadSkippedFrame())); + connect(processor, SIGNAL(HashAlreadyBeingCached(NodeDependency, QByteArray)), this, SLOT(ThreadSkippedFrame(NodeDependency, QByteArray))); connect(processor, SIGNAL(CompletedDownload(NodeDependency, QByteArray)), this, SLOT(ThreadCompletedDownload(NodeDependency, QByteArray))); connect(processor, SIGNAL(HashAlreadyExists(NodeDependency, QByteArray)), this, SLOT(ThreadHashAlreadyExists(NodeDependency, QByteArray))); } @@ -185,6 +185,11 @@ VideoRenderFrameCache *VideoRenderBackend::frame_cache() return &frame_cache_; } +ColorProcessorCache *VideoRenderBackend::color_cache() +{ + return &color_cache_; +} + const char *VideoRenderBackend::GetCachedFrame(const rational &time) { last_time_requested_ = time; @@ -232,3 +237,9 @@ NodeInput *VideoRenderBackend::GetDependentInput() { return viewer_node()->texture_input(); } + +void VideoRenderBackend::CompletedFrame() +{ + caching_ = false; + CacheNext(); +} diff --git a/app/render/backend/videorenderbackend.h b/app/render/backend/videorenderbackend.h index c6875e1a9..261f736e7 100644 --- a/app/render/backend/videorenderbackend.h +++ b/app/render/backend/videorenderbackend.h @@ -23,6 +23,7 @@ #include +#include "colorprocessorcache.h" #include "node/output/viewer/viewer.h" #include "renderbackend.h" #include "render/pixelformat.h" @@ -81,6 +82,8 @@ protected: VideoRenderFrameCache* frame_cache(); + ColorProcessorCache* color_cache(); + const VideoRenderingParams& params() const; /** @@ -92,6 +95,8 @@ protected: virtual void ConnectWorkerToThis(RenderWorker* processor) override; + void CompletedFrame(); + signals: void CachedFrameReady(const rational& time, QVariant value); void CachedTimeReady(const rational& time); @@ -103,6 +108,8 @@ private: VideoRenderFrameCache frame_cache_; + ColorProcessorCache color_cache_; + rational last_time_requested_; private slots: diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index d5b6a399c..b6de16da0 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -4,9 +4,10 @@ #include "node/node.h" #include "render/pixelservice.h" -VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, VideoRenderFrameCache *frame_cache, QObject *parent) : +VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, ColorProcessorCache *color_cache, VideoRenderFrameCache *frame_cache, QObject *parent) : RenderWorker(decoder_cache, parent), - frame_cache_(frame_cache) + frame_cache_(frame_cache), + color_cache_(color_cache) { } @@ -36,7 +37,7 @@ NodeValueTable VideoRenderWorker::RenderInternal(const NodeDependency& path) emit CompletedFrame(path, hash, value); } else { // Another thread must be caching this already, nothing to be done - emit HashAlreadyBeingCached(); + emit HashAlreadyBeingCached(path, hash); } return value; @@ -85,7 +86,7 @@ void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, c // We have one exception for FOOTAGE types, since we resolve the footage into a frame in the renderer if (input->data_type() == NodeParam::kFootage) { StreamPtr stream = ResolveStreamFromInput(input); - DecoderPtr decoder = ResolveDecoderFromInput(input); + DecoderPtr decoder = ResolveDecoderFromInput(stream); if (decoder != nullptr) { // Add footage details to hash @@ -99,10 +100,19 @@ void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, c // Footage stream hash->addData(QString::number(stream->index()).toUtf8()); - // Footage timestamp - hash->addData(QString::number(decoder->GetTimestampFromTime(time)).toUtf8()); + if (stream->type() == Stream::kImage || stream->type() == Stream::kVideo) { + ImageStreamPtr video_stream = std::static_pointer_cast(stream); - // FIXME: Add colorspace and alpha assoc + // Footage timestamp + hash->addData(QString::number(decoder->GetTimestampFromTime(time)).toUtf8()); + + // Current colorspace + // FIXME: Handle empty colorspace... + hash->addData(video_stream->colorspace().toUtf8()); + + // Alpha associated setting + hash->addData(QString::number(video_stream->premultiplied_alpha()).toUtf8()); + } } } } @@ -171,3 +181,8 @@ NodeValueTable VideoRenderWorker::RenderBlock(TrackOutput *track, const TimeRang return table; } + +ColorProcessorCache *VideoRenderWorker::color_cache() const +{ + return color_cache_; +} diff --git a/app/render/backend/videorenderworker.h b/app/render/backend/videorenderworker.h index 7ed95ec5b..320abee66 100644 --- a/app/render/backend/videorenderworker.h +++ b/app/render/backend/videorenderworker.h @@ -3,6 +3,7 @@ #include +#include "colorprocessorcache.h" #include "node/dependency.h" #include "render/videoparams.h" #include "renderworker.h" @@ -11,7 +12,7 @@ class VideoRenderWorker : public RenderWorker { Q_OBJECT public: - VideoRenderWorker(DecoderCache* decoder_cache, VideoRenderFrameCache* frame_cache, QObject* parent = nullptr); + VideoRenderWorker(DecoderCache* decoder_cache, ColorProcessorCache* color_cache, VideoRenderFrameCache* frame_cache, QObject* parent = nullptr); void SetParameters(const VideoRenderingParams& video_params); @@ -23,7 +24,7 @@ signals: void CompletedDownload(NodeDependency path, QByteArray hash); - void HashAlreadyBeingCached(); + void HashAlreadyBeingCached(NodeDependency path, QByteArray hash); void HashAlreadyExists(NodeDependency path, QByteArray hash); @@ -44,6 +45,8 @@ protected: virtual NodeValueTable RenderBlock(TrackOutput *track, const TimeRange& range) override; + ColorProcessorCache* color_cache() const; + private: void ProcessNode(); @@ -53,6 +56,8 @@ private: VideoRenderFrameCache* frame_cache_; + ColorProcessorCache* color_cache_; + QByteArray download_buffer_; private slots: diff --git a/app/render/colorprocessor.h b/app/render/colorprocessor.h index e575e2574..001b449fb 100644 --- a/app/render/colorprocessor.h +++ b/app/render/colorprocessor.h @@ -4,8 +4,10 @@ #include namespace OCIO = OCIO_NAMESPACE::v1; +#include "common/constructors.h" #include "decoder/frame.h" + class ColorProcessor; using ColorProcessorPtr = std::shared_ptr; @@ -19,6 +21,8 @@ public: QString view, const QString& look); + DISABLE_COPY_MOVE(ColorProcessor) + static ColorProcessorPtr Create(const QString& source_space, const QString& dest_space); static ColorProcessorPtr Create(const QString& source_space,