From faffbb0f380fe445236da3041c1819499ea20f6b Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Tue, 25 Feb 2020 14:57:49 +1100 Subject: [PATCH] renderer: re-use already created textures if the images are known not to change --- app/codec/ffmpeg/ffmpegdecoder.cpp | 1 + app/render/backend/audio/audioworker.cpp | 17 ++- app/render/backend/audio/audioworker.h | 2 +- app/render/backend/audiorenderworker.cpp | 10 -- app/render/backend/audiorenderworker.h | 2 - app/render/backend/opengl/openglproxy.cpp | 145 ++++++++++++--------- app/render/backend/opengl/openglproxy.h | 10 +- app/render/backend/opengl/openglworker.cpp | 4 +- app/render/backend/opengl/openglworker.h | 4 +- app/render/backend/rendercache.h | 2 + app/render/backend/renderworker.cpp | 8 +- app/render/backend/renderworker.h | 4 +- app/render/backend/videorenderworker.cpp | 5 - app/render/backend/videorenderworker.h | 2 - 14 files changed, 118 insertions(+), 98 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index ad40ea404..48f0f9649 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -49,6 +49,7 @@ FFmpegDecoder::FFmpegDecoder() : cache_at_eof_(false), opts_(nullptr) { + // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints clear_timer_.setInterval(250); clear_timer_.moveToThread(qApp->thread()); connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent); diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index d7f445e5e..9740b74f4 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -5,11 +5,22 @@ AudioWorker::AudioWorker(DecoderCache* decoder_cache, QObject *parent) : { } -void AudioWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table) +void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) { - Q_UNUSED(stream) + if (stream->type() != Stream::kAudio) { + return; + } - table->Push(NodeParam::kSamples, frame->ToByteArray()); + if (decoder->HasConformedVersion(audio_params())) { + FramePtr frame = nullptr; + frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params()); + + if (frame) { + table->Push(NodeParam::kSamples, frame->ToByteArray()); + } + } else { + emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params()); + } } void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params_in, NodeValueTable *output_params) diff --git a/app/render/backend/audio/audioworker.h b/app/render/backend/audio/audioworker.h index 5bc4db40e..00c063b76 100644 --- a/app/render/backend/audio/audioworker.h +++ b/app/render/backend/audio/audioworker.h @@ -9,7 +9,7 @@ public: AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); protected: - virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override; + virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override; virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, const NodeValueDatabase& input_params, NodeValueTable* output_params) override; diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index e97cb23af..5e8a37954 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -23,16 +23,6 @@ void AudioRenderWorker::CloseInternal() // Nothing to init yet } -FramePtr AudioRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range) -{ - if (decoder->HasConformedVersion(audio_params_)) { - return decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params_); - } else { - emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params_); - return nullptr; - } -} - NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const TimeRange &range) { QList active_blocks = track->BlocksAtTimeRange(range); diff --git a/app/render/backend/audiorenderworker.h b/app/render/backend/audiorenderworker.h index 9354959b9..8bde8ace4 100644 --- a/app/render/backend/audiorenderworker.h +++ b/app/render/backend/audiorenderworker.h @@ -19,8 +19,6 @@ protected: virtual void CloseInternal() override; - virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) override; - virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) override; const AudioRenderingParams& audio_params() const; diff --git a/app/render/backend/opengl/openglproxy.cpp b/app/render/backend/opengl/openglproxy.cpp index 68aa6abf8..262a98ef8 100644 --- a/app/render/backend/opengl/openglproxy.cpp +++ b/app/render/backend/opengl/openglproxy.cpp @@ -43,7 +43,7 @@ bool OpenGLProxy::Init() return true; } -void OpenGLProxy::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table) +void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) { // Ensure stream is video or image type if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) { @@ -55,86 +55,107 @@ void OpenGLProxy::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable // Set up OCIO context QString colorspace_match = QStringLiteral("%1:%2").arg(video_stream->footage()->project()->ocio_config(), video_stream->colorspace()); - OpenGLColorProcessorPtr color_processor = std::static_pointer_cast(color_cache_.Get(colorspace_match)); + OpenGLTextureCache::ReferencePtr footage_tex_ref = nullptr; - if (!color_processor) { - color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager()->GetConfig(), - video_stream->colorspace(), - OCIO::ROLE_SCENE_LINEAR); - color_cache_.Add(colorspace_match, color_processor); - } + if (stream->type() == Stream::kImage && still_image_cache_.Has(stream.get())) { + CachedStill cs = still_image_cache_.Get(stream.get()); - ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode()); - - // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU - if (ocio_method == ColorManager::kOCIOAccurate) { - // If alpha is associated, disassociate for the color transform - if (video_stream->premultiplied_alpha()) { - ColorManager::DisassociateAlpha(frame); - } - - // Convert frame to float for OCIO - frame = PixelService::ConvertPixelFormat(frame, PixelFormat::PIX_FMT_RGBA32F); - - // Perform color transform - color_processor->ConvertFrame(frame); - - // Associate alpha - if (video_stream->premultiplied_alpha()) { - ColorManager::ReassociateAlpha(frame); + if (cs.colorspace == colorspace_match && cs.alpha_is_associated == video_stream->premultiplied_alpha()) { + footage_tex_ref = cs.texture; } else { - ColorManager::AssociateAlpha(frame); + still_image_cache_.Remove(stream.get()); } } - VideoRenderingParams footage_params(frame->width(), frame->height(), stream->timebase(), frame->format(), video_params_.mode()); + // Since this is a still image, we could likely optimize this + if (!footage_tex_ref) { + OpenGLColorProcessorPtr color_processor = std::static_pointer_cast(color_cache_.Get(colorspace_match)); - OpenGLTextureCache::ReferencePtr footage_tex_ref = texture_cache_.Get(ctx_, footage_params, frame->data()); - - if (ocio_method == ColorManager::kOCIOFast) { - if (!color_processor->IsEnabled()) { - color_processor->Enable(ctx_, video_stream->premultiplied_alpha()); + if (!color_processor) { + color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager()->GetConfig(), + video_stream->colorspace(), + OCIO::ROLE_SCENE_LINEAR); + color_cache_.Add(colorspace_match, color_processor); } - // Check frame aspect ratio - if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) { - int new_width = frame->width(); - int new_height = frame->height(); + ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode()); - // Scale the frame in a way that does not reduce the resolution - if (frame->sample_aspect_ratio() > 1) { - // Make wider - new_width = qRound(static_cast(new_width) * frame->sample_aspect_ratio().toDouble()); - } else { - // Make taller - new_height = qRound(static_cast(new_height) / frame->sample_aspect_ratio().toDouble()); + FramePtr frame = decoder->RetrieveVideo(range.in());; + + // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU + if (ocio_method == ColorManager::kOCIOAccurate) { + // If alpha is associated, disassociate for the color transform + if (video_stream->premultiplied_alpha()) { + ColorManager::DisassociateAlpha(frame); } - footage_params = VideoRenderingParams(new_width, - new_height, - footage_params.time_base(), - footage_params.format(), - footage_params.mode()); + // Convert frame to float for OCIO + frame = PixelService::ConvertPixelFormat(frame, PixelFormat::PIX_FMT_RGBA32F); + + // Perform color transform + color_processor->ConvertFrame(frame); + + // Associate alpha + if (video_stream->premultiplied_alpha()) { + ColorManager::ReassociateAlpha(frame); + } else { + ColorManager::AssociateAlpha(frame); + } } - // Create destination texture - OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, footage_params); + VideoRenderingParams footage_params(frame->width(), frame->height(), stream->timebase(), frame->format(), video_params_.mode()); - buffer_.Attach(associated_tex_ref->texture(), true); - buffer_.Bind(); - footage_tex_ref->texture()->Bind(); + footage_tex_ref = texture_cache_.Get(ctx_, footage_params, frame->data()); - // Set viewport for texture size - functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height()); + if (ocio_method == ColorManager::kOCIOFast) { + if (!color_processor->IsEnabled()) { + color_processor->Enable(ctx_, video_stream->premultiplied_alpha()); + } - // Blit old texture to new texture through OCIO shader - color_processor->ProcessOpenGL(); + // Check frame aspect ratio + if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) { + int new_width = frame->width(); + int new_height = frame->height(); - footage_tex_ref->texture()->Release(); - buffer_.Release(); - buffer_.Detach(); + // Scale the frame in a way that does not reduce the resolution + if (frame->sample_aspect_ratio() > 1) { + // Make wider + new_width = qRound(static_cast(new_width) * frame->sample_aspect_ratio().toDouble()); + } else { + // Make taller + new_height = qRound(static_cast(new_height) / frame->sample_aspect_ratio().toDouble()); + } - footage_tex_ref = associated_tex_ref; + footage_params = VideoRenderingParams(new_width, + new_height, + footage_params.time_base(), + footage_params.format(), + footage_params.mode()); + } + + // Create destination texture + OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, footage_params); + + buffer_.Attach(associated_tex_ref->texture(), true); + buffer_.Bind(); + footage_tex_ref->texture()->Bind(); + + // Set viewport for texture size + functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height()); + + // Blit old texture to new texture through OCIO shader + color_processor->ProcessOpenGL(); + + footage_tex_ref->texture()->Release(); + buffer_.Release(); + buffer_.Detach(); + + footage_tex_ref = associated_tex_ref; + } + + if (stream->type() == Stream::kImage) { + still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha()}); + } } table->Push(NodeParam::kTexture, QVariant::fromValue(footage_tex_ref)); diff --git a/app/render/backend/opengl/openglproxy.h b/app/render/backend/opengl/openglproxy.h index 0faf68b13..6e77a2a9d 100644 --- a/app/render/backend/opengl/openglproxy.h +++ b/app/render/backend/opengl/openglproxy.h @@ -41,7 +41,7 @@ public: void Close(); - void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table); + void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table); void RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params); @@ -65,6 +65,14 @@ private: OpenGLTextureCache texture_cache_; + struct CachedStill { + OpenGLTextureCache::ReferencePtr texture; + QString colorspace; + bool alpha_is_associated; + }; + + RenderCache still_image_cache_; + private slots: void FinishInit(); diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index e92005f4d..14580fea2 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -14,9 +14,9 @@ OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, DecoderCache* dec { } -void OpenGLWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable *table) +void OpenGLWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable *table) { - emit RequestFrameToValue(stream, frame, table); + emit RequestFrameToValue(decoder, stream, range, table); } void OpenGLWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable *output_params) diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index 1d3d1bf64..914f839a8 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -16,14 +16,14 @@ public: QObject* parent = nullptr); signals: - void RequestFrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table); + void RequestFrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table); void RequestRunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params); void RequestTextureToBuffer(const QVariant& texture, QByteArray& buffer); protected: - virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override; + virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override; virtual void RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params) override; diff --git a/app/render/backend/rendercache.h b/app/render/backend/rendercache.h index 7372e19be..06ae9ce42 100644 --- a/app/render/backend/rendercache.h +++ b/app/render/backend/rendercache.h @@ -18,6 +18,8 @@ public: bool Has(K key) const {return values_.contains(key);} + void Remove(K key) {values_.remove(key);} + private: QHash values_; diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index 9efb7ecc6..63aa3d308 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -135,6 +135,8 @@ const NodeDependency &RenderWorker::CurrentPath() const return path_; } + +#include "common/functiontimer.h" NodeValueDatabase RenderWorker::GenerateDatabase(const Node* node, const TimeRange &range) { NodeValueDatabase database; @@ -163,11 +165,7 @@ NodeValueDatabase RenderWorker::GenerateDatabase(const Node* node, const TimeRan Decoder::RetrieveState state = decoder->GetRetrieveState(input_time.out()); if (state == Decoder::kReady) { - FramePtr frame = RetrieveFromDecoder(decoder, input_time); - - if (frame) { - FrameToValue(stream, frame, &table); - } + FrameToValue(decoder, stream, input_time, &table); } else { ReportUnavailableFootage(stream, state, input_time.out()); } diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index 683182227..7965e297e 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -41,9 +41,7 @@ protected: StreamPtr ResolveStreamFromInput(NodeInput* input); DecoderPtr ResolveDecoderFromInput(StreamPtr stream); - virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) = 0; - - virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) = 0; + virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) = 0; NodeValueTable ProcessNode(const NodeDependency &dep); diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index 6fe79f7ee..b33b69e39 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -87,11 +87,6 @@ NodeValueTable VideoRenderWorker::RenderInternal(const NodeDependency& path, con return value; } -FramePtr VideoRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range) -{ - return decoder->RetrieveVideo(range.in()); -} - void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, const Node* n, const rational& time) { // Resolve BlockList diff --git a/app/render/backend/videorenderworker.h b/app/render/backend/videorenderworker.h index c3de8fd16..007a5fed2 100644 --- a/app/render/backend/videorenderworker.h +++ b/app/render/backend/videorenderworker.h @@ -71,8 +71,6 @@ protected: virtual NodeValueTable RenderInternal(const NodeDependency& CurrentPath, const qint64& job_time) override; - virtual FramePtr RetrieveFromDecoder(DecoderPtr decoder, const TimeRange& range) override; - virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) override; virtual void ReportUnavailableFootage(StreamPtr stream, Decoder::RetrieveState state, const rational& stream_time) override;