From 8795e51452b5eb464f9eb12c77a76a08841edc88 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Fri, 15 May 2020 00:52:45 +1000 Subject: [PATCH] renderer: massive overhaul to move to a vastly simplified threading system --- app/node/output/viewer/viewer.cpp | 18 +- app/node/output/viewer/viewer.h | 5 +- app/node/traverser.cpp | 37 +- app/node/traverser.h | 14 +- app/render/backend/CMakeLists.txt | 3 + app/render/backend/opengl/CMakeLists.txt | 2 + app/render/backend/opengl/openglbackend.cpp | 41 +- app/render/backend/opengl/openglbackend.h | 10 +- app/render/backend/opengl/openglproxy.cpp | 71 ++- app/render/backend/opengl/openglproxy.h | 18 +- app/render/backend/opengl/openglworker.cpp | 77 ++++ app/render/backend/opengl/openglworker.h | 48 ++ app/render/backend/renderbackend.cpp | 441 ++++-------------- app/render/backend/renderbackend.h | 75 +--- app/render/backend/renderworker.cpp | 467 ++++++++++++++++++++ app/render/backend/renderworker.h | 166 +++++++ app/widget/viewer/gizmotraverser.cpp | 2 +- app/widget/viewer/gizmotraverser.h | 2 +- app/widget/viewer/viewer.cpp | 47 +- app/widget/viewer/viewer.h | 8 +- 20 files changed, 1022 insertions(+), 530 deletions(-) create mode 100644 app/render/backend/opengl/openglworker.cpp create mode 100644 app/render/backend/opengl/openglworker.h create mode 100644 app/render/backend/renderworker.cpp create mode 100644 app/render/backend/renderworker.h diff --git a/app/node/output/viewer/viewer.cpp b/app/node/output/viewer/viewer.cpp index 289298ab8..cf9f67c7c 100644 --- a/app/node/output/viewer/viewer.cpp +++ b/app/node/output/viewer/viewer.cpp @@ -86,14 +86,14 @@ QString ViewerOutput::Description() const void ViewerOutput::InvalidateCache(const TimeRange &range, NodeInput *from, NodeInput *source) { - if (from == texture_input()) { - emit GraphChangedFrom(from, source); + if (from == texture_input_ || from == samples_input_) { + emit GraphChangedFrom(source); - video_frame_cache_.Invalidate(range); - } else if (from == samples_input()) { - emit GraphChangedFrom(from, source); - - audio_playback_cache_.Invalidate(range); + if (from == texture_input_) { + video_frame_cache_.Invalidate(range); + } else { + audio_playback_cache_.Invalidate(range); + } } Node::InvalidateCache(range, from, source); @@ -114,14 +114,14 @@ void ViewerOutput::set_video_params(const VideoParams &video) emit SizeChanged(video_params_.width(), video_params_.height()); emit TimebaseChanged(video_params_.time_base()); - emit VideoParamsChanged(); + emit ParamsChanged(); } void ViewerOutput::set_audio_params(const AudioParams &audio) { audio_params_ = audio; - emit AudioParamsChanged(); + emit ParamsChanged(); } rational ViewerOutput::GetLength() diff --git a/app/node/output/viewer/viewer.h b/app/node/output/viewer/viewer.h index 2c51e0181..99aba5dac 100644 --- a/app/node/output/viewer/viewer.h +++ b/app/node/output/viewer/viewer.h @@ -113,7 +113,7 @@ public: signals: void TimebaseChanged(const rational&); - void GraphChangedFrom(NodeInput* from, NodeInput* source); + void GraphChangedFrom(NodeInput* source); void VisibleInvalidated(NodeInput* source); @@ -121,8 +121,7 @@ signals: void SizeChanged(int width, int height); - void VideoParamsChanged(); - void AudioParamsChanged(); + void ParamsChanged(); void BlockAdded(Block* block, TrackReference track); void BlockRemoved(Block* block); diff --git a/app/node/traverser.cpp b/app/node/traverser.cpp index d99647a2b..ae10271cf 100644 --- a/app/node/traverser.cpp +++ b/app/node/traverser.cpp @@ -24,7 +24,7 @@ OLIVE_NAMESPACE_ENTER -NodeValueDatabase NodeTraverser::GenerateDatabase(const Node* node, const TimeRange &range) const +NodeValueDatabase NodeTraverser::GenerateDatabase(const Node* node, const TimeRange &range) { NodeValueDatabase database; @@ -38,17 +38,7 @@ NodeValueDatabase NodeTraverser::GenerateDatabase(const Node* node, const TimeRa TimeRange input_time = node->InputTimeAdjustment(input, range); - NodeValueTable table; - - if (input->IsConnected()) { - // Value will equal something from the connected node, follow it - table = GenerateTable(input->get_connected_node(), range); - } else { - // Push onto the table the value at this time from the input - QVariant input_value = input->get_value_at_time(range.in()); - - table.Push(input->data_type(), input_value); - } + NodeValueTable table = ProcessInput(input, input_time); // Exception for Footage types where we actually retrieve some Footage data from a decoder if (input->data_type() == NodeParam::kFootage) { @@ -73,7 +63,24 @@ NodeValueDatabase NodeTraverser::GenerateDatabase(const Node* node, const TimeRa return database; } -NodeValueTable NodeTraverser::GenerateTable(const Node *n, const TimeRange& range) const +NodeValueTable NodeTraverser::ProcessInput(NodeInput* input, const TimeRange& range) +{ + if (input->IsConnected()) { + // Value will equal something from the connected node, follow it + return GenerateTable(input->get_connected_node(), range); + } else if (!input->IsArray()) { + // Push onto the table the value at this time from the input + QVariant input_value = input->get_value_at_time(range.in()); + + NodeValueTable table; + table.Push(input->data_type(), input_value); + return table; + } + + return NodeValueTable(); +} + +NodeValueTable NodeTraverser::GenerateTable(const Node *n, const TimeRange& range) { if (n->IsTrack()) { // If the range is not wholly contained in this Block, we'll need to do some extra processing @@ -93,12 +100,12 @@ NodeValueTable NodeTraverser::GenerateTable(const Node *n, const TimeRange& rang return table; } -NodeValueTable NodeTraverser::GenerateTable(const Node *n, const rational &in, const rational &out) const +NodeValueTable NodeTraverser::GenerateTable(const Node *n, const rational &in, const rational &out) { return GenerateTable(n, TimeRange(in, out)); } -NodeValueTable NodeTraverser::GenerateBlockTable(const TrackOutput *track, const TimeRange &range) const +NodeValueTable NodeTraverser::GenerateBlockTable(const TrackOutput *track, const TimeRange &range) { // By default, just follow the in point Block* active_block = track->BlockAtTime(range.in()); diff --git a/app/node/traverser.h b/app/node/traverser.h index 2706e193d..9b0512e3c 100644 --- a/app/node/traverser.h +++ b/app/node/traverser.h @@ -34,20 +34,22 @@ class NodeTraverser : public CancelableObject public: NodeTraverser() = default; - NodeValueTable GenerateTable(const Node *n, const TimeRange &range) const; - NodeValueTable GenerateTable(const Node *n, const rational &in, const rational& out) const; + NodeValueTable GenerateTable(const Node *n, const TimeRange &range); + NodeValueTable GenerateTable(const Node *n, const rational &in, const rational& out); - NodeValueDatabase GenerateDatabase(const Node *node, const TimeRange &range) const; + NodeValueDatabase GenerateDatabase(const Node *node, const TimeRange &range); protected: - virtual NodeValueTable GenerateBlockTable(const TrackOutput *track, const TimeRange& range) const; + NodeValueTable ProcessInput(NodeInput *input, const TimeRange &range); - virtual void FootageProcessingEvent(StreamPtr, const TimeRange&, NodeValueTable*) const {} + virtual NodeValueTable GenerateBlockTable(const TrackOutput *track, const TimeRange& range); + + virtual void FootageProcessingEvent(StreamPtr, const TimeRange&, NodeValueTable*) {} virtual void ProcessNodeEvent(const Node*, const TimeRange&, NodeValueDatabase&, - NodeValueTable&) const {} + NodeValueTable&) {} private: static StreamPtr ResolveStreamFromInput(NodeInput* input); diff --git a/app/render/backend/CMakeLists.txt b/app/render/backend/CMakeLists.txt index adc39f19c..bff0940f1 100644 --- a/app/render/backend/CMakeLists.txt +++ b/app/render/backend/CMakeLists.txt @@ -27,6 +27,9 @@ set(OLIVE_SOURCES render/backend/renderbackend.h render/backend/renderbackend.cpp + render/backend/renderworker.h + render/backend/renderworker.cpp + render/backend/rendercache.h render/backend/colorprocessorcache.h render/backend/decodercache.h diff --git a/app/render/backend/opengl/CMakeLists.txt b/app/render/backend/opengl/CMakeLists.txt index 75af58e48..e2972b85f 100644 --- a/app/render/backend/opengl/CMakeLists.txt +++ b/app/render/backend/opengl/CMakeLists.txt @@ -33,5 +33,7 @@ set(OLIVE_SOURCES render/backend/opengl/opengltexture.cpp render/backend/opengl/opengltexturecache.h render/backend/opengl/opengltexturecache.cpp + render/backend/opengl/openglworker.h + render/backend/opengl/openglworker.cpp PARENT_SCOPE ) diff --git a/app/render/backend/opengl/openglbackend.cpp b/app/render/backend/opengl/openglbackend.cpp index 7370537c5..cc67d33da 100644 --- a/app/render/backend/opengl/openglbackend.cpp +++ b/app/render/backend/opengl/openglbackend.cpp @@ -20,12 +20,51 @@ #include "openglbackend.h" +#include "openglworker.h" + OLIVE_NAMESPACE_ENTER OpenGLBackend::OpenGLBackend(QObject* parent) : - RenderBackend(parent) + RenderBackend(parent), + proxy_(nullptr) { } +OpenGLBackend::~OpenGLBackend() +{ + Close(); + + ClearProxy(); +} + +RenderWorker *OpenGLBackend::CreateNewWorker() +{ + if (!proxy_) { + proxy_ = new OpenGLProxy(); + + QThread* proxy_thread = new QThread(); + proxy_thread->start(QThread::IdlePriority); + proxy_->moveToThread(proxy_thread); + + if (!proxy_->Init()) { + ClearProxy(); + return nullptr; + } + } + + return new OpenGLWorker(proxy_); +} + +void OpenGLBackend::ClearProxy() +{ + if (proxy_) { + proxy_->thread()->quit(); + proxy_->thread()->wait(); + proxy_->thread()->deleteLater(); + proxy_->deleteLater(); + proxy_ = nullptr; + } +} + OLIVE_NAMESPACE_EXIT diff --git a/app/render/backend/opengl/openglbackend.h b/app/render/backend/opengl/openglbackend.h index 5f9b8daf4..a1df6dcab 100644 --- a/app/render/backend/opengl/openglbackend.h +++ b/app/render/backend/opengl/openglbackend.h @@ -21,6 +21,7 @@ #ifndef OPENGLBACKEND_H #define OPENGLBACKEND_H +#include "openglproxy.h" #include "render/backend/renderbackend.h" OLIVE_NAMESPACE_ENTER @@ -30,12 +31,15 @@ class OpenGLBackend : public RenderBackend public: OpenGLBackend(QObject* parent = nullptr); + virtual ~OpenGLBackend() override; + protected: - virtual void TextureToFrame(const QVariant& texture, FramePtr frame) const override; + virtual RenderWorker* CreateNewWorker() override; - virtual NodeValue FrameToTexture(FramePtr frame) const override; +private: + void ClearProxy(); - virtual void ProcessNodeEvent(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params) const override; + OpenGLProxy* proxy_; }; diff --git a/app/render/backend/opengl/openglproxy.cpp b/app/render/backend/opengl/openglproxy.cpp index 576fe81c0..3e125d0a1 100644 --- a/app/render/backend/opengl/openglproxy.cpp +++ b/app/render/backend/opengl/openglproxy.cpp @@ -67,7 +67,7 @@ bool OpenGLProxy::Init() return true; } -NodeValue OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream) +NodeValue OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream, const VideoRenderingParams& params) { ImageStreamPtr video_stream = std::static_pointer_cast(stream); @@ -83,7 +83,7 @@ NodeValue OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream) color_cache_.Add(colorspace_match, color_processor); } - ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode()); + ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(params.mode()); // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU if (ocio_method == ColorManager::kOCIOAccurate) { @@ -144,7 +144,7 @@ NodeValue OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream) VideoRenderingParams dest_params(frame_params.width(), frame_params.height(), - video_params_.format(), + params.format(), frame_params.divider()); // Create destination texture @@ -180,12 +180,12 @@ void OpenGLProxy::Close() ctx_ = nullptr; } -void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params) +void OpenGLProxy::RunNodeAccelerated(const Node *node, + const TimeRange &range, + NodeValueDatabase &input_params, + NodeValueTable &output_params, + const VideoRenderingParams& params) { - if (!(node->GetCapabilities(input_params) & Node::kShader)) { - return; - } - OpenGLShaderPtr shader = shader_cache_.Get(node->ShaderID(input_params)); if (!shader) { @@ -213,12 +213,12 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, N // Create the output textures QList dst_refs; - dst_refs.append(texture_cache_.Get(ctx_, video_params_)); + dst_refs.append(texture_cache_.Get(ctx_, params)); GLuint iterative_input = 0; // If this node requires multiple iterations, get a texture for it too if (node->ShaderIterations() > 1 && node->ShaderIterativeInput()) { - dst_refs.append(texture_cache_.Get(ctx_, video_params_)); + dst_refs.append(texture_cache_.Get(ctx_, params)); } // Lock the shader so no other thread interferes as we set parameters and draw (and we don't interfere with any others) @@ -359,12 +359,12 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, N } // Set up OpenGL parameters as necessary - functions_->glViewport(0, 0, video_params_.effective_width(), video_params_.effective_height()); + functions_->glViewport(0, 0, params.effective_width(), params.effective_height()); // Provide some standard args shader->setUniformValue("ove_resolution", - static_cast(video_params_.width()), - static_cast(video_params_.height())); + static_cast(params.width()), + static_cast(params.height())); if (node->IsBlock() && static_cast(node)->type() == Block::kTransition) { const TransitionBlock* transition_node = static_cast(node); @@ -424,7 +424,9 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, N output_params.Push(NodeParam::kTexture, QVariant::fromValue(output_tex)); } -void OpenGLProxy::TextureToBuffer(const QVariant &tex_in, int width, int height, const QMatrix4x4 &matrix, void *buffer, int linesize) +void OpenGLProxy::TextureToBuffer(const QVariant& tex_in, + FramePtr frame, + const QMatrix4x4& matrix) { OpenGLTextureCache::ReferencePtr texture = tex_in.value(); @@ -432,23 +434,21 @@ void OpenGLProxy::TextureToBuffer(const QVariant &tex_in, int width, int height, return; } - QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions(); - OpenGLTextureCache::ReferencePtr download_tex; - if (width != texture->texture()->width() || height != texture->texture()->height()) { + functions_->glViewport(0, 0, frame->width(), frame->height()); + + if (frame->width() != texture->texture()->width() + || frame->height() != texture->texture()->height()) { // Resize the texture if necessary - OpenGLTextureCache::ReferencePtr resized = texture_cache_.Get(ctx_, - VideoRenderingParams(width, height, texture->texture()->format())); + OpenGLTextureCache::ReferencePtr resized = texture_cache_.Get(ctx_, frame->video_params()); buffer_.Attach(resized->texture(), true); buffer_.Bind(); texture->texture()->Bind(); - f->glViewport(0, 0, width, height); - // Blit to this new texture OpenGLRenderFunctions::Blit(copy_pipeline_, false, matrix); @@ -468,31 +468,22 @@ void OpenGLProxy::TextureToBuffer(const QVariant &tex_in, int width, int height, buffer_.Attach(download_tex->texture()); buffer_.Bind(); - f->glPixelStorei(GL_PACK_ROW_LENGTH, linesize); + functions_->glPixelStorei(GL_PACK_ROW_LENGTH, frame->linesize_pixels()); - f->glReadPixels(0, - 0, - width, - height, - OpenGLRenderFunctions::GetPixelFormat(video_params_.format()), - OpenGLRenderFunctions::GetPixelType(video_params_.format()), - buffer); + functions_->glReadPixels(0, + 0, + frame->width(), + frame->height(), + OpenGLRenderFunctions::GetPixelFormat(texture->texture()->format()), + OpenGLRenderFunctions::GetPixelType(texture->texture()->format()), + frame->data()); - f->glPixelStorei(GL_PACK_ROW_LENGTH, 0); + functions_->glPixelStorei(GL_PACK_ROW_LENGTH, 0); buffer_.Release(); buffer_.Detach(); } -void OpenGLProxy::SetParameters(const VideoRenderingParams ¶ms) -{ - video_params_ = params; - - if (functions_ != nullptr && video_params_.is_valid()) { - functions_->glViewport(0, 0, video_params_.effective_width(), video_params_.effective_height()); - } -} - void OpenGLProxy::FinishInit() { // Make context current on that surface @@ -505,8 +496,6 @@ void OpenGLProxy::FinishInit() functions_ = ctx_->functions(); functions_->glBlendFunc(GL_ONE, GL_ZERO); - SetParameters(video_params_); - buffer_.Create(ctx_); copy_pipeline_ = OpenGLShader::CreateDefault(); diff --git a/app/render/backend/opengl/openglproxy.h b/app/render/backend/opengl/openglproxy.h index 9fc5072b5..232d94b1c 100644 --- a/app/render/backend/opengl/openglproxy.h +++ b/app/render/backend/opengl/openglproxy.h @@ -66,14 +66,20 @@ public: void Close(); - void SetParameters(const VideoRenderingParams& params); - public slots: - void RunNodeAccelerated(const OLIVE_NAMESPACE::Node *node, const OLIVE_NAMESPACE::TimeRange &range, OLIVE_NAMESPACE::NodeValueDatabase &input_params, OLIVE_NAMESPACE::NodeValueTable& output_params); + void RunNodeAccelerated(const OLIVE_NAMESPACE::Node *node, + const OLIVE_NAMESPACE::TimeRange &range, + OLIVE_NAMESPACE::NodeValueDatabase &input_params, + OLIVE_NAMESPACE::NodeValueTable& output_params, + const OLIVE_NAMESPACE::VideoRenderingParams ¶ms); - void TextureToBuffer(const QVariant& texture, int width, int height, const QMatrix4x4& matrix, void *buffer, int linesize); + void TextureToBuffer(const QVariant& texture, + OLIVE_NAMESPACE::FramePtr frame, + const QMatrix4x4& matrix); - OLIVE_NAMESPACE::NodeValue FrameToValue(OLIVE_NAMESPACE::FramePtr frame, OLIVE_NAMESPACE::StreamPtr stream); + OLIVE_NAMESPACE::NodeValue FrameToValue(OLIVE_NAMESPACE::FramePtr frame, + OLIVE_NAMESPACE::StreamPtr stream, + const OLIVE_NAMESPACE::VideoRenderingParams ¶ms); private: QOpenGLContext* ctx_; @@ -85,8 +91,6 @@ private: OpenGLColorProcessorCache color_cache_; - VideoRenderingParams video_params_; - OpenGLShaderPtr copy_pipeline_; OpenGLShaderCache shader_cache_; diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp new file mode 100644 index 000000000..84e1eaac3 --- /dev/null +++ b/app/render/backend/opengl/openglworker.cpp @@ -0,0 +1,77 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "openglworker.h" + +OLIVE_NAMESPACE_ENTER + +OpenGLWorker::OpenGLWorker(OpenGLProxy* proxy) : + proxy_(proxy) +{ +} + +void OpenGLWorker::TextureToFrame(const QVariant &texture, FramePtr frame, const QMatrix4x4& mat) const +{ + QMetaObject::invokeMethod(proxy_, + "TextureToBuffer", + Qt::BlockingQueuedConnection, + Q_ARG(const QVariant&, texture), + OLIVE_NS_ARG(FramePtr, frame), + Q_ARG(const QMatrix4x4&, mat)); +} + +NodeValue OpenGLWorker::FrameToTexture(DecoderPtr decoder, StreamPtr stream, const TimeRange &range) const +{ + FramePtr frame = decoder->RetrieveVideo(range.in(), + video_params().divider(), + video_params().mode() == RenderMode::kOffline); + + NodeValue value; + + if (frame) { + QMetaObject::invokeMethod(proxy_, + "FrameToValue", + Qt::BlockingQueuedConnection, + OLIVE_NS_RETURN_ARG(NodeValue, value), + OLIVE_NS_ARG(FramePtr, frame), + OLIVE_NS_ARG(StreamPtr, stream), + OLIVE_NS_CONST_ARG(VideoRenderingParams&, video_params())); + } + + return value; +} + +void OpenGLWorker::ProcessNodeEvent(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params) +{ + RenderWorker::ProcessNodeEvent(node, range, input_params, output_params); + + if (node->GetCapabilities(input_params) & Node::kShader) { + QMetaObject::invokeMethod(proxy_, + "RunNodeAccelerated", + Qt::BlockingQueuedConnection, + OLIVE_NS_CONST_ARG(Node*, node), + OLIVE_NS_CONST_ARG(TimeRange&, range), + OLIVE_NS_ARG(NodeValueDatabase&, input_params), + OLIVE_NS_ARG(NodeValueTable&, output_params), + OLIVE_NS_CONST_ARG(VideoRenderingParams&, video_params())); + } +} + +OLIVE_NAMESPACE_EXIT diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h new file mode 100644 index 000000000..dcf08fae7 --- /dev/null +++ b/app/render/backend/opengl/openglworker.h @@ -0,0 +1,48 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef OPENGLWORKER_H +#define OPENGLWORKER_H + +#include "openglproxy.h" +#include "render/backend/renderworker.h" + +OLIVE_NAMESPACE_ENTER + +class OpenGLWorker : public RenderWorker +{ +public: + OpenGLWorker(OpenGLProxy* proxy); + +protected: + virtual void TextureToFrame(const QVariant& texture, FramePtr frame, const QMatrix4x4 &mat) const override; + + virtual NodeValue FrameToTexture(DecoderPtr decoder, StreamPtr stream, const TimeRange &range) const override; + + virtual void ProcessNodeEvent(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params) override; + +private: + OpenGLProxy* proxy_; + +}; + +OLIVE_NAMESPACE_EXIT + +#endif // OPENGLWORKER_H diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index d4faf3801..77f2a1523 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -41,6 +41,11 @@ RenderBackend::RenderBackend(QObject *parent) : cancel_dialog_ = new RenderCancelDialog(Core::instance()->main_window()); } +RenderBackend::~RenderBackend() +{ + Close(); +} + void RenderBackend::SetViewerNode(ViewerOutput *viewer_node) { if (viewer_node_ == viewer_node) { @@ -52,8 +57,9 @@ void RenderBackend::SetViewerNode(ViewerOutput *viewer_node) CancelQueue(); // Delete all of our copied nodes - video_copy_map_.Clear(); - audio_copy_map_.Clear(); + foreach (RenderWorker* instance, instance_pool_) { + instance->Close(); + } disconnect(viewer_node_, &ViewerOutput::GraphChangedFrom, @@ -70,15 +76,10 @@ void RenderBackend::SetViewerNode(ViewerOutput *viewer_node) viewer_node_ = viewer_node; if (viewer_node_) { - // Start copying viewer - video_copy_map_.Init(viewer_node_); - audio_copy_map_.Init(viewer_node_); - - video_copy_map_.Queue(viewer_node_->texture_input()); - audio_copy_map_.Queue(viewer_node_->samples_input()); - - video_copy_map_.ProcessQueue(); - audio_copy_map_.ProcessQueue(); + // Initiate instances with new node + foreach (RenderWorker* instance, instance_pool_) { + instance->Init(viewer_node_); + } connect(viewer_node_, &ViewerOutput::GraphChangedFrom, @@ -95,49 +96,30 @@ void RenderBackend::SetViewerNode(ViewerOutput *viewer_node) void RenderBackend::CancelQueue() { // FIXME: Implement something better than this... - video_copy_map_.thread_pool()->waitForDone(); - audio_copy_map_.thread_pool()->waitForDone(); -} - -QByteArray HashInternal(Node *node, - const VideoRenderingParams ¶ms, - const rational &time) -{ - QCryptographicHash hasher(QCryptographicHash::Sha1); - - // Embed video parameters into this hash - hasher.addData(reinterpret_cast(¶ms.effective_width()), sizeof(int)); - hasher.addData(reinterpret_cast(¶ms.effective_height()), sizeof(int)); - hasher.addData(reinterpret_cast(¶ms.format()), sizeof(PixelFormat::Format)); - hasher.addData(reinterpret_cast(¶ms.mode()), sizeof(RenderMode::Mode)); - - node->Hash(hasher, time); - - return hasher.result(); + thread_pool_.waitForDone(); } QFuture RenderBackend::Hash(const rational &time) { - if (!viewer_node_) { - return QFuture(); - } + RenderWorker* instance = GetInstanceFromPool(); - return QtConcurrent::run(video_copy_map_.thread_pool(), - HashInternal, - viewer_node_, - video_params(), + return QtConcurrent::run(&thread_pool_, + instance, + &RenderWorker::Hash, time); } -QFuture RenderBackend::RenderFrame(const rational &time) +QFuture RenderBackend::RenderFrame(const rational &time, bool clear_queue) { - if (!viewer_node_) { - return QFuture(); + if (clear_queue) { + thread_pool_.clear(); } - return QtConcurrent::run(video_copy_map_.thread_pool(), - this, - &RenderBackend::RenderFrameInternal, + RenderWorker* instance = GetInstanceFromPool(); + + return QtConcurrent::run(&thread_pool_, + instance, + &RenderWorker::RenderFrame, time); } @@ -161,207 +143,67 @@ void RenderBackend::SetSampleFormat(const SampleFormat::Format &sample_fmt) sample_fmt_ = sample_fmt; } -void RenderBackend::NodeGraphChanged(NodeInput *from, NodeInput *source) +void RenderBackend::SetVideoDownloadMatrix(const QMatrix4x4 &mat) { - if (from == viewer_node_->texture_input()) { - video_copy_map_.Queue(source); - } else if (from == viewer_node_->samples_input()) { - audio_copy_map_.Queue(source); + video_download_matrix_ = mat; +} + +void RenderBackend::NodeGraphChanged(NodeInput *source) +{ + QLinkedList::iterator i; + + for (i=instance_pool_.begin(); i!=instance_pool_.end(); i++) { + (*i)->Queue(source); } } -void RenderBackend::FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable *table) const +void RenderBackend::Close() { - if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) { + CancelQueue(); - ImageStreamPtr video_stream = std::static_pointer_cast(stream); - rational time_match = (stream->type() == Stream::kImage) ? rational() : input_time.in(); - QString colorspace_match = video_stream->get_colorspace_match_string(); - - NodeValue value; - bool found_cache = false; - - if (still_image_cache_.Has(stream.get())) { - CachedStill cs = still_image_cache_.Get(stream.get()); - - if (cs.colorspace == colorspace_match - && cs.alpha_is_associated == video_stream->premultiplied_alpha() - && cs.divider == video_params_.divider() - && cs.time == time_match) { - value = cs.texture; - found_cache = true; - } else { - still_image_cache_.Remove(stream.get()); - } - } - - if (!found_cache) { - - value = GetDataFromStream(stream, input_time); - - still_image_cache_.Add(stream.get(), {value, - colorspace_match, - video_stream->premultiplied_alpha(), - video_params_.divider(), - time_match}); - - } - - table->Push(value); - - } else if (stream->type() != Stream::kAudio) { - - table->Push(GetDataFromStream(stream, input_time)); - - } -} - -NodeValue RenderBackend::GetDataFromStream(StreamPtr stream, const TimeRange &input_time) const -{ - DecoderPtr decoder = ResolveDecoderFromInput(stream); - - if (decoder) { - return FrameToTexture(decoder, stream, input_time); - } - - return NodeValue(); -} - -NodeValueTable RenderBackend::GenerateBlockTable(const TrackOutput *track, const TimeRange &range) const -{ - if (track->track_type() == Timeline::kTrackTypeAudio) { - - QList active_blocks = track->BlocksAtTimeRange(range); - - // All these blocks will need to output to a buffer so we create one here - SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params_, - audio_params_.time_to_samples(range.length())); - block_range_buffer->fill(0); - - NodeValueTable merged_table; - - // Loop through active blocks retrieving their audio - foreach (Block* b, active_blocks) { - TimeRange range_for_block(qMax(b->in(), range.in()), - qMin(b->out(), range.out())); - - int destination_offset = audio_params_.time_to_samples(range_for_block.in() - range.in()); - int max_dest_sz = audio_params_.time_to_samples(range_for_block.length()); - - // Destination buffer - NodeValueTable table = ProcessNode(NodeDependency(b, range_for_block)); - QVariant sample_val = table.Take(NodeParam::kSamples); - SampleBufferPtr samples_from_this_block; - - if (sample_val.isNull() - || !(samples_from_this_block = sample_val.value())) { - // If we retrieved no samples from this block, do nothing - continue; - } - - // Stretch samples here - rational abs_speed = qAbs(b->speed()); - - if (abs_speed != 1) { - samples_from_this_block->speed(abs_speed.toDouble()); - } - - if (b->is_reversed()) { - // Reverse the audio buffer - samples_from_this_block->reverse(); - } - - int copy_length = qMin(max_dest_sz, samples_from_this_block->sample_count_per_channel()); - - // Copy samples into destination buffer - block_range_buffer->set(samples_from_this_block->const_data(), destination_offset, copy_length); - - { - // Save waveform to file - Block* src_block = static_cast(copy_map_->key(b)); - QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString()); - QDir waveform_loc = local_appdata_dir.filePath(QStringLiteral("waveform")); - waveform_loc.mkpath("."); - QString wave_fn(waveform_loc.filePath(QString::number(reinterpret_cast(src_block)))); - QFile wave_file(wave_fn); - - if (wave_file.open(QFile::ReadWrite)) { - // We use S32 as a size-compatible substitute for SampleSummer::Sum which is 4 bytes in size - AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S32); - int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()); - - { - // Write metadata header - SampleSummer::Info info; - info.channels = audio_params_.channel_count(); - wave_file.write(reinterpret_cast(&info), sizeof(SampleSummer::Info)); - } - - qint64 start_offset = sizeof(SampleSummer::Info) + waveform_params.time_to_bytes(range_for_block.in() - b->in()); - qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()); - qint64 end_offset = start_offset + length_offset; - - if (wave_file.size() < end_offset) { - wave_file.resize(end_offset); - } - - wave_file.seek(start_offset); - - for (int i=0;isample_count_per_channel();i+=chunk_size) { - QVector summary = SampleSummer::SumSamples(samples_from_this_block, - i, - qMin(chunk_size, samples_from_this_block->sample_count_per_channel() - i)); - - wave_file.write(reinterpret_cast(summary.constData()), - summary.size() * sizeof(SampleSummer::Sum)); - } - - wave_file.close(); - - if (src_block->type() == Block::kClip) { - emit static_cast(src_block)->PreviewUpdated(); - } - } - } - - NodeValueTable::Merge({merged_table, table}); - } - - merged_table.Push(NodeParam::kSamples, QVariant::fromValue(block_range_buffer)); - - return merged_table; - - } else { - return NodeTraverser::GenerateBlockTable(track, range); - } -} - -FramePtr RenderBackend::RenderFrameInternal(const rational &time) const -{ - NodeValueTable table = GenerateTable(viewer_node_, - TimeRange(time, time + viewer_node_->video_params().time_base())); - - QVariant texture = table.Get(NodeParam::kTexture); - - FramePtr frame = Frame::Create(); - frame->set_video_params(video_params()); - frame->allocate(); - - if (texture.isNull()) { - memset(frame->data(), 0, frame->allocated_size()); - } else { - TextureToFrame(texture, frame); - } - - return frame; + qDeleteAll(instance_pool_); + instance_pool_.clear(); } VideoRenderingParams RenderBackend::video_params() const { - return VideoRenderingParams(viewer_node_->video_params(), - pix_fmt_, - render_mode_, - divider_); + return VideoRenderingParams(viewer_node_->video_params(), pix_fmt_, render_mode_, divider_); +} + +AudioRenderingParams RenderBackend::audio_params() const +{ + return AudioRenderingParams(viewer_node_->audio_params(), sample_fmt_); +} + +RenderWorker *RenderBackend::GetInstanceFromPool() +{ + RenderWorker* instance = nullptr; + + QLinkedList::iterator i; + + for (i=instance_pool_.begin(); i!=instance_pool_.end(); i++) { + if ((*i)->IsAvailable()) { + instance = *i; + break; + } + } + + if (!instance) { + instance = CreateNewWorker(); + instance_pool_.append(instance); + + if (viewer_node_) { + instance->Init(viewer_node_); + } + } + + instance->SetAvailable(false); + instance->ProcessQueue(); + instance->SetVideoParams(video_params()); + instance->SetAudioParams(audio_params()); + instance->SetVideoDownloadMatrix(video_download_matrix_); + + return instance; } void RenderBackend::AudioCallback() @@ -390,135 +232,4 @@ bool RenderBackend::ConformWaitInfo::operator==(const RenderBackend::ConformWait && rhs.affected_range == affected_range; } -RenderBackend::CopyMap::CopyMap() : - original_viewer_(nullptr), - copied_viewer_(nullptr) -{ -} - -void RenderBackend::CopyMap::Init(ViewerOutput *viewer) -{ - original_viewer_ = viewer; - - copied_viewer_ = static_cast(original_viewer_->copy()); - copy_map_.insert(original_viewer_, copied_viewer_); -} - -void RenderBackend::CopyMap::Queue(NodeInput *input) -{ - if (!queued_updates_.isEmpty()) { - // Remove any inputs that are dependents of this input since they may have been removed since - // it was queued - QList deps = input->GetDependencies(); - - for (int i=0;iparentNode())) { - // We don't need to queue this value since this input supersedes it - queued_updates_.removeAt(i); - i--; - } - } - } - - queued_updates_.append(input); -} - -void RenderBackend::CopyMap::ProcessQueue() -{ - while (!queued_updates_.isEmpty()) { - CopyNodeInputValue(queued_updates_.takeFirst()); - } -} - -void RenderBackend::CopyMap::Clear() -{ - qDeleteAll(copy_map_); - copy_map_.clear(); - - original_viewer_ = nullptr; - copied_viewer_ = nullptr; -} - -void RenderBackend::CopyMap::CopyNodeInputValue(NodeInput *input) -{ - // Find our copy of this parameter - Node* our_copy_node = copy_map_.value(input->parentNode()); - NodeInput* our_copy = our_copy_node->GetInputWithID(input->id()); - - // Copy the standard/keyframe values between these two inputs - NodeInput::CopyValues(input, - our_copy, - false); - - // Handle connections - if (input->IsConnected() || our_copy->IsConnected()) { - // If one of the inputs is connected, it's likely this change came from connecting or - // disconnecting whatever was connected to it - - { - // We start by removing all old dependencies from the map - QList old_deps = our_copy->GetExclusiveDependencies(); - - foreach (Node* i, old_deps) { - Node* n = copy_map_.take(copy_map_.key(i)); - delete n; - } - - // And clear any other edges - while (!our_copy->edges().isEmpty()) { - NodeParam::DisconnectEdge(our_copy->edges().first()); - } - } - - // Then we copy all node dependencies and connections (if there are any) - CopyNodeMakeConnection(input, our_copy); - } - - // Call on sub-elements too - if (input->IsArray()) { - foreach (NodeInput* i, static_cast(input)->sub_params()) { - CopyNodeInputValue(i); - } - } -} - -Node* RenderBackend::CopyMap::CopyNodeConnections(Node* src_node) -{ - // Check if this node is already in the map - Node* dst_node = copy_map_.value(src_node); - - // If not, create it now - if (!dst_node) { - dst_node = src_node->copy(); - copy_map_.insert(src_node, dst_node); - } - - // Make sure its values are copied - Node::CopyInputs(src_node, dst_node, false); - - // Copy all connections - QList src_node_inputs = src_node->GetInputsIncludingArrays(); - QList dst_node_inputs = dst_node->GetInputsIncludingArrays(); - - for (int i=0;iIsConnected()) { - Node* dst_node = CopyNodeConnections(src_input->get_connected_node()); - - NodeOutput* corresponding_output = dst_node->GetOutputWithID(src_input->get_connected_output()->id()); - - NodeParam::ConnectEdge(corresponding_output, - dst_input); - } -} - OLIVE_NAMESPACE_EXIT diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index cd226a0a6..e2bf33ef5 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -27,17 +27,19 @@ #include "decodercache.h" #include "node/graph.h" #include "node/output/viewer/viewer.h" -#include "node/traverser.h" #include "render/backend/colorprocessorcache.h" +#include "renderworker.h" OLIVE_NAMESPACE_ENTER -class RenderBackend : public QObject, public NodeTraverser +class RenderBackend : public QObject { Q_OBJECT public: RenderBackend(QObject* parent = nullptr); + virtual ~RenderBackend() override; + void SetViewerNode(ViewerOutput* viewer_node); void CancelQueue(); @@ -50,7 +52,7 @@ public: /** * @brief Asynchronously generate a frame at a given time */ - QFuture RenderFrame(const rational& time); + QFuture RenderFrame(const rational& time, bool clear_queue); void SetDivider(const int& divider); @@ -60,83 +62,38 @@ public: void SetSampleFormat(const SampleFormat::Format& sample_fmt); + void SetVideoDownloadMatrix(const QMatrix4x4& mat); + public slots: - void NodeGraphChanged(NodeInput *from, NodeInput *source); + void NodeGraphChanged(NodeInput *source); protected: - virtual void TextureToFrame(const QVariant& texture, FramePtr frame) const = 0; + virtual RenderWorker* CreateNewWorker() = 0; - virtual NodeValue FrameToTexture(FramePtr frame) const = 0; + void Close(); - virtual void FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable* table) const override; + VideoRenderingParams video_params() const; - virtual NodeValueTable GenerateBlockTable(const TrackOutput *track, const TimeRange &range) const override; + AudioRenderingParams audio_params() const; private: - FramePtr RenderFrameInternal(const rational& time) const; + RenderWorker *GetInstanceFromPool(); - /** - * @brief Internal reference to attached viewer node - */ ViewerOutput* viewer_node_; RenderCancelDialog* cancel_dialog_; - VideoRenderingParams video_params() const; + QLinkedList instance_pool_; - NodeValue GetDataFromStream(StreamPtr stream, const TimeRange& input_time) const; - - DecoderPtr ResolveDecoderFromInput(StreamPtr stream) const; - - class CopyMap { - public: - CopyMap(); - - void Init(ViewerOutput* viewer); - - void Queue(NodeInput* input); - - void ProcessQueue(); - - void Clear(); - - QThreadPool* thread_pool() { - return &thread_pool_; - } - - private: - void CopyNodeInputValue(NodeInput* input); - Node *CopyNodeConnections(Node *src_node); - void CopyNodeMakeConnection(NodeInput *src_input, NodeInput *dst_input); - - ViewerOutput* original_viewer_; - ViewerOutput* copied_viewer_; - QList queued_updates_; - QHash copy_map_; - QThreadPool thread_pool_; - - }; + QThreadPool thread_pool_; // VIDEO MEMBERS - CopyMap video_copy_map_; int divider_; RenderMode::Mode render_mode_; PixelFormat::Format pix_fmt_; - - ColorProcessorCache color_cache_; - - struct CachedStill { - NodeValue texture; - QString colorspace; - bool alpha_is_associated; - int divider; - rational time; - }; - - RenderCache still_image_cache_; + QMatrix4x4 video_download_matrix_; // AUDIO MEMBERS - CopyMap audio_copy_map_; SampleFormat::Format sample_fmt_; struct ConformWaitInfo { diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp new file mode 100644 index 000000000..5e71c28f8 --- /dev/null +++ b/app/render/backend/renderworker.cpp @@ -0,0 +1,467 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "renderworker.h" + +#include + +#include "audio/sumsamples.h" +#include "config/config.h" +#include "node/block/clip/clip.h" + +OLIVE_NAMESPACE_ENTER + +RenderWorker::RenderWorker() : + viewer_(nullptr), + available_(true) +{ +} + +RenderWorker::~RenderWorker() +{ + Close(); +} + +QByteArray RenderWorker::Hash(const rational &time) const +{ + if (!viewer_) { + return QByteArray(); + } + + QCryptographicHash hasher(QCryptographicHash::Sha1); + + // Embed video parameters into this hash + hasher.addData(reinterpret_cast(&video_params_.effective_width()), sizeof(int)); + hasher.addData(reinterpret_cast(&video_params_.effective_height()), sizeof(int)); + hasher.addData(reinterpret_cast(&video_params_.format()), sizeof(PixelFormat::Format)); + hasher.addData(reinterpret_cast(&video_params_.mode()), sizeof(RenderMode::Mode)); + + viewer_->Hash(hasher, time); + + return hasher.result(); +} + +FramePtr RenderWorker::RenderFrame(const rational &time) +{ + if (!viewer_) { + return nullptr; + } + + NodeValueTable table = GenerateTable(viewer_, + TimeRange(time, time + video_params_.time_base())); + + QVariant texture = table.Get(NodeParam::kTexture); + + if (texture.isNull()) { + return nullptr; + } + + FramePtr frame = Frame::Create(); + frame->set_video_params(video_params_); + frame->set_timestamp(time); + frame->allocate(); + + if (texture.isNull()) { + // Blank frame out + memset(frame->data(), 0, frame->allocated_size()); + } else { + // Dump texture contents to frame + TextureToFrame(texture, frame, video_download_matrix_); + } + + return frame; +} + +NodeValueTable RenderWorker::GenerateBlockTable(const TrackOutput *track, const TimeRange &range) +{ + if (track->track_type() == Timeline::kTrackTypeAudio) { + + QList active_blocks = track->BlocksAtTimeRange(range); + + // All these blocks will need to output to a buffer so we create one here + SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params_, + audio_params_.time_to_samples(range.length())); + block_range_buffer->fill(0); + + NodeValueTable merged_table; + + // Loop through active blocks retrieving their audio + foreach (Block* b, active_blocks) { + TimeRange range_for_block(qMax(b->in(), range.in()), + qMin(b->out(), range.out())); + + int destination_offset = audio_params_.time_to_samples(range_for_block.in() - range.in()); + int max_dest_sz = audio_params_.time_to_samples(range_for_block.length()); + + // Destination buffer + NodeValueTable table = GenerateTable(b, range_for_block); + QVariant sample_val = table.Take(NodeParam::kSamples); + SampleBufferPtr samples_from_this_block; + + if (sample_val.isNull() + || !(samples_from_this_block = sample_val.value())) { + // If we retrieved no samples from this block, do nothing + continue; + } + + // Stretch samples here + rational abs_speed = qAbs(b->speed()); + + if (abs_speed != 1) { + samples_from_this_block->speed(abs_speed.toDouble()); + } + + if (b->is_reversed()) { + // Reverse the audio buffer + samples_from_this_block->reverse(); + } + + int copy_length = qMin(max_dest_sz, samples_from_this_block->sample_count_per_channel()); + + // Copy samples into destination buffer + block_range_buffer->set(samples_from_this_block->const_data(), destination_offset, copy_length); + + { + // Save waveform to file + Block* src_block = static_cast(copy_map_.key(b)); + QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString()); + QDir waveform_loc = local_appdata_dir.filePath(QStringLiteral("waveform")); + waveform_loc.mkpath("."); + QString wave_fn(waveform_loc.filePath(QString::number(reinterpret_cast(src_block)))); + QFile wave_file(wave_fn); + + if (wave_file.open(QFile::ReadWrite)) { + // We use S32 as a size-compatible substitute for SampleSummer::Sum which is 4 bytes in + // size + AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, + audio_params_.channel_layout(), + SampleFormat::SAMPLE_FMT_S32); + + int chunk_size = (audio_params_.sample_rate() / waveform_params.sample_rate()); + + { + // Write metadata header + SampleSummer::Info info; + info.channels = audio_params_.channel_count(); + wave_file.write(reinterpret_cast(&info), sizeof(SampleSummer::Info)); + } + + qint64 start_offset = sizeof(SampleSummer::Info) + waveform_params.time_to_bytes(range_for_block.in() - b->in()); + qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()); + qint64 end_offset = start_offset + length_offset; + + if (wave_file.size() < end_offset) { + wave_file.resize(end_offset); + } + + wave_file.seek(start_offset); + + for (int i=0;isample_count_per_channel();i+=chunk_size) { + QVector summary = SampleSummer::SumSamples(samples_from_this_block, + i, + qMin(chunk_size, samples_from_this_block->sample_count_per_channel() - i)); + + wave_file.write(reinterpret_cast(summary.constData()), + summary.size() * sizeof(SampleSummer::Sum)); + } + + wave_file.close(); + + if (src_block->type() == Block::kClip) { + emit static_cast(src_block)->PreviewUpdated(); + } + } + } + + NodeValueTable::Merge({merged_table, table}); + } + + merged_table.Push(NodeParam::kSamples, QVariant::fromValue(block_range_buffer)); + + return merged_table; + + } else { + return NodeTraverser::GenerateBlockTable(track, range); + } +} + +void RenderWorker::ProcessNodeEvent(const Node *node, const TimeRange &range, NodeValueDatabase &input_params_in, NodeValueTable &output_params) +{ + // Check if node processes samples + if (!(node->GetCapabilities(input_params_in) & Node::kSampleProcessor)) { + return; + } + + // Copy database so we can make some temporary modifications to it + NodeValueDatabase input_params = input_params_in; + NodeInput* sample_input = node->ProcessesSamplesFrom(input_params); + + // Try to find the sample buffer in the table + QVariant samples_var = input_params[sample_input].Get(NodeParam::kSamples); + + // If there isn't one, there's nothing to do + if (samples_var.isNull()) { + return; + } + + SampleBufferPtr input_buffer = samples_var.value(); + + if (!input_buffer) { + return; + } + + SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(input_buffer->audio_params(), input_buffer->sample_count_per_channel()); + + int sample_count = input_buffer->sample_count_per_channel(); + + // FIXME: Hardcoded float sample format + for (int i=0;i(sample_out_of_channel) / static_cast(audio_params_.sample_rate()); + + rational this_sample_time = rational::fromDouble(range.in().toDouble() + sample_to_second); + + // Update all non-sample and non-footage inputs + foreach (NodeParam* param, node->parameters()) { + if (param->type() == NodeParam::kInput + && param != sample_input) { + NodeInput* input = static_cast(param); + + // If the input isn't keyframing, we don't need to update it unless it's connected, in which case it may change + if (input->IsConnected() || input->is_keyframing()) { + input_params.Insert(input, ProcessInput(input, TimeRange(this_sample_time, this_sample_time))); + } + } + } + + node->ProcessSamples(input_params, + audio_params_, + input_buffer, + output_buffer, + i); + } + + output_params.Push(NodeParam::kSamples, QVariant::fromValue(output_buffer)); +} + +void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable *table) +{ + if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) { + + ImageStreamPtr video_stream = std::static_pointer_cast(stream); + rational time_match = (stream->type() == Stream::kImage) ? rational() : input_time.in(); + QString colorspace_match = video_stream->get_colorspace_match_string(); + + NodeValue value; + bool found_cache = false; + + if (still_image_cache_.Has(stream.get())) { + CachedStill cs = still_image_cache_.Get(stream.get()); + + if (cs.colorspace == colorspace_match + && cs.alpha_is_associated == video_stream->premultiplied_alpha() + && cs.divider == video_params_.divider() + && cs.time == time_match) { + value = cs.texture; + found_cache = true; + } else { + still_image_cache_.Remove(stream.get()); + } + } + + if (!found_cache) { + + value = GetDataFromStream(stream, input_time); + + still_image_cache_.Add(stream.get(), {value, + colorspace_match, + video_stream->premultiplied_alpha(), + video_params_.divider(), + time_match}); + + } + + table->Push(value); + + } else if (stream->type() == Stream::kAudio) { + + table->Push(GetDataFromStream(stream, input_time)); + + } +} + +NodeValue RenderWorker::GetDataFromStream(StreamPtr stream, const TimeRange &input_time) +{ + DecoderPtr decoder = ResolveDecoderFromInput(stream); + + if (decoder) { + return FrameToTexture(decoder, stream, input_time); + } + + return NodeValue(); +} + +DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream) +{ + // Access a map of Node inputs and decoder instances and retrieve a frame! + + DecoderPtr decoder = decoder_cache_.Get(stream.get()); + + if (!decoder && stream) { + // Create a new Decoder here + decoder = Decoder::CreateFromID(stream->footage()->decoder()); + decoder->set_stream(stream); + + if (decoder->Open()) { + decoder_cache_.Add(stream.get(), decoder); + } else { + decoder = nullptr; + qWarning() << "Failed to open decoder for" << stream->footage()->filename() + << "::" << stream->index(); + } + } + + return decoder; +} + +void RenderWorker::Queue(NodeInput *input) +{ + if (!queued_updates_.isEmpty()) { + // Remove any inputs that are dependents of this input since they may have been removed since + // it was queued + QList deps = input->GetDependencies(); + + for (int i=0;iparentNode())) { + // We don't need to queue this value since this input supersedes it + queued_updates_.removeAt(i); + i--; + } + } + } + + queued_updates_.append(input); +} + +void RenderWorker::ProcessQueue() +{ + while (!queued_updates_.isEmpty()) { + CopyNodeInputValue(queued_updates_.takeFirst()); + } +} + +void RenderWorker::CopyNodeInputValue(NodeInput *input) +{ + // Find our copy of this parameter + Node* our_copy_node = copy_map_.value(input->parentNode()); + NodeInput* our_copy = our_copy_node->GetInputWithID(input->id()); + + // Copy the standard/keyframe values between these two inputs + NodeInput::CopyValues(input, + our_copy, + false); + + // Handle connections + if (input->IsConnected() || our_copy->IsConnected()) { + // If one of the inputs is connected, it's likely this change came from connecting or + // disconnecting whatever was connected to it + + // We start by removing all old dependencies from the map + QList old_deps = our_copy->GetExclusiveDependencies(); + foreach (Node* i, old_deps) { + delete copy_map_.take(copy_map_.key(i)); + } + + // And clear any other edges + while (!our_copy->edges().isEmpty()) { + NodeParam::DisconnectEdge(our_copy->edges().first()); + } + + // Then we copy all node dependencies and connections (if there are any) + CopyNodeMakeConnection(input, our_copy); + } + + // Call on sub-elements too + if (input->IsArray()) { + foreach (NodeInput* i, static_cast(input)->sub_params()) { + CopyNodeInputValue(i); + } + } +} + +Node* RenderWorker::CopyNodeConnections(Node* src_node) +{ + // Check if this node is already in the map + Node* dst_node = copy_map_.value(src_node); + + // If not, create it now + if (!dst_node) { + dst_node = src_node->copy(); + copy_map_.insert(src_node, dst_node); + } + + // Make sure its values are copied + Node::CopyInputs(src_node, dst_node, false); + + // Copy all connections + QList src_node_inputs = src_node->GetInputsIncludingArrays(); + QList dst_node_inputs = dst_node->GetInputsIncludingArrays(); + + for (int i=0;iIsConnected()) { + Node* dst_node = CopyNodeConnections(src_input->get_connected_node()); + + NodeOutput* corresponding_output = dst_node->GetOutputWithID(src_input->get_connected_output()->id()); + + NodeParam::ConnectEdge(corresponding_output, + dst_input); + } +} + +void RenderWorker::Init(ViewerOutput* viewer) +{ + viewer_ = static_cast(viewer->copy()); + copy_map_.insert(viewer, viewer_); + + Queue(viewer->texture_input()); + Queue(viewer->samples_input()); + ProcessQueue(); +} + +void RenderWorker::Close() +{ + qDeleteAll(copy_map_); + copy_map_.clear(); + + viewer_ = nullptr; +} + +OLIVE_NAMESPACE_EXIT diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h new file mode 100644 index 000000000..0afdb6b26 --- /dev/null +++ b/app/render/backend/renderworker.h @@ -0,0 +1,166 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef RENDERWORKER_H +#define RENDERWORKER_H + +#include + +#include "decodercache.h" +#include "node/traverser.h" +#include "node/output/viewer/viewer.h" +#include "render/backend/rendercache.h" + +OLIVE_NAMESPACE_ENTER + +class RenderWorker : public QObject, public NodeTraverser +{ + Q_OBJECT +public: + RenderWorker(); + + virtual ~RenderWorker() override; + + void Init(ViewerOutput *viewer); + void Close(); + + void Queue(NodeInput* input); + void ProcessQueue(); + + bool IsAvailable() const + { + return available_; + } + + void SetAvailable(bool a) + { + available_ = a; + } + + ViewerOutput* GetViewer() const + { + return viewer_; + } + + void SetVideoParams(const VideoRenderingParams& params) + { + video_params_ = params; + } + + void SetAudioParams(const AudioRenderingParams& params) + { + audio_params_ = params; + } + + void SetVideoDownloadMatrix(const QMatrix4x4& mat) + { + video_download_matrix_ = mat; + } + + /** + * @brief Return a unique ID for the image generated at this time + * + * This hash should always be unique to this image and can therefore be used to match existing + * cached frames. + * + * @return + * + * SHA-1 hash or empty QByteArray if no viewer node is set. + */ + QByteArray Hash(const rational &time) const; + + /** + * @brief Render the frame at this time + * + * Produces a fully rendered frame from the connected viewer at this time. + * + * @return + * + * A frame corresponding to the set video parameters. If no nodes are active at the time, this + * function will still return a blank frame with the same parameters. If no viewer node is set, + * nullptr is returned. + */ + FramePtr RenderFrame(const rational& time); + + SampleBufferPtr RenderAudio(ViewerOutput* viewer, + const TimeRange& range, + const SampleFormat::Format& sample_fmt); + +protected: + virtual void TextureToFrame(const QVariant& texture, FramePtr frame, const QMatrix4x4 &mat) const = 0; + + virtual NodeValue FrameToTexture(DecoderPtr decoder, StreamPtr stream, const TimeRange &range) const = 0; + + virtual void FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable* table) override; + + virtual NodeValueTable GenerateBlockTable(const TrackOutput *track, const TimeRange &range) override; + + virtual void ProcessNodeEvent(const Node *node, const TimeRange &range, NodeValueDatabase &input_params_in, NodeValueTable &output_params) override; + + const VideoRenderingParams& video_params() const + { + return video_params_; + } + + const AudioRenderingParams& audio_params() const + { + return audio_params_; + } + +signals: + void AudioConformUnavailable(); + +private: + NodeValue GetDataFromStream(StreamPtr stream, const TimeRange& input_time); + + DecoderPtr ResolveDecoderFromInput(StreamPtr stream); + + void CopyNodeInputValue(NodeInput* input); + Node *CopyNodeConnections(Node *src_node); + void CopyNodeMakeConnection(NodeInput *src_input, NodeInput *dst_input); + + VideoRenderingParams video_params_; + + AudioRenderingParams audio_params_; + + struct CachedStill { + NodeValue texture; + QString colorspace; + bool alpha_is_associated; + int divider; + rational time; + }; + + RenderCache still_image_cache_; + + QMatrix4x4 video_download_matrix_; + + DecoderCache decoder_cache_; + + ViewerOutput* viewer_; + QList queued_updates_; + QHash copy_map_; + bool available_; + +}; + +OLIVE_NAMESPACE_EXIT + +#endif // RENDERWORKER_H diff --git a/app/widget/viewer/gizmotraverser.cpp b/app/widget/viewer/gizmotraverser.cpp index c8fdafe29..1e77fd925 100644 --- a/app/widget/viewer/gizmotraverser.cpp +++ b/app/widget/viewer/gizmotraverser.cpp @@ -22,7 +22,7 @@ OLIVE_NAMESPACE_ENTER -void GizmoTraverser::FootageProcessingEvent(StreamPtr stream, const TimeRange &/*input_time*/, NodeValueTable *table) const +void GizmoTraverser::FootageProcessingEvent(StreamPtr stream, const TimeRange &/*input_time*/, NodeValueTable *table) { if (stream->type() == Stream::kVideo || stream->type() == Stream::kAudio) { diff --git a/app/widget/viewer/gizmotraverser.h b/app/widget/viewer/gizmotraverser.h index 9292f7b5d..97b2cfc43 100644 --- a/app/widget/viewer/gizmotraverser.h +++ b/app/widget/viewer/gizmotraverser.h @@ -31,7 +31,7 @@ public: GizmoTraverser() = default; protected: - virtual void FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable* table) const override; + virtual void FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable* table) override; }; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index c39d3e2f2..8ab4d7169 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -159,7 +159,7 @@ void ViewerWidget::ConnectNodeInternal(ViewerOutput *n) connect(n, &ViewerOutput::TimebaseChanged, this, &ViewerWidget::SetTimebase); connect(n, &ViewerOutput::SizeChanged, this, &ViewerWidget::SizeChangedSlot); connect(n, &ViewerOutput::LengthChanged, this, &ViewerWidget::LengthChangedSlot); - connect(n, &ViewerOutput::VideoParamsChanged, this, &ViewerWidget::UpdateRendererParameters); + connect(n, &ViewerOutput::ParamsChanged, this, &ViewerWidget::UpdateRendererParameters); connect(n, &ViewerOutput::VisibleInvalidated, this, &ViewerWidget::InvalidateVisible); connect(n, &ViewerOutput::GraphChangedFrom, this, &ViewerWidget::UpdateStack); @@ -202,7 +202,7 @@ void ViewerWidget::DisconnectNodeInternal(ViewerOutput *n) disconnect(n, &ViewerOutput::TimebaseChanged, this, &ViewerWidget::SetTimebase); disconnect(n, &ViewerOutput::SizeChanged, this, &ViewerWidget::SizeChangedSlot); disconnect(n, &ViewerOutput::LengthChanged, this, &ViewerWidget::LengthChangedSlot); - disconnect(n, &ViewerOutput::VideoParamsChanged, this, &ViewerWidget::UpdateRendererParameters); + disconnect(n, &ViewerOutput::ParamsChanged, this, &ViewerWidget::UpdateRendererParameters); disconnect(n, &ViewerOutput::VisibleInvalidated, this, &ViewerWidget::InvalidateVisible); disconnect(n, &ViewerOutput::GraphChangedFrom, this, &ViewerWidget::UpdateStack); @@ -234,12 +234,14 @@ void ViewerWidget::resizeEvent(QResizeEvent *event) { TimeBasedWidget::resizeEvent(event); + /* int new_div = CalculateDivider(); if (new_div != divider_) { divider_ = new_div; UpdateRendererParameters(); } + */ UpdateMinimumScale(); } @@ -365,11 +367,20 @@ void ViewerWidget::UpdateTextureFromNode(const rational& time) QString frame_fn = GetCachedFilenameFromTime(time); if (frame_fn.isEmpty()) { - // FIXME: Connect QFutureWatcher to this - renderer_->RenderFrame(time); + + QFutureWatcher* watcher = new QFutureWatcher(); + connect(watcher, + &QFutureWatcher::finished, + this, + &ViewerWidget::RendererGeneratedFrame); + + watcher->setFuture(renderer_->RenderFrame(time, true)); + } else { + FramePtr f = DecodeCachedImage(frame_fn); SetDisplayImage(f, false); + } } else { SetDisplayImage(nullptr, false); @@ -518,12 +529,15 @@ QString ViewerWidget::GetCachedFilenameFromTime(const rational &time) { if (FrameExistsAtTime(time)) { QByteArray hash = GetConnectedNode()->video_frame_cache()->GetHash(time); - return GetConnectedNode()->video_frame_cache()->CachePathName( - hash, - PixelFormat::instance()->GetConfiguredFormatForMode(RenderMode::kOffline)); - } else { - return QString(); + + if (!hash.isEmpty()) { + + return GetConnectedNode()->video_frame_cache()->CachePathName( + hash, + PixelFormat::instance()->GetConfiguredFormatForMode(RenderMode::kOffline)); + } } + return QString(); } bool ViewerWidget::FrameExistsAtTime(const rational &time) @@ -649,7 +663,8 @@ void ViewerWidget::ContextMenuSetCustomSafeMargins() QMessageBox::warning(this, tr("Invalid custom ratio"), - tr("Failed to parse \"%1\" into an aspect ratio. Please format a rational fraction with a ':' or a '/' separator.").arg(s), + tr("Failed to parse \"%1\" into an aspect ratio. Please format a rational " + "fraction with a ':' or a '/' separator.").arg(s), QMessageBox::Ok); } } @@ -664,17 +679,17 @@ void ViewerWidget::ContextMenuScopeTriggered(QAction *action) emit RequestScopePanel(static_cast(action->data().toInt())); } -void ViewerWidget::RendererGeneratedFrame(FramePtr f) +void ViewerWidget::RendererGeneratedFrame() { - SetDisplayImage(f, false); + QFutureWatcher* watcher = static_cast*>(sender()); + FramePtr frame = watcher->result(); + watcher->deleteLater(); + + SetDisplayImage(frame, false); } void ViewerWidget::UpdateRendererParameters() { - if (!GetConnectedNode()) { - return; - } - RenderMode::Mode render_mode = RenderMode::kOffline; renderer_->SetDivider(divider_); diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 836e96d69..635a4ff60 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -85,11 +85,13 @@ public: void ForceUpdate(); - RenderBackend* renderer() const { + RenderBackend* renderer() const + { return renderer_; } - ColorManager* color_manager() const { + ColorManager* color_manager() const + { return display_widget_->color_manager(); } @@ -254,7 +256,7 @@ private slots: void ContextMenuScopeTriggered(QAction* action); - void RendererGeneratedFrame(FramePtr f); + void RendererGeneratedFrame(); };