From 3b224c76384f5da98c0e79b3f7ff21a2d6bc78d5 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Wed, 6 Nov 2019 09:14:20 +1100 Subject: [PATCH] multithreaded download and files that probably should have been in the last commit For testing the new iteration, the texture cache disk download was written into the main thread instead of into the separate threads. Now they're back in separate threads again. Also I think some of these files probably should have been in the previous commit. --- app/node/block/clip/clip.cpp | 9 + app/node/block/clip/clip.h | 2 + app/node/input/media/video/video.cpp | 4 +- app/node/node.h | 5 - app/node/param.cpp | 2 +- app/render/backend/CMakeLists.txt | 2 + app/render/backend/opengl/openglbackend.cpp | 90 ++-- app/render/backend/opengl/openglbackend.h | 4 +- .../backend/opengl/openglshadercache.cpp | 5 + app/render/backend/opengl/openglshadercache.h | 2 + app/render/backend/opengl/openglworker.cpp | 337 +++------------ app/render/backend/opengl/openglworker.h | 48 +-- app/render/backend/videorenderbackend.h | 3 - app/render/backend/videorenderworker.cpp | 393 +++++++++++++++++- app/render/backend/videorenderworker.h | 82 +++- app/widget/timelinewidget/tool/import.cpp | 16 +- 16 files changed, 606 insertions(+), 398 deletions(-) diff --git a/app/node/block/clip/clip.cpp b/app/node/block/clip/clip.cpp index 06b95dc40..b4b9da2cb 100644 --- a/app/node/block/clip/clip.cpp +++ b/app/node/block/clip/clip.cpp @@ -87,3 +87,12 @@ TimeRange ClipBlock::InputTimeAdjustment(NodeInput *input, const TimeRange &inpu return Block::InputTimeAdjustment(input, input_time); } + +QVariant ClipBlock::Value(NodeOutput *output) +{ + if (output == buffer_output()) { + // We just pass through the texture here, the renderer should have gotten the correct time from InputTimeAdjustment + return texture_input()->value(); + } + return Block::Value(output); +} diff --git a/app/node/block/clip/clip.h b/app/node/block/clip/clip.h index 967de76e6..917495fe1 100644 --- a/app/node/block/clip/clip.h +++ b/app/node/block/clip/clip.h @@ -46,6 +46,8 @@ public: virtual TimeRange InputTimeAdjustment(NodeInput* input, const TimeRange& input_time) override; + virtual QVariant Value(NodeOutput* output) override; + private: NodeInput* texture_input_; diff --git a/app/node/input/media/video/video.cpp b/app/node/input/media/video/video.cpp index da9b3f5c8..6f2bd38db 100644 --- a/app/node/input/media/video/video.cpp +++ b/app/node/input/media/video/video.cpp @@ -58,13 +58,13 @@ QString VideoInput::Code(NodeOutput *output) if (output == texture_output()) { return "#version 110\n" "\n" - "varying vec2 olive_tex_coord;\n" + "varying vec2 v_texcoord;\n" "\n" "uniform sampler2D footage_in;\n" "uniform mat4 matrix_in;\n" "\n" "void main(void) {\n" - " gl_FragColor = texture2D(footage_in, vec2(vec4(olive_tex_coord, 0.0, 1.0) * matrix_in));\n" + " gl_FragColor = texture2D(footage_in, vec2(vec4(v_texcoord, 0.0, 1.0) * matrix_in));\n" "}\n"; } diff --git a/app/node/node.h b/app/node/node.h index d3bba3de7..a66575096 100644 --- a/app/node/node.h +++ b/app/node/node.h @@ -164,11 +164,6 @@ public: */ void DisconnectAll(); - /** - * @brief Add's unique information about this Node at the given time to a QCryptographicHash - */ - //virtual void Hash(QCryptographicHash* hash, NodeOutput *from, const rational& time); - /** * @brief Convert a pointer to a value that can be sent between NodeParams */ diff --git a/app/node/param.cpp b/app/node/param.cpp index bc3083df9..f28a404fe 100644 --- a/app/node/param.cpp +++ b/app/node/param.cpp @@ -202,7 +202,6 @@ QByteArray NodeParam::ValueToBytes(const NodeParam::DataType &type, const QVaria case kFont: return ValueToBytesInternal(value); // FIXME: This should probably be a QFont? case kFile: return ValueToBytesInternal(value); case kMatrix: return ValueToBytesInternal(value); - case kFootage: return ValueToBytesInternal(value); // FIXME: Unsustainble, find some other way to match Footage case kRational: return ValueToBytesInternal(value); case kVec2: return ValueToBytesInternal(value); case kVec3: return ValueToBytesInternal(value); @@ -210,6 +209,7 @@ QByteArray NodeParam::ValueToBytes(const NodeParam::DataType &type, const QVaria // These types have no persistent input case kNone: + case kFootage: case kTexture: case kBlock: case kTrack: diff --git a/app/render/backend/CMakeLists.txt b/app/render/backend/CMakeLists.txt index ea717129a..71955734b 100644 --- a/app/render/backend/CMakeLists.txt +++ b/app/render/backend/CMakeLists.txt @@ -27,6 +27,8 @@ set(OLIVE_SOURCES render/backend/audiorenderbackend.cpp render/backend/videorenderbackend.h render/backend/videorenderbackend.cpp + render/backend/videorenderworker.h + render/backend/videorenderworker.cpp render/backend/decodercache.h render/backend/decodercache.cpp diff --git a/app/render/backend/opengl/openglbackend.cpp b/app/render/backend/opengl/openglbackend.cpp index bf540964b..f905b95ef 100644 --- a/app/render/backend/opengl/openglbackend.cpp +++ b/app/render/backend/opengl/openglbackend.cpp @@ -40,7 +40,7 @@ bool OpenGLBackend::InitInternal() // Connect to it connect(processor, SIGNAL(RequestSibling(NodeDependency)), this, SLOT(ThreadRequestedSibling(NodeDependency))); - connect(processor, SIGNAL(CompletedFrame(NodeDependency)), this, SLOT(CompletedFrame(NodeDependency))); + connect(processor, SIGNAL(CompletedFrame(NodeDependency)), this, SLOT(ThreadCompletedFrame(NodeDependency))); // Finally, we can move it to its own thread processor->moveToThread(thread); @@ -66,16 +66,21 @@ bool OpenGLBackend::InitInternal() void OpenGLBackend::GenerateFrame(const rational &time) { + qDebug() << "Compiled state:" << compiled_; if (!compiled_) { Compile(); } NodeDependency dep = NodeDependency(viewer_node()->texture_input()->get_connected_output(), time, time); - QMetaObject::invokeMethod(processors_.first(), - "Render", - Qt::QueuedConnection, - Q_ARG(NodeDependency, dep)); + foreach (OpenGLWorker* worker, processors_) { + if (worker->IsAvailable() || worker == processors_.last()) { + QMetaObject::invokeMethod(worker, + "Render", + Qt::QueuedConnection, + Q_ARG(NodeDependency, dep)); + } + } } void OpenGLBackend::CloseInternal() @@ -196,66 +201,28 @@ bool OpenGLBackend::TraverseCompiling(Node *n) return true; } -#include -#include -#include "common/define.h" -#include "render/pixelservice.h" -void OpenGLBackend::CompletedFrame(NodeDependency path) +void OpenGLBackend::ThreadCompletedFrame(NodeDependency path) { caching_ = false; OpenGLTexturePtr texture = path.node()->get_cached_value(path.range()).value(); qDebug() << "Retrieved texture for time" << path.in(); - qDebug() << "Texture is" << texture.get(); - - if (texture == nullptr) { - QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions(); - QOpenGLExtraFunctions* xf = QOpenGLContext::currentContext()->extraFunctions(); - - PixelFormatInfo format_info = PixelService::GetPixelFormatInfo(params().format()); - QVector data_buffer(PixelService::GetBufferSize(params().format(), params().width(), params().height())); - qDebug() << "Created buffer of size" << data_buffer.size(); - - // Set up OIIO::ImageSpec for compressing cached images on disk - OIIO::ImageSpec spec(params().width(), params().height(), kRGBAChannels, format_info.oiio_desc); - spec.attribute("compression", "dwaa:200"); - - f->glBindFramebuffer(GL_READ_FRAMEBUFFER, copy_buffer_.buffer()); - - xf->glFramebufferTexture2D(GL_READ_FRAMEBUFFER, - GL_COLOR_ATTACHMENT0, - GL_TEXTURE_2D, - texture->texture(), - 0); - - f->glReadPixels(0, - 0, - texture->width(), - texture->height(), - format_info.pixel_format, - format_info.gl_pixel_type, - data_buffer.data()); - - xf->glFramebufferTexture2D(GL_READ_FRAMEBUFFER, - GL_COLOR_ATTACHMENT0, - GL_TEXTURE_2D, - 0, - 0); - - f->glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); - + if (texture != nullptr) { QString cache_fn = CachePathName(QStringLiteral("%1-%2").arg(QString::number(path.in().numerator()), QString::number(path.in().denominator())).toLatin1()); - std::string working_fn_std = cache_fn.toStdString(); - std::unique_ptr out = OIIO::ImageOutput::create(working_fn_std); - - if (out) { - out->open(working_fn_std, spec); - out->write_image(format_info.oiio_desc, data_buffer.data()); - out->close(); - } else { - qWarning() << "Failed to open output file:" << cache_fn; + // Find an available worker to download this texture + foreach (OpenGLWorker* worker, processors_) { + // Check if one is available, but worst case if none of them are available, just queue it on the last worker since + // it's the least likely to get work + if (worker->IsAvailable() || worker == processors_.last()) { + QMetaObject::invokeMethod(worker, + "Download", + Q_ARG(NodeDependency, path), + Q_ARG(QVariant, QVariant::fromValue(texture)), + Q_ARG(QString, cache_fn)); + break; + } } } @@ -273,7 +240,6 @@ void OpenGLBackend::ThreadCallback(OpenGLTexturePtr texture, const rational& tim // We received a texture, time to start downloading it QString fn = CachePathName(hash); - qDebug() << "INSERT DOWNLOAD CODE!"; /*download_threads_[last_download_thread_%download_threads_.size()]->Queue(texture, fn, hash);*/ @@ -317,12 +283,10 @@ void OpenGLBackend::ThreadCallback(OpenGLTexturePtr texture, const rational& tim void OpenGLBackend::ThreadRequestedSibling(NodeDependency dep) { - Q_UNUSED(dep) - // Try to queue another thread to run this dep in advance - for (int i=1;iIsAvailable()) { - QMetaObject::invokeMethod(processors_.at(i), + foreach (OpenGLWorker* worker, processors_) { + if (worker->IsAvailable()) { + QMetaObject::invokeMethod(worker, "RenderAsSibling", Qt::QueuedConnection, Q_ARG(NodeDependency, dep)); diff --git a/app/render/backend/opengl/openglbackend.h b/app/render/backend/opengl/openglbackend.h index 1ba85187a..963d39350 100644 --- a/app/render/backend/opengl/openglbackend.h +++ b/app/render/backend/opengl/openglbackend.h @@ -45,13 +45,13 @@ private: bool compiled_; private slots: - void CompletedFrame(NodeDependency path); + void ThreadCompletedFrame(NodeDependency path); + void ThreadRequestedSibling(NodeDependency dep); void ThreadCallback(OpenGLTexturePtr texture, const rational& time, const QByteArray& hash); - void ThreadRequestedSibling(NodeDependency dep); void ThreadSkippedFrame(const rational &time, const QByteArray &hash); diff --git a/app/render/backend/opengl/openglshadercache.cpp b/app/render/backend/opengl/openglshadercache.cpp index 724c1cadc..92df2171f 100644 --- a/app/render/backend/opengl/openglshadercache.cpp +++ b/app/render/backend/opengl/openglshadercache.cpp @@ -27,3 +27,8 @@ OpenGLShaderPtr OpenGLShaderCache::GetShader(NodeOutput *output) { return compiled_nodes_.value(GenerateShaderID(output)); } + +bool OpenGLShaderCache::HasShader(NodeOutput *output) +{ + return compiled_nodes_.contains(GenerateShaderID(output)); +} diff --git a/app/render/backend/opengl/openglshadercache.h b/app/render/backend/opengl/openglshadercache.h index 16408db27..c4bcd18fc 100644 --- a/app/render/backend/opengl/openglshadercache.h +++ b/app/render/backend/opengl/openglshadercache.h @@ -20,6 +20,8 @@ public: OpenGLShaderPtr GetShader(NodeOutput* output); + bool HasShader(NodeOutput* output); + private: QString GenerateShaderID(NodeOutput* output); diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index 259d98f0d..701347461 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -1,19 +1,15 @@ #include "openglworker.h" -#include - #include "functions.h" -#include "node/block/block.h" #include "node/node.h" +#include "render/pixelservice.h" OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, QObject *parent) : - QObject(parent), + VideoRenderWorker(decoder_cache, parent), share_ctx_(share_ctx), ctx_(nullptr), functions_(nullptr), - shader_cache_(shader_cache), - decoder_cache_(decoder_cache), - working_(0) + shader_cache_(shader_cache) { surface_.create(); } @@ -23,17 +19,7 @@ OpenGLWorker::~OpenGLWorker() surface_.destroy(); } -bool OpenGLWorker::IsStarted() -{ - return ctx_ != nullptr; -} - -void OpenGLWorker::SetParameters(const VideoRenderingParams &video_params) -{ - video_params_ = video_params; -} - -void OpenGLWorker::Init() +bool OpenGLWorker::InitInternal() { // Create context object ctx_ = new QOpenGLContext(); @@ -44,8 +30,7 @@ void OpenGLWorker::Init() // Create OpenGL context (automatically destroys any existing if there is one) if (!ctx_->create()) { qWarning() << "Failed to create OpenGL context in thread" << thread(); - Close(); - return; + return false; } ctx_->moveToThread(this->thread()); @@ -54,14 +39,26 @@ void OpenGLWorker::Init() // The rest of the initialization needs to occur in the other thread, so we signal for it to start QMetaObject::invokeMethod(this, "FinishInit", Qt::QueuedConnection); + + return true; } -bool OpenGLWorker::IsAvailable() +QVariant OpenGLWorker::FrameToTexture(FramePtr frame) { - return (working_ == 0); + OpenGLTexturePtr footage_tex = std::make_shared(); + footage_tex->Create(ctx_, frame); + + // FIXME: Alpha association and color management + + return QVariant::fromValue(footage_tex); } -void OpenGLWorker::Close() +bool OpenGLWorker::OutputIsShader(NodeOutput* output) +{ + return shader_cache_->HasShader(output); +} + +void OpenGLWorker::CloseInternal() { buffer_.Destroy(); @@ -69,124 +66,26 @@ void OpenGLWorker::Close() delete ctx_; } -void OpenGLWorker::Render(NodeDependency path) +void OpenGLWorker::ParametersChangedEvent() { - NodeOutput* output = path.node(); - Node* node = output->parent(); - - QList all_nodes_in_graph; - all_nodes_in_graph.append(node); - all_nodes_in_graph.append(node->GetDependencies()); - - // Lock all Nodes to prevent UI changes during this render - foreach (Node* dep, all_nodes_in_graph) { - dep->LockUserInput(); - } - - // Start traversing graph - RenderAsSibling(path); - - // Start OpenGL flushing now while we do clean up work on the CPU - functions_->glFlush(); - - // Unlock all Nodes so changes can be made again - foreach (Node* dep, all_nodes_in_graph) { - dep->UnlockUserInput(); - } - - // Now we need the texture done so we call glFinish() - functions_->glFinish(); - - emit CompletedFrame(path); -} - -void OpenGLWorker::UpdateViewportFromParams() -{ - if (functions_ != nullptr && video_params_.is_valid()) { - functions_->glViewport(0, 0, video_params_.effective_width(), video_params_.effective_height()); + if (functions_ != nullptr && video_params().is_valid()) { + functions_->glViewport(0, 0, video_params().effective_width(), video_params().effective_height()); } } -Node *OpenGLWorker::ValidateBlock(Node *n, const rational& time) +QVariant OpenGLWorker::RunNodeAsShader(NodeOutput *out) { - if (n->IsBlock()) { - Block* block = static_cast(n); + OpenGLShaderPtr shader = shader_cache_->GetShader(out); + Node* node = out->parent(); - while (block->in() > time && block != nullptr) { - // This Block is too late, find an earlier one - block = block->previous(); - } + // Create the output texture + OpenGLTexturePtr output = std::make_shared(); + output->Create(ctx_, video_params().effective_width(), video_params().effective_height(), video_params().format()); - while (block->out() <= time && block != nullptr) { - // This block is too early, find a later one - block = block->next(); - } + buffer_.Attach(output); - // By this point, we should have the correct Block or nullptr if there's no Block here - return block; - } + buffer_.Bind(); - return n; -} - -QList OpenGLWorker::ProcessNodeInputsForTime(Node *n, const TimeRange &time) -{ - QList connected_inputs; - - // Now we need to gather information about this Node's inputs - foreach (NodeParam* param, n->parameters()) { - // Check if this parameter is an input and if the Node is dependent on it - if (param->type() == NodeParam::kInput) { - NodeInput* input = static_cast(param); - - if (input->dependent()) { - // If we're here, this input is necessary and we need to acquire the value for this Node - if (input->IsConnected()) { - // If it's connected to something, we need to retrieve that output at some point - connected_inputs.append(input); - } else { - // If it isn't connected, it'll have the value we need inside it. We just need to store it for the node. - input->set_stored_value(input->get_value_at_time(n->InputTimeAdjustment(input, time).in())); - } - - // Special types like FOOTAGE require extra work from us (to decrease node complexity dealing with decoders) - if (input->data_type() == NodeParam::kFootage) { - input->set_stored_value(0); - - // Access a map of Node inputs and decoder instances and retrieve a frame! - StreamPtr stream = input->get_value_at_time(0).value(); - DecoderPtr decoder = decoder_cache_->GetDecoder(stream.get()); - - if (decoder == nullptr && stream != nullptr) { - // Init decoder - decoder = Decoder::CreateFromID(stream->footage()->decoder()); - decoder->set_stream(stream); - decoder_cache_->AddDecoder(stream.get(), decoder); - } - - // By this point we should definitely have a decoder, and if we don't something's gone terribly wrong - if (decoder != nullptr) { - FramePtr frame = decoder->Retrieve(time.in()); - - if (frame != nullptr) { - OpenGLTexturePtr footage_tex = std::make_shared(); - footage_tex->Create(ctx_, frame, OpenGLTexture::kDoubleBuffer); - - // FIXME: Alpha association and color management - - input->set_stored_value(QVariant::fromValue(footage_tex)); - } - } - } - } - } - } - - return connected_inputs; -} - -OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shader) -{ shader->bind(); unsigned int input_texture_count = 0; @@ -229,11 +128,14 @@ OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shade case NodeInput::kFootage: { OpenGLTexturePtr texture = input->value().value(); + qDebug() << " Binding" << texture->texture() << "from" << input << "to GL_TEXTURE" << input_texture_count; + functions_->glActiveTexture(GL_TEXTURE0 + input_texture_count); functions_->glBindTexture(GL_TEXTURE_2D, texture->texture()); // Set value to bound texture shader->setUniformValue(variable_location, input_texture_count); + qDebug() << " Setting" << input->id() << "to" << input_texture_count; input_texture_count++; break; @@ -253,20 +155,9 @@ OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shade } } - // Create the output texture - OpenGLTexturePtr output = std::make_shared(); - output->Create(ctx_, video_params_.width(), video_params_.height(), video_params_.format()); - - buffer_.Attach(output); - - buffer_.Bind(); - + qDebug() << " Blitting with shader!"; olive::gl::Blit(shader); - buffer_.Release(); - - buffer_.Detach(); - // Release any textures we bound before while (input_texture_count > 0) { input_texture_count--; @@ -278,7 +169,35 @@ OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shade shader->release(); - return output; + buffer_.Release(); + + buffer_.Detach(); + + functions_->glFinish(); + + return QVariant::fromValue(output); +} + +void OpenGLWorker::TextureToBuffer(const QVariant &tex_in, QByteArray &buffer) +{ + OpenGLTexturePtr texture = tex_in.value(); + + PixelFormatInfo format_info = PixelService::GetPixelFormatInfo(video_params().format()); + + QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions(); + buffer_.Attach(texture); + f->glBindFramebuffer(GL_READ_FRAMEBUFFER, buffer_.buffer()); + + f->glReadPixels(0, + 0, + texture->width(), + texture->height(), + format_info.pixel_format, + format_info.gl_pixel_type, + buffer.data()); + + f->glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); + buffer_.Detach(); } void OpenGLWorker::FinishInit() @@ -294,133 +213,7 @@ void OpenGLWorker::FinishInit() // Set up OpenGL parameters as necessary functions_->glEnable(GL_BLEND); - UpdateViewportFromParams(); + ParametersChangedEvent(); buffer_.Create(ctx_); - - //qDebug() << "Context in" << ctx_->thread() << "successfully finished"; -} - -void OpenGLWorker::RenderAsSibling(NodeDependency dep) -{ - NodeOutput* output = dep.node(); - Node* original_node = output->parent(); - Node* node; - rational time = dep.in(); - QList connected_inputs; - OpenGLShaderPtr shader; - - // Set working state - working_++; - - original_node->LockProcessing(); - - // Firstly we check if this node is a "Block", if it is that means it's part of a linked list of mutually exclusive - // nodes based on time and we might need to locate which Block to attach to - if ((node = ValidateBlock(original_node, time)) == nullptr) { - // ValidateBlock() may have returned nullptr if there was no Block found at this time so no texture to return - output->cache_value(dep.range(), 0); - - original_node->UnlockProcessing(); - goto end_render; - } - - if (original_node != node) { - // Ensure output is the output matching the node as it may have changed - output = static_cast(node->GetParameterWithID(output->id())); - - // Switch locks - original_node->UnlockProcessing(); - node->LockProcessing(); - } - - // Check if the output already has a value for this time - if (output->has_cached_value(dep.range())) { - // If so, we don't need to do anything, we can just send this value and exit here - dep.node()->cache_value(dep.range(), output->get_cached_value(dep.range())); - - node->UnlockProcessing(); - goto end_render; - } - - // We need to run the Node's code to get the correct value for this time - - connected_inputs = ProcessNodeInputsForTime(node, dep.range()); - - // For each connected input, we need to acquire the value from another node - while (!connected_inputs.isEmpty()) { - - // Remove any inputs from the list that we have valid cached values for already - for (int i=0;iget_connected_output(); - TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); - - if (connected_output->has_cached_value(input_time)) { - // This output already has this value, no need to process it again - input->set_stored_value(connected_output->get_cached_value(input_time)); - connected_inputs.removeAt(i); - i--; - } - } - - // For every connected input except the first, we'll request another Node to do it - int input_for_this_thread = -1; - - for (int i=0;iget_connected_node()->IsProcessingLocked()) { - if (input_for_this_thread == -1) { - // Store this later since we can process it on this thread as we wait for other threads - input_for_this_thread = i; - } else { - TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); - - emit RequestSibling(NodeDependency(input->get_connected_output(), - input_time)); - } - } - } - - if (input_for_this_thread > -1) { - // In the mean time, this thread can go off to do the first parameter - NodeInput* input = connected_inputs.at(input_for_this_thread); - TimeRange input_range = node->InputTimeAdjustment(input, dep.range()); - RenderAsSibling(NodeDependency(input->get_connected_output(), - input_range)); - input->set_stored_value(input->get_connected_output()->get_cached_value(input_range)); - connected_inputs.removeAt(input_for_this_thread); - } else { - // Nothing for this thread to do. We'll wait 0.5 sec and check again for other nodes - // FIXME: It would be nicer if this thread could do other nodes during this time - QThread::msleep(500); - } - } - - // By this point, the node should have all the inputs it needs to render correctly - - // Check if we have a shader for this output - shader = shader_cache_->GetShader(output); - - if (shader != nullptr) { - // Run code - OpenGLTexturePtr texture = RunNodeAsShader(node, shader); - - output->cache_value(dep.range(), QVariant::fromValue(texture)); - } else { - // Generate the value as expected - QVariant value = node->Value(output); - - // Place the value into the output - output->cache_value(dep.range(), value); - } - - // We're done! - node->UnlockProcessing(); - -end_render: - // End this working state - working_--; } diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index 75dc0ec42..85e8ec900 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -1,29 +1,21 @@ #ifndef OPENGLPROCESSOR_H #define OPENGLPROCESSOR_H -#include #include #include -#include "../decodercache.h" -#include "node/dependency.h" +#include "../videorenderworker.h" #include "openglframebuffer.h" #include "openglshadercache.h" -#include "render/videoparams.h" -class OpenGLWorker : public QObject { +class OpenGLWorker : public VideoRenderWorker { Q_OBJECT public: OpenGLWorker(QOpenGLContext* share_ctx, OpenGLShaderCache* shader_cache, DecoderCache* decoder_cache, QObject* parent = nullptr); virtual ~OpenGLWorker() override; - Q_DISABLE_COPY_MOVE(OpenGLWorker) - - bool IsStarted(); - - void SetParameters(const VideoRenderingParams& video_params); - +protected: /** * @brief Initialize OpenGL instance in whatever thread this object is a part of * @@ -45,35 +37,21 @@ public: * the main thread from using the context during initialization and preventing more than one shared context being made * at the same time (which may or may not actually make a difference). */ - void Init(); + virtual bool InitInternal() override; - bool IsAvailable(); + virtual void CloseInternal() override; -public slots: - void Close(); + virtual QVariant FrameToTexture(FramePtr frame) override; - void Render(NodeDependency path); + virtual bool OutputIsShader(NodeOutput *output) override; - void RenderAsSibling(NodeDependency dep); + virtual QVariant RunNodeAsShader(NodeOutput *output) override; - //void Download(); + virtual void TextureToBuffer(const QVariant& texture, QByteArray& buffer) override; -signals: - void RequestSibling(NodeDependency path); - - void CompletedFrame(NodeDependency path); + virtual void ParametersChangedEvent() override; private: - void ProcessNode(); - - void UpdateViewportFromParams(); - - Node* ValidateBlock(Node* n, const rational& time); - - QList ProcessNodeInputsForTime(Node* n, const TimeRange& time); - - OpenGLTexturePtr RunNodeAsShader(Node *node, OpenGLShaderPtr shader); - QOpenGLContext* share_ctx_; QOpenGLContext* ctx_; @@ -83,14 +61,8 @@ private: OpenGLFramebuffer buffer_; - VideoRenderingParams video_params_; - OpenGLShaderCache* shader_cache_; - DecoderCache* decoder_cache_; - - QAtomicInt working_; - private slots: void FinishInit(); diff --git a/app/render/backend/videorenderbackend.h b/app/render/backend/videorenderbackend.h index c4e9eeb09..03711501e 100644 --- a/app/render/backend/videorenderbackend.h +++ b/app/render/backend/videorenderbackend.h @@ -148,9 +148,6 @@ private: QByteArray cache_frame_load_buffer_; - /*QVector download_threads_; - int last_download_thread_;*/ - private slots: diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index 469c6ceda..b3f261331 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -1,6 +1,397 @@ #include "videorenderworker.h" -VideoRenderWorker::VideoRenderWorker() +#include + +#include "common/define.h" +#include "node/block/block.h" +#include "node/node.h" +#include "render/pixelservice.h" + +VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, QObject *parent) : + QObject(parent), + decoder_cache_(decoder_cache), + started_(false) { } + +bool VideoRenderWorker::IsAvailable() +{ + return (working_ == 0); +} + +void VideoRenderWorker::Close() +{ + CloseInternal(); + + started_ = false; +} + +const VideoRenderingParams &VideoRenderWorker::video_params() +{ + return video_params_; +} + +DecoderCache *VideoRenderWorker::decoder_cache() +{ + return decoder_cache_; +} + +void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, const rational& time) +{ + // Resolve BlockList + if (n->IsBlock() && (n = ValidateBlock(n, time)) == nullptr) { + return; + } + + // Add this Node's ID + hash->addData(n->id().toUtf8()); + + foreach (NodeParam* param, n->parameters()) { + // For each input, try to hash its value + if (param->type() == NodeParam::kInput) { + NodeInput* input = static_cast(param); + + // Get time adjustment + TimeRange range = n->InputTimeAdjustment(input, TimeRange(time, time)); + + // For a single frame, we only care about one of the times + rational input_time = range.in(); + + if (input->IsConnected()) { + // Traverse down this edge + HashNodeRecursively(hash, input->get_connected_node(), input_time); + } else { + // Grab the value at this time + QVariant value = input->get_value_at_time(input_time); + hash->addData(NodeParam::ValueToBytes(input->data_type(), value)); + } + + // We have one exception for FOOTAGE types, since we resolve the footage into a frame in the renderer + if (input->data_type() == NodeParam::kFootage) { + StreamPtr stream = ResolveStreamFromInput(input); + DecoderPtr decoder = ResolveDecoderFromInput(input); + + if (decoder != nullptr) { + // Add footage details to hash + + // Footage filename + hash->addData(stream->footage()->filename().toUtf8()); + + // Footage last modified date + hash->addData(stream->footage()->timestamp().toString().toUtf8()); + + // Footage stream + hash->addData(QString::number(stream->index()).toUtf8()); + + // Footage timestamp + hash->addData(QString::number(decoder->GetTimestampFromTime(time)).toUtf8()); + + // FIXME: Add colorspace and alpha assoc + } + } + } + } +} + +StreamPtr VideoRenderWorker::ResolveStreamFromInput(NodeInput *input) +{ + return input->get_value_at_time(0).value(); +} + +DecoderPtr VideoRenderWorker::ResolveDecoderFromInput(NodeInput *input) +{ + // Access a map of Node inputs and decoder instances and retrieve a frame! + StreamPtr stream = ResolveStreamFromInput(input); + DecoderPtr decoder = decoder_cache()->GetDecoder(stream.get()); + + if (decoder == nullptr && stream != nullptr) { + // Init decoder + decoder = Decoder::CreateFromID(stream->footage()->decoder()); + decoder->set_stream(stream); + decoder_cache()->AddDecoder(stream.get(), decoder); + } + + return decoder; +} + +Node *VideoRenderWorker::ValidateBlock(Node *n, const rational& time) +{ + if (n->IsBlock()) { + Block* block = static_cast(n); + + while (block != nullptr && block->in() > time) { + // This Block is too late, find an earlier one + block = block->previous(); + } + + while (block != nullptr && block->out() <= time) { + // This block is too early, find a later one + block = block->next(); + } + + // By this point, we should have the correct Block or nullptr if there's no Block here + return block; + } + + return n; +} + +void VideoRenderWorker::SetParameters(const VideoRenderingParams &video_params) +{ + video_params_ = video_params; + + ParametersChangedEvent(); +} + +bool VideoRenderWorker::Init() +{ + if (started_) { + return true; + } + + started_ = InitInternal(); + + if (started_) { + download_buffer_.resize(PixelService::GetBufferSize(video_params().format(), video_params().effective_width(), video_params().effective_height())); + } else { + Close(); + } + + return started_; +} + +bool VideoRenderWorker::IsStarted() +{ + return started_; +} + +void VideoRenderWorker::Render(NodeDependency path) +{ + NodeOutput* output = path.node(); + Node* node = output->parent(); + + QList all_nodes_in_graph; + all_nodes_in_graph.append(node); + all_nodes_in_graph.append(node->GetDependencies()); + + // Lock all Nodes to prevent UI changes during this render + foreach (Node* dep, all_nodes_in_graph) { + dep->LockUserInput(); + } + + // Start traversing graph + RenderAsSibling(path); + + // Unlock all Nodes so changes can be made again + foreach (Node* dep, all_nodes_in_graph) { + dep->UnlockUserInput(); + } + + emit CompletedFrame(path); +} + +QList VideoRenderWorker::ProcessNodeInputsForTime(Node *n, const TimeRange &time) +{ + QList connected_inputs; + + // Now we need to gather information about this Node's inputs + foreach (NodeParam* param, n->parameters()) { + // Check if this parameter is an input and if the Node is dependent on it + if (param->type() == NodeParam::kInput) { + NodeInput* input = static_cast(param); + + if (input->dependent()) { + // If we're here, this input is necessary and we need to acquire the value for this Node + if (input->IsConnected()) { + // If it's connected to something, we need to retrieve that output at some point + connected_inputs.append(input); + } else { + // If it isn't connected, it'll have the value we need inside it. We just need to store it for the node. + input->set_stored_value(input->get_value_at_time(n->InputTimeAdjustment(input, time).in())); + } + + // Special types like FOOTAGE require extra work from us (to decrease node complexity dealing with decoders) + if (input->data_type() == NodeParam::kFootage) { + input->set_stored_value(0); + + DecoderPtr decoder = ResolveDecoderFromInput(input); + + // By this point we should definitely have a decoder, and if we don't something's gone terribly wrong + if (decoder != nullptr) { + FramePtr frame = decoder->Retrieve(time.in()); + + if (frame != nullptr) { + QVariant value = FrameToTexture(frame); + + input->set_stored_value(value); + + qDebug() << "Placing texture" << value << "into input" << input; + } + } + } + } + } + } + + return connected_inputs; +} + +void VideoRenderWorker::RenderAsSibling(NodeDependency dep) +{ + NodeOutput* output = dep.node(); + Node* original_node = output->parent(); + Node* node; + rational time = dep.in(); + QList connected_inputs; + QVariant value; + + // Set working state + working_++; + + qDebug() << "Processing" << original_node->id() << original_node; + + original_node->LockProcessing(); + + // Firstly we check if this node is a "Block", if it is that means it's part of a linked list of mutually exclusive + // nodes based on time and we might need to locate which Block to attach to + if ((node = ValidateBlock(original_node, time)) == nullptr) { + // ValidateBlock() may have returned nullptr if there was no Block found at this time so no texture to return + output->cache_value(dep.range(), 0); + + original_node->UnlockProcessing(); + goto end_render; + } + + if (original_node != node) { + // Ensure output is the output matching the node as it may have changed + output = static_cast(node->GetParameterWithID(output->id())); + + // Switch locks + original_node->UnlockProcessing(); + node->LockProcessing(); + + qDebug() << "Deftly switched from" << original_node->id() << original_node << "to" << node->id() << node; + } + + // Check if the output already has a value for this time + if (output->has_cached_value(dep.range())) { + // If so, we don't need to do anything, we can just send this value and exit here + dep.node()->cache_value(dep.range(), output->get_cached_value(dep.range())); + + qDebug() << "Found a cached value on" << node->id() << output->id() << "at" << dep.range().in() << "-" << dep.range().out(); + + node->UnlockProcessing(); + goto end_render; + } + + // We need to run the Node's code to get the correct value for this time + + connected_inputs = ProcessNodeInputsForTime(node, dep.range()); + + // For each connected input, we need to acquire the value from another node + while (!connected_inputs.isEmpty()) { + + // Remove any inputs from the list that we have valid cached values for already + for (int i=0;iget_connected_output(); + TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); + + if (connected_output->has_cached_value(input_time)) { + // This output already has this value, no need to process it again + input->set_stored_value(connected_output->get_cached_value(input_time)); + connected_inputs.removeAt(i); + i--; + } + } + + // For every connected input except the first, we'll request another Node to do it + int input_for_this_thread = -1; + + for (int i=0;iget_connected_node()->IsProcessingLocked()) { + if (input_for_this_thread == -1) { + // Store this later since we can process it on this thread as we wait for other threads + input_for_this_thread = i; + } else { + TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); + + emit RequestSibling(NodeDependency(input->get_connected_output(), + input_time)); + } + } + } + + if (input_for_this_thread > -1) { + // In the mean time, this thread can go off to do the first parameter + NodeInput* input = connected_inputs.at(input_for_this_thread); + TimeRange input_range = node->InputTimeAdjustment(input, dep.range()); + RenderAsSibling(NodeDependency(input->get_connected_output(), + input_range)); + input->set_stored_value(input->get_connected_output()->get_cached_value(input_range)); + connected_inputs.removeAt(input_for_this_thread); + } else { + // Nothing for this thread to do. We'll wait 0.5 sec and check again for other nodes + // FIXME: It would be nicer if this thread could do other nodes during this time + QThread::msleep(500); + } + } + + // By this point, the node should have all the inputs it needs to render correctly + + // Check if we have a shader for this output + if (OutputIsShader(output)) { + // Run code + value = RunNodeAsShader(output); + } else { + // Generate the value as expected + value = node->Value(output); + } + + // Place the value into the output + output->cache_value(dep.range(), value); + dep.node()->cache_value(dep.range(), value); + + // We're done! + node->UnlockProcessing(); + +end_render: + // End this working state + working_--; +} + + + +void VideoRenderWorker::Download(NodeDependency dep, QVariant texture, QString filename) +{ + working_++; + + PixelFormatInfo format_info = PixelService::GetPixelFormatInfo(video_params().format()); + + // Set up OIIO::ImageSpec for compressing cached images on disk + OIIO::ImageSpec spec(video_params().effective_width(), video_params().effective_height(), kRGBAChannels, format_info.oiio_desc); + spec.attribute("compression", "dwaa:200"); + + TextureToBuffer(texture, download_buffer_); + + std::string working_fn_std = filename.toStdString(); + + std::unique_ptr out = OIIO::ImageOutput::create(working_fn_std); + + if (out) { + qDebug() << "Saving to" << filename; + out->open(working_fn_std, spec); + out->write_image(format_info.oiio_desc, download_buffer_.data()); + out->close(); + + emit CompletedDownload(dep); + } else { + qWarning() << "Failed to open output file:" << filename; + } + + working_--; +} diff --git a/app/render/backend/videorenderworker.h b/app/render/backend/videorenderworker.h index dc017d07f..ed3486731 100644 --- a/app/render/backend/videorenderworker.h +++ b/app/render/backend/videorenderworker.h @@ -1,11 +1,87 @@ #ifndef VIDEORENDERWORKER_H #define VIDEORENDERWORKER_H +#include +#include -class VideoRenderWorker -{ +#include "decodercache.h" +#include "node/dependency.h" +#include "render/videoparams.h" + +class VideoRenderWorker : public QObject { + Q_OBJECT public: - VideoRenderWorker(); + VideoRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); + + Q_DISABLE_COPY_MOVE(VideoRenderWorker) + + bool IsStarted(); + + void SetParameters(const VideoRenderingParams& video_params); + + virtual bool Init(); + + bool IsAvailable(); + +public slots: + void Close(); + + void Render(NodeDependency path); + + void RenderAsSibling(NodeDependency dep); + + void Download(NodeDependency dep, QVariant texture, QString filename); + +signals: + void RequestSibling(NodeDependency path); + + void CompletedFrame(NodeDependency path); + + void CompletedDownload(NodeDependency path); + +protected: + virtual bool InitInternal() = 0; + + virtual void CloseInternal() = 0; + + virtual QVariant FrameToTexture(FramePtr frame) = 0; + + const VideoRenderingParams& video_params(); + + DecoderCache* decoder_cache(); + + Node* ValidateBlock(Node* n, const rational& time); + + virtual void ParametersChangedEvent(){} + + virtual bool OutputIsShader(NodeOutput *output) = 0; + + virtual QVariant RunNodeAsShader(NodeOutput *output) = 0; + + virtual void TextureToBuffer(const QVariant& texture, QByteArray& buffer) = 0; + +private: + void ProcessNode(); + + StreamPtr ResolveStreamFromInput(NodeInput* input); + DecoderPtr ResolveDecoderFromInput(NodeInput* input); + + QList ProcessNodeInputsForTime(Node* n, const TimeRange& time); + + void HashNodeRecursively(QCryptographicHash* hash, Node *n, const rational &time); + + VideoRenderingParams video_params_; + + DecoderCache* decoder_cache_; + + QAtomicInt working_; + + QByteArray download_buffer_; + + bool started_; + +private slots: + }; #endif // VIDEORENDERWORKER_H diff --git a/app/widget/timelinewidget/tool/import.cpp b/app/widget/timelinewidget/tool/import.cpp index 5eb9a9c92..5c5656e22 100644 --- a/app/widget/timelinewidget/tool/import.cpp +++ b/app/widget/timelinewidget/tool/import.cpp @@ -218,14 +218,14 @@ void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event) ClipBlock* clip = new ClipBlock(); VideoInput* media = new VideoInput(); - TransformDistort* transform = new TransformDistort(); - OpacityNode* opacity = new OpacityNode(); + //TransformDistort* transform = new TransformDistort(); + //OpacityNode* opacity = new OpacityNode(); // Set parents to node_memory_manager in case no TimelineOutput receives this signal clip->setParent(&node_memory_manager); media->setParent(&node_memory_manager); - transform->setParent(&node_memory_manager); - opacity->setParent(&node_memory_manager); + //transform->setParent(&node_memory_manager); + //opacity->setParent(&node_memory_manager); StreamPtr footage_stream = ghost->data(TimelineViewGhostItem::kAttachedFootage).value(); media->SetFootage(footage_stream); @@ -233,10 +233,10 @@ void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event) clip->set_length(ghost->Length()); clip->set_block_name(footage_stream->footage()->name()); - NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input()); - NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input()); - NodeParam::ConnectEdge(transform->matrix_output(), media->matrix_input()); - + //NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input()); + //NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input()); + //NodeParam::ConnectEdge(transform->matrix_output(), media->matrix_input()); + NodeParam::ConnectEdge(media->texture_output(), clip->texture_input()); if (event->GetModifiers() & Qt::ControlModifier) { //emit parent()->RequestInsertBlockAtTime(clip, ghost->GetAdjustedIn());