diff --git a/app/node/block/clip/clip.cpp b/app/node/block/clip/clip.cpp index 06b95dc40..b4b9da2cb 100644 --- a/app/node/block/clip/clip.cpp +++ b/app/node/block/clip/clip.cpp @@ -87,3 +87,12 @@ TimeRange ClipBlock::InputTimeAdjustment(NodeInput *input, const TimeRange &inpu return Block::InputTimeAdjustment(input, input_time); } + +QVariant ClipBlock::Value(NodeOutput *output) +{ + if (output == buffer_output()) { + // We just pass through the texture here, the renderer should have gotten the correct time from InputTimeAdjustment + return texture_input()->value(); + } + return Block::Value(output); +} diff --git a/app/node/block/clip/clip.h b/app/node/block/clip/clip.h index 967de76e6..917495fe1 100644 --- a/app/node/block/clip/clip.h +++ b/app/node/block/clip/clip.h @@ -46,6 +46,8 @@ public: virtual TimeRange InputTimeAdjustment(NodeInput* input, const TimeRange& input_time) override; + virtual QVariant Value(NodeOutput* output) override; + private: NodeInput* texture_input_; diff --git a/app/node/input/media/video/video.cpp b/app/node/input/media/video/video.cpp index da9b3f5c8..6f2bd38db 100644 --- a/app/node/input/media/video/video.cpp +++ b/app/node/input/media/video/video.cpp @@ -58,13 +58,13 @@ QString VideoInput::Code(NodeOutput *output) if (output == texture_output()) { return "#version 110\n" "\n" - "varying vec2 olive_tex_coord;\n" + "varying vec2 v_texcoord;\n" "\n" "uniform sampler2D footage_in;\n" "uniform mat4 matrix_in;\n" "\n" "void main(void) {\n" - " gl_FragColor = texture2D(footage_in, vec2(vec4(olive_tex_coord, 0.0, 1.0) * matrix_in));\n" + " gl_FragColor = texture2D(footage_in, vec2(vec4(v_texcoord, 0.0, 1.0) * matrix_in));\n" "}\n"; } diff --git a/app/node/node.h b/app/node/node.h index d3bba3de7..a66575096 100644 --- a/app/node/node.h +++ b/app/node/node.h @@ -164,11 +164,6 @@ public: */ void DisconnectAll(); - /** - * @brief Add's unique information about this Node at the given time to a QCryptographicHash - */ - //virtual void Hash(QCryptographicHash* hash, NodeOutput *from, const rational& time); - /** * @brief Convert a pointer to a value that can be sent between NodeParams */ diff --git a/app/node/param.cpp b/app/node/param.cpp index bc3083df9..f28a404fe 100644 --- a/app/node/param.cpp +++ b/app/node/param.cpp @@ -202,7 +202,6 @@ QByteArray NodeParam::ValueToBytes(const NodeParam::DataType &type, const QVaria case kFont: return ValueToBytesInternal(value); // FIXME: This should probably be a QFont? case kFile: return ValueToBytesInternal(value); case kMatrix: return ValueToBytesInternal(value); - case kFootage: return ValueToBytesInternal(value); // FIXME: Unsustainble, find some other way to match Footage case kRational: return ValueToBytesInternal(value); case kVec2: return ValueToBytesInternal(value); case kVec3: return ValueToBytesInternal(value); @@ -210,6 +209,7 @@ QByteArray NodeParam::ValueToBytes(const NodeParam::DataType &type, const QVaria // These types have no persistent input case kNone: + case kFootage: case kTexture: case kBlock: case kTrack: diff --git a/app/render/backend/CMakeLists.txt b/app/render/backend/CMakeLists.txt index ea717129a..71955734b 100644 --- a/app/render/backend/CMakeLists.txt +++ b/app/render/backend/CMakeLists.txt @@ -27,6 +27,8 @@ set(OLIVE_SOURCES render/backend/audiorenderbackend.cpp render/backend/videorenderbackend.h render/backend/videorenderbackend.cpp + render/backend/videorenderworker.h + render/backend/videorenderworker.cpp render/backend/decodercache.h render/backend/decodercache.cpp diff --git a/app/render/backend/opengl/openglbackend.cpp b/app/render/backend/opengl/openglbackend.cpp index bf540964b..f905b95ef 100644 --- a/app/render/backend/opengl/openglbackend.cpp +++ b/app/render/backend/opengl/openglbackend.cpp @@ -40,7 +40,7 @@ bool OpenGLBackend::InitInternal() // Connect to it connect(processor, SIGNAL(RequestSibling(NodeDependency)), this, SLOT(ThreadRequestedSibling(NodeDependency))); - connect(processor, SIGNAL(CompletedFrame(NodeDependency)), this, SLOT(CompletedFrame(NodeDependency))); + connect(processor, SIGNAL(CompletedFrame(NodeDependency)), this, SLOT(ThreadCompletedFrame(NodeDependency))); // Finally, we can move it to its own thread processor->moveToThread(thread); @@ -66,16 +66,21 @@ bool OpenGLBackend::InitInternal() void OpenGLBackend::GenerateFrame(const rational &time) { + qDebug() << "Compiled state:" << compiled_; if (!compiled_) { Compile(); } NodeDependency dep = NodeDependency(viewer_node()->texture_input()->get_connected_output(), time, time); - QMetaObject::invokeMethod(processors_.first(), - "Render", - Qt::QueuedConnection, - Q_ARG(NodeDependency, dep)); + foreach (OpenGLWorker* worker, processors_) { + if (worker->IsAvailable() || worker == processors_.last()) { + QMetaObject::invokeMethod(worker, + "Render", + Qt::QueuedConnection, + Q_ARG(NodeDependency, dep)); + } + } } void OpenGLBackend::CloseInternal() @@ -196,66 +201,28 @@ bool OpenGLBackend::TraverseCompiling(Node *n) return true; } -#include -#include -#include "common/define.h" -#include "render/pixelservice.h" -void OpenGLBackend::CompletedFrame(NodeDependency path) +void OpenGLBackend::ThreadCompletedFrame(NodeDependency path) { caching_ = false; OpenGLTexturePtr texture = path.node()->get_cached_value(path.range()).value(); qDebug() << "Retrieved texture for time" << path.in(); - qDebug() << "Texture is" << texture.get(); - - if (texture == nullptr) { - QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions(); - QOpenGLExtraFunctions* xf = QOpenGLContext::currentContext()->extraFunctions(); - - PixelFormatInfo format_info = PixelService::GetPixelFormatInfo(params().format()); - QVector data_buffer(PixelService::GetBufferSize(params().format(), params().width(), params().height())); - qDebug() << "Created buffer of size" << data_buffer.size(); - - // Set up OIIO::ImageSpec for compressing cached images on disk - OIIO::ImageSpec spec(params().width(), params().height(), kRGBAChannels, format_info.oiio_desc); - spec.attribute("compression", "dwaa:200"); - - f->glBindFramebuffer(GL_READ_FRAMEBUFFER, copy_buffer_.buffer()); - - xf->glFramebufferTexture2D(GL_READ_FRAMEBUFFER, - GL_COLOR_ATTACHMENT0, - GL_TEXTURE_2D, - texture->texture(), - 0); - - f->glReadPixels(0, - 0, - texture->width(), - texture->height(), - format_info.pixel_format, - format_info.gl_pixel_type, - data_buffer.data()); - - xf->glFramebufferTexture2D(GL_READ_FRAMEBUFFER, - GL_COLOR_ATTACHMENT0, - GL_TEXTURE_2D, - 0, - 0); - - f->glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); - + if (texture != nullptr) { QString cache_fn = CachePathName(QStringLiteral("%1-%2").arg(QString::number(path.in().numerator()), QString::number(path.in().denominator())).toLatin1()); - std::string working_fn_std = cache_fn.toStdString(); - std::unique_ptr out = OIIO::ImageOutput::create(working_fn_std); - - if (out) { - out->open(working_fn_std, spec); - out->write_image(format_info.oiio_desc, data_buffer.data()); - out->close(); - } else { - qWarning() << "Failed to open output file:" << cache_fn; + // Find an available worker to download this texture + foreach (OpenGLWorker* worker, processors_) { + // Check if one is available, but worst case if none of them are available, just queue it on the last worker since + // it's the least likely to get work + if (worker->IsAvailable() || worker == processors_.last()) { + QMetaObject::invokeMethod(worker, + "Download", + Q_ARG(NodeDependency, path), + Q_ARG(QVariant, QVariant::fromValue(texture)), + Q_ARG(QString, cache_fn)); + break; + } } } @@ -273,7 +240,6 @@ void OpenGLBackend::ThreadCallback(OpenGLTexturePtr texture, const rational& tim // We received a texture, time to start downloading it QString fn = CachePathName(hash); - qDebug() << "INSERT DOWNLOAD CODE!"; /*download_threads_[last_download_thread_%download_threads_.size()]->Queue(texture, fn, hash);*/ @@ -317,12 +283,10 @@ void OpenGLBackend::ThreadCallback(OpenGLTexturePtr texture, const rational& tim void OpenGLBackend::ThreadRequestedSibling(NodeDependency dep) { - Q_UNUSED(dep) - // Try to queue another thread to run this dep in advance - for (int i=1;iIsAvailable()) { - QMetaObject::invokeMethod(processors_.at(i), + foreach (OpenGLWorker* worker, processors_) { + if (worker->IsAvailable()) { + QMetaObject::invokeMethod(worker, "RenderAsSibling", Qt::QueuedConnection, Q_ARG(NodeDependency, dep)); diff --git a/app/render/backend/opengl/openglbackend.h b/app/render/backend/opengl/openglbackend.h index 1ba85187a..963d39350 100644 --- a/app/render/backend/opengl/openglbackend.h +++ b/app/render/backend/opengl/openglbackend.h @@ -45,13 +45,13 @@ private: bool compiled_; private slots: - void CompletedFrame(NodeDependency path); + void ThreadCompletedFrame(NodeDependency path); + void ThreadRequestedSibling(NodeDependency dep); void ThreadCallback(OpenGLTexturePtr texture, const rational& time, const QByteArray& hash); - void ThreadRequestedSibling(NodeDependency dep); void ThreadSkippedFrame(const rational &time, const QByteArray &hash); diff --git a/app/render/backend/opengl/openglshadercache.cpp b/app/render/backend/opengl/openglshadercache.cpp index 724c1cadc..92df2171f 100644 --- a/app/render/backend/opengl/openglshadercache.cpp +++ b/app/render/backend/opengl/openglshadercache.cpp @@ -27,3 +27,8 @@ OpenGLShaderPtr OpenGLShaderCache::GetShader(NodeOutput *output) { return compiled_nodes_.value(GenerateShaderID(output)); } + +bool OpenGLShaderCache::HasShader(NodeOutput *output) +{ + return compiled_nodes_.contains(GenerateShaderID(output)); +} diff --git a/app/render/backend/opengl/openglshadercache.h b/app/render/backend/opengl/openglshadercache.h index 16408db27..c4bcd18fc 100644 --- a/app/render/backend/opengl/openglshadercache.h +++ b/app/render/backend/opengl/openglshadercache.h @@ -20,6 +20,8 @@ public: OpenGLShaderPtr GetShader(NodeOutput* output); + bool HasShader(NodeOutput* output); + private: QString GenerateShaderID(NodeOutput* output); diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index 259d98f0d..701347461 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -1,19 +1,15 @@ #include "openglworker.h" -#include - #include "functions.h" -#include "node/block/block.h" #include "node/node.h" +#include "render/pixelservice.h" OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, QObject *parent) : - QObject(parent), + VideoRenderWorker(decoder_cache, parent), share_ctx_(share_ctx), ctx_(nullptr), functions_(nullptr), - shader_cache_(shader_cache), - decoder_cache_(decoder_cache), - working_(0) + shader_cache_(shader_cache) { surface_.create(); } @@ -23,17 +19,7 @@ OpenGLWorker::~OpenGLWorker() surface_.destroy(); } -bool OpenGLWorker::IsStarted() -{ - return ctx_ != nullptr; -} - -void OpenGLWorker::SetParameters(const VideoRenderingParams &video_params) -{ - video_params_ = video_params; -} - -void OpenGLWorker::Init() +bool OpenGLWorker::InitInternal() { // Create context object ctx_ = new QOpenGLContext(); @@ -44,8 +30,7 @@ void OpenGLWorker::Init() // Create OpenGL context (automatically destroys any existing if there is one) if (!ctx_->create()) { qWarning() << "Failed to create OpenGL context in thread" << thread(); - Close(); - return; + return false; } ctx_->moveToThread(this->thread()); @@ -54,14 +39,26 @@ void OpenGLWorker::Init() // The rest of the initialization needs to occur in the other thread, so we signal for it to start QMetaObject::invokeMethod(this, "FinishInit", Qt::QueuedConnection); + + return true; } -bool OpenGLWorker::IsAvailable() +QVariant OpenGLWorker::FrameToTexture(FramePtr frame) { - return (working_ == 0); + OpenGLTexturePtr footage_tex = std::make_shared(); + footage_tex->Create(ctx_, frame); + + // FIXME: Alpha association and color management + + return QVariant::fromValue(footage_tex); } -void OpenGLWorker::Close() +bool OpenGLWorker::OutputIsShader(NodeOutput* output) +{ + return shader_cache_->HasShader(output); +} + +void OpenGLWorker::CloseInternal() { buffer_.Destroy(); @@ -69,124 +66,26 @@ void OpenGLWorker::Close() delete ctx_; } -void OpenGLWorker::Render(NodeDependency path) +void OpenGLWorker::ParametersChangedEvent() { - NodeOutput* output = path.node(); - Node* node = output->parent(); - - QList all_nodes_in_graph; - all_nodes_in_graph.append(node); - all_nodes_in_graph.append(node->GetDependencies()); - - // Lock all Nodes to prevent UI changes during this render - foreach (Node* dep, all_nodes_in_graph) { - dep->LockUserInput(); - } - - // Start traversing graph - RenderAsSibling(path); - - // Start OpenGL flushing now while we do clean up work on the CPU - functions_->glFlush(); - - // Unlock all Nodes so changes can be made again - foreach (Node* dep, all_nodes_in_graph) { - dep->UnlockUserInput(); - } - - // Now we need the texture done so we call glFinish() - functions_->glFinish(); - - emit CompletedFrame(path); -} - -void OpenGLWorker::UpdateViewportFromParams() -{ - if (functions_ != nullptr && video_params_.is_valid()) { - functions_->glViewport(0, 0, video_params_.effective_width(), video_params_.effective_height()); + if (functions_ != nullptr && video_params().is_valid()) { + functions_->glViewport(0, 0, video_params().effective_width(), video_params().effective_height()); } } -Node *OpenGLWorker::ValidateBlock(Node *n, const rational& time) +QVariant OpenGLWorker::RunNodeAsShader(NodeOutput *out) { - if (n->IsBlock()) { - Block* block = static_cast(n); + OpenGLShaderPtr shader = shader_cache_->GetShader(out); + Node* node = out->parent(); - while (block->in() > time && block != nullptr) { - // This Block is too late, find an earlier one - block = block->previous(); - } + // Create the output texture + OpenGLTexturePtr output = std::make_shared(); + output->Create(ctx_, video_params().effective_width(), video_params().effective_height(), video_params().format()); - while (block->out() <= time && block != nullptr) { - // This block is too early, find a later one - block = block->next(); - } + buffer_.Attach(output); - // By this point, we should have the correct Block or nullptr if there's no Block here - return block; - } + buffer_.Bind(); - return n; -} - -QList OpenGLWorker::ProcessNodeInputsForTime(Node *n, const TimeRange &time) -{ - QList connected_inputs; - - // Now we need to gather information about this Node's inputs - foreach (NodeParam* param, n->parameters()) { - // Check if this parameter is an input and if the Node is dependent on it - if (param->type() == NodeParam::kInput) { - NodeInput* input = static_cast(param); - - if (input->dependent()) { - // If we're here, this input is necessary and we need to acquire the value for this Node - if (input->IsConnected()) { - // If it's connected to something, we need to retrieve that output at some point - connected_inputs.append(input); - } else { - // If it isn't connected, it'll have the value we need inside it. We just need to store it for the node. - input->set_stored_value(input->get_value_at_time(n->InputTimeAdjustment(input, time).in())); - } - - // Special types like FOOTAGE require extra work from us (to decrease node complexity dealing with decoders) - if (input->data_type() == NodeParam::kFootage) { - input->set_stored_value(0); - - // Access a map of Node inputs and decoder instances and retrieve a frame! - StreamPtr stream = input->get_value_at_time(0).value(); - DecoderPtr decoder = decoder_cache_->GetDecoder(stream.get()); - - if (decoder == nullptr && stream != nullptr) { - // Init decoder - decoder = Decoder::CreateFromID(stream->footage()->decoder()); - decoder->set_stream(stream); - decoder_cache_->AddDecoder(stream.get(), decoder); - } - - // By this point we should definitely have a decoder, and if we don't something's gone terribly wrong - if (decoder != nullptr) { - FramePtr frame = decoder->Retrieve(time.in()); - - if (frame != nullptr) { - OpenGLTexturePtr footage_tex = std::make_shared(); - footage_tex->Create(ctx_, frame, OpenGLTexture::kDoubleBuffer); - - // FIXME: Alpha association and color management - - input->set_stored_value(QVariant::fromValue(footage_tex)); - } - } - } - } - } - } - - return connected_inputs; -} - -OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shader) -{ shader->bind(); unsigned int input_texture_count = 0; @@ -229,11 +128,14 @@ OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shade case NodeInput::kFootage: { OpenGLTexturePtr texture = input->value().value(); + qDebug() << " Binding" << texture->texture() << "from" << input << "to GL_TEXTURE" << input_texture_count; + functions_->glActiveTexture(GL_TEXTURE0 + input_texture_count); functions_->glBindTexture(GL_TEXTURE_2D, texture->texture()); // Set value to bound texture shader->setUniformValue(variable_location, input_texture_count); + qDebug() << " Setting" << input->id() << "to" << input_texture_count; input_texture_count++; break; @@ -253,20 +155,9 @@ OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shade } } - // Create the output texture - OpenGLTexturePtr output = std::make_shared(); - output->Create(ctx_, video_params_.width(), video_params_.height(), video_params_.format()); - - buffer_.Attach(output); - - buffer_.Bind(); - + qDebug() << " Blitting with shader!"; olive::gl::Blit(shader); - buffer_.Release(); - - buffer_.Detach(); - // Release any textures we bound before while (input_texture_count > 0) { input_texture_count--; @@ -278,7 +169,35 @@ OpenGLTexturePtr OpenGLWorker::RunNodeAsShader(Node* node, OpenGLShaderPtr shade shader->release(); - return output; + buffer_.Release(); + + buffer_.Detach(); + + functions_->glFinish(); + + return QVariant::fromValue(output); +} + +void OpenGLWorker::TextureToBuffer(const QVariant &tex_in, QByteArray &buffer) +{ + OpenGLTexturePtr texture = tex_in.value(); + + PixelFormatInfo format_info = PixelService::GetPixelFormatInfo(video_params().format()); + + QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions(); + buffer_.Attach(texture); + f->glBindFramebuffer(GL_READ_FRAMEBUFFER, buffer_.buffer()); + + f->glReadPixels(0, + 0, + texture->width(), + texture->height(), + format_info.pixel_format, + format_info.gl_pixel_type, + buffer.data()); + + f->glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); + buffer_.Detach(); } void OpenGLWorker::FinishInit() @@ -294,133 +213,7 @@ void OpenGLWorker::FinishInit() // Set up OpenGL parameters as necessary functions_->glEnable(GL_BLEND); - UpdateViewportFromParams(); + ParametersChangedEvent(); buffer_.Create(ctx_); - - //qDebug() << "Context in" << ctx_->thread() << "successfully finished"; -} - -void OpenGLWorker::RenderAsSibling(NodeDependency dep) -{ - NodeOutput* output = dep.node(); - Node* original_node = output->parent(); - Node* node; - rational time = dep.in(); - QList connected_inputs; - OpenGLShaderPtr shader; - - // Set working state - working_++; - - original_node->LockProcessing(); - - // Firstly we check if this node is a "Block", if it is that means it's part of a linked list of mutually exclusive - // nodes based on time and we might need to locate which Block to attach to - if ((node = ValidateBlock(original_node, time)) == nullptr) { - // ValidateBlock() may have returned nullptr if there was no Block found at this time so no texture to return - output->cache_value(dep.range(), 0); - - original_node->UnlockProcessing(); - goto end_render; - } - - if (original_node != node) { - // Ensure output is the output matching the node as it may have changed - output = static_cast(node->GetParameterWithID(output->id())); - - // Switch locks - original_node->UnlockProcessing(); - node->LockProcessing(); - } - - // Check if the output already has a value for this time - if (output->has_cached_value(dep.range())) { - // If so, we don't need to do anything, we can just send this value and exit here - dep.node()->cache_value(dep.range(), output->get_cached_value(dep.range())); - - node->UnlockProcessing(); - goto end_render; - } - - // We need to run the Node's code to get the correct value for this time - - connected_inputs = ProcessNodeInputsForTime(node, dep.range()); - - // For each connected input, we need to acquire the value from another node - while (!connected_inputs.isEmpty()) { - - // Remove any inputs from the list that we have valid cached values for already - for (int i=0;iget_connected_output(); - TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); - - if (connected_output->has_cached_value(input_time)) { - // This output already has this value, no need to process it again - input->set_stored_value(connected_output->get_cached_value(input_time)); - connected_inputs.removeAt(i); - i--; - } - } - - // For every connected input except the first, we'll request another Node to do it - int input_for_this_thread = -1; - - for (int i=0;iget_connected_node()->IsProcessingLocked()) { - if (input_for_this_thread == -1) { - // Store this later since we can process it on this thread as we wait for other threads - input_for_this_thread = i; - } else { - TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); - - emit RequestSibling(NodeDependency(input->get_connected_output(), - input_time)); - } - } - } - - if (input_for_this_thread > -1) { - // In the mean time, this thread can go off to do the first parameter - NodeInput* input = connected_inputs.at(input_for_this_thread); - TimeRange input_range = node->InputTimeAdjustment(input, dep.range()); - RenderAsSibling(NodeDependency(input->get_connected_output(), - input_range)); - input->set_stored_value(input->get_connected_output()->get_cached_value(input_range)); - connected_inputs.removeAt(input_for_this_thread); - } else { - // Nothing for this thread to do. We'll wait 0.5 sec and check again for other nodes - // FIXME: It would be nicer if this thread could do other nodes during this time - QThread::msleep(500); - } - } - - // By this point, the node should have all the inputs it needs to render correctly - - // Check if we have a shader for this output - shader = shader_cache_->GetShader(output); - - if (shader != nullptr) { - // Run code - OpenGLTexturePtr texture = RunNodeAsShader(node, shader); - - output->cache_value(dep.range(), QVariant::fromValue(texture)); - } else { - // Generate the value as expected - QVariant value = node->Value(output); - - // Place the value into the output - output->cache_value(dep.range(), value); - } - - // We're done! - node->UnlockProcessing(); - -end_render: - // End this working state - working_--; } diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index 75dc0ec42..85e8ec900 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -1,29 +1,21 @@ #ifndef OPENGLPROCESSOR_H #define OPENGLPROCESSOR_H -#include #include #include -#include "../decodercache.h" -#include "node/dependency.h" +#include "../videorenderworker.h" #include "openglframebuffer.h" #include "openglshadercache.h" -#include "render/videoparams.h" -class OpenGLWorker : public QObject { +class OpenGLWorker : public VideoRenderWorker { Q_OBJECT public: OpenGLWorker(QOpenGLContext* share_ctx, OpenGLShaderCache* shader_cache, DecoderCache* decoder_cache, QObject* parent = nullptr); virtual ~OpenGLWorker() override; - Q_DISABLE_COPY_MOVE(OpenGLWorker) - - bool IsStarted(); - - void SetParameters(const VideoRenderingParams& video_params); - +protected: /** * @brief Initialize OpenGL instance in whatever thread this object is a part of * @@ -45,35 +37,21 @@ public: * the main thread from using the context during initialization and preventing more than one shared context being made * at the same time (which may or may not actually make a difference). */ - void Init(); + virtual bool InitInternal() override; - bool IsAvailable(); + virtual void CloseInternal() override; -public slots: - void Close(); + virtual QVariant FrameToTexture(FramePtr frame) override; - void Render(NodeDependency path); + virtual bool OutputIsShader(NodeOutput *output) override; - void RenderAsSibling(NodeDependency dep); + virtual QVariant RunNodeAsShader(NodeOutput *output) override; - //void Download(); + virtual void TextureToBuffer(const QVariant& texture, QByteArray& buffer) override; -signals: - void RequestSibling(NodeDependency path); - - void CompletedFrame(NodeDependency path); + virtual void ParametersChangedEvent() override; private: - void ProcessNode(); - - void UpdateViewportFromParams(); - - Node* ValidateBlock(Node* n, const rational& time); - - QList ProcessNodeInputsForTime(Node* n, const TimeRange& time); - - OpenGLTexturePtr RunNodeAsShader(Node *node, OpenGLShaderPtr shader); - QOpenGLContext* share_ctx_; QOpenGLContext* ctx_; @@ -83,14 +61,8 @@ private: OpenGLFramebuffer buffer_; - VideoRenderingParams video_params_; - OpenGLShaderCache* shader_cache_; - DecoderCache* decoder_cache_; - - QAtomicInt working_; - private slots: void FinishInit(); diff --git a/app/render/backend/videorenderbackend.h b/app/render/backend/videorenderbackend.h index c4e9eeb09..03711501e 100644 --- a/app/render/backend/videorenderbackend.h +++ b/app/render/backend/videorenderbackend.h @@ -148,9 +148,6 @@ private: QByteArray cache_frame_load_buffer_; - /*QVector download_threads_; - int last_download_thread_;*/ - private slots: diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index 469c6ceda..b3f261331 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -1,6 +1,397 @@ #include "videorenderworker.h" -VideoRenderWorker::VideoRenderWorker() +#include + +#include "common/define.h" +#include "node/block/block.h" +#include "node/node.h" +#include "render/pixelservice.h" + +VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, QObject *parent) : + QObject(parent), + decoder_cache_(decoder_cache), + started_(false) { } + +bool VideoRenderWorker::IsAvailable() +{ + return (working_ == 0); +} + +void VideoRenderWorker::Close() +{ + CloseInternal(); + + started_ = false; +} + +const VideoRenderingParams &VideoRenderWorker::video_params() +{ + return video_params_; +} + +DecoderCache *VideoRenderWorker::decoder_cache() +{ + return decoder_cache_; +} + +void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, const rational& time) +{ + // Resolve BlockList + if (n->IsBlock() && (n = ValidateBlock(n, time)) == nullptr) { + return; + } + + // Add this Node's ID + hash->addData(n->id().toUtf8()); + + foreach (NodeParam* param, n->parameters()) { + // For each input, try to hash its value + if (param->type() == NodeParam::kInput) { + NodeInput* input = static_cast(param); + + // Get time adjustment + TimeRange range = n->InputTimeAdjustment(input, TimeRange(time, time)); + + // For a single frame, we only care about one of the times + rational input_time = range.in(); + + if (input->IsConnected()) { + // Traverse down this edge + HashNodeRecursively(hash, input->get_connected_node(), input_time); + } else { + // Grab the value at this time + QVariant value = input->get_value_at_time(input_time); + hash->addData(NodeParam::ValueToBytes(input->data_type(), value)); + } + + // We have one exception for FOOTAGE types, since we resolve the footage into a frame in the renderer + if (input->data_type() == NodeParam::kFootage) { + StreamPtr stream = ResolveStreamFromInput(input); + DecoderPtr decoder = ResolveDecoderFromInput(input); + + if (decoder != nullptr) { + // Add footage details to hash + + // Footage filename + hash->addData(stream->footage()->filename().toUtf8()); + + // Footage last modified date + hash->addData(stream->footage()->timestamp().toString().toUtf8()); + + // Footage stream + hash->addData(QString::number(stream->index()).toUtf8()); + + // Footage timestamp + hash->addData(QString::number(decoder->GetTimestampFromTime(time)).toUtf8()); + + // FIXME: Add colorspace and alpha assoc + } + } + } + } +} + +StreamPtr VideoRenderWorker::ResolveStreamFromInput(NodeInput *input) +{ + return input->get_value_at_time(0).value(); +} + +DecoderPtr VideoRenderWorker::ResolveDecoderFromInput(NodeInput *input) +{ + // Access a map of Node inputs and decoder instances and retrieve a frame! + StreamPtr stream = ResolveStreamFromInput(input); + DecoderPtr decoder = decoder_cache()->GetDecoder(stream.get()); + + if (decoder == nullptr && stream != nullptr) { + // Init decoder + decoder = Decoder::CreateFromID(stream->footage()->decoder()); + decoder->set_stream(stream); + decoder_cache()->AddDecoder(stream.get(), decoder); + } + + return decoder; +} + +Node *VideoRenderWorker::ValidateBlock(Node *n, const rational& time) +{ + if (n->IsBlock()) { + Block* block = static_cast(n); + + while (block != nullptr && block->in() > time) { + // This Block is too late, find an earlier one + block = block->previous(); + } + + while (block != nullptr && block->out() <= time) { + // This block is too early, find a later one + block = block->next(); + } + + // By this point, we should have the correct Block or nullptr if there's no Block here + return block; + } + + return n; +} + +void VideoRenderWorker::SetParameters(const VideoRenderingParams &video_params) +{ + video_params_ = video_params; + + ParametersChangedEvent(); +} + +bool VideoRenderWorker::Init() +{ + if (started_) { + return true; + } + + started_ = InitInternal(); + + if (started_) { + download_buffer_.resize(PixelService::GetBufferSize(video_params().format(), video_params().effective_width(), video_params().effective_height())); + } else { + Close(); + } + + return started_; +} + +bool VideoRenderWorker::IsStarted() +{ + return started_; +} + +void VideoRenderWorker::Render(NodeDependency path) +{ + NodeOutput* output = path.node(); + Node* node = output->parent(); + + QList all_nodes_in_graph; + all_nodes_in_graph.append(node); + all_nodes_in_graph.append(node->GetDependencies()); + + // Lock all Nodes to prevent UI changes during this render + foreach (Node* dep, all_nodes_in_graph) { + dep->LockUserInput(); + } + + // Start traversing graph + RenderAsSibling(path); + + // Unlock all Nodes so changes can be made again + foreach (Node* dep, all_nodes_in_graph) { + dep->UnlockUserInput(); + } + + emit CompletedFrame(path); +} + +QList VideoRenderWorker::ProcessNodeInputsForTime(Node *n, const TimeRange &time) +{ + QList connected_inputs; + + // Now we need to gather information about this Node's inputs + foreach (NodeParam* param, n->parameters()) { + // Check if this parameter is an input and if the Node is dependent on it + if (param->type() == NodeParam::kInput) { + NodeInput* input = static_cast(param); + + if (input->dependent()) { + // If we're here, this input is necessary and we need to acquire the value for this Node + if (input->IsConnected()) { + // If it's connected to something, we need to retrieve that output at some point + connected_inputs.append(input); + } else { + // If it isn't connected, it'll have the value we need inside it. We just need to store it for the node. + input->set_stored_value(input->get_value_at_time(n->InputTimeAdjustment(input, time).in())); + } + + // Special types like FOOTAGE require extra work from us (to decrease node complexity dealing with decoders) + if (input->data_type() == NodeParam::kFootage) { + input->set_stored_value(0); + + DecoderPtr decoder = ResolveDecoderFromInput(input); + + // By this point we should definitely have a decoder, and if we don't something's gone terribly wrong + if (decoder != nullptr) { + FramePtr frame = decoder->Retrieve(time.in()); + + if (frame != nullptr) { + QVariant value = FrameToTexture(frame); + + input->set_stored_value(value); + + qDebug() << "Placing texture" << value << "into input" << input; + } + } + } + } + } + } + + return connected_inputs; +} + +void VideoRenderWorker::RenderAsSibling(NodeDependency dep) +{ + NodeOutput* output = dep.node(); + Node* original_node = output->parent(); + Node* node; + rational time = dep.in(); + QList connected_inputs; + QVariant value; + + // Set working state + working_++; + + qDebug() << "Processing" << original_node->id() << original_node; + + original_node->LockProcessing(); + + // Firstly we check if this node is a "Block", if it is that means it's part of a linked list of mutually exclusive + // nodes based on time and we might need to locate which Block to attach to + if ((node = ValidateBlock(original_node, time)) == nullptr) { + // ValidateBlock() may have returned nullptr if there was no Block found at this time so no texture to return + output->cache_value(dep.range(), 0); + + original_node->UnlockProcessing(); + goto end_render; + } + + if (original_node != node) { + // Ensure output is the output matching the node as it may have changed + output = static_cast(node->GetParameterWithID(output->id())); + + // Switch locks + original_node->UnlockProcessing(); + node->LockProcessing(); + + qDebug() << "Deftly switched from" << original_node->id() << original_node << "to" << node->id() << node; + } + + // Check if the output already has a value for this time + if (output->has_cached_value(dep.range())) { + // If so, we don't need to do anything, we can just send this value and exit here + dep.node()->cache_value(dep.range(), output->get_cached_value(dep.range())); + + qDebug() << "Found a cached value on" << node->id() << output->id() << "at" << dep.range().in() << "-" << dep.range().out(); + + node->UnlockProcessing(); + goto end_render; + } + + // We need to run the Node's code to get the correct value for this time + + connected_inputs = ProcessNodeInputsForTime(node, dep.range()); + + // For each connected input, we need to acquire the value from another node + while (!connected_inputs.isEmpty()) { + + // Remove any inputs from the list that we have valid cached values for already + for (int i=0;iget_connected_output(); + TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); + + if (connected_output->has_cached_value(input_time)) { + // This output already has this value, no need to process it again + input->set_stored_value(connected_output->get_cached_value(input_time)); + connected_inputs.removeAt(i); + i--; + } + } + + // For every connected input except the first, we'll request another Node to do it + int input_for_this_thread = -1; + + for (int i=0;iget_connected_node()->IsProcessingLocked()) { + if (input_for_this_thread == -1) { + // Store this later since we can process it on this thread as we wait for other threads + input_for_this_thread = i; + } else { + TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); + + emit RequestSibling(NodeDependency(input->get_connected_output(), + input_time)); + } + } + } + + if (input_for_this_thread > -1) { + // In the mean time, this thread can go off to do the first parameter + NodeInput* input = connected_inputs.at(input_for_this_thread); + TimeRange input_range = node->InputTimeAdjustment(input, dep.range()); + RenderAsSibling(NodeDependency(input->get_connected_output(), + input_range)); + input->set_stored_value(input->get_connected_output()->get_cached_value(input_range)); + connected_inputs.removeAt(input_for_this_thread); + } else { + // Nothing for this thread to do. We'll wait 0.5 sec and check again for other nodes + // FIXME: It would be nicer if this thread could do other nodes during this time + QThread::msleep(500); + } + } + + // By this point, the node should have all the inputs it needs to render correctly + + // Check if we have a shader for this output + if (OutputIsShader(output)) { + // Run code + value = RunNodeAsShader(output); + } else { + // Generate the value as expected + value = node->Value(output); + } + + // Place the value into the output + output->cache_value(dep.range(), value); + dep.node()->cache_value(dep.range(), value); + + // We're done! + node->UnlockProcessing(); + +end_render: + // End this working state + working_--; +} + + + +void VideoRenderWorker::Download(NodeDependency dep, QVariant texture, QString filename) +{ + working_++; + + PixelFormatInfo format_info = PixelService::GetPixelFormatInfo(video_params().format()); + + // Set up OIIO::ImageSpec for compressing cached images on disk + OIIO::ImageSpec spec(video_params().effective_width(), video_params().effective_height(), kRGBAChannels, format_info.oiio_desc); + spec.attribute("compression", "dwaa:200"); + + TextureToBuffer(texture, download_buffer_); + + std::string working_fn_std = filename.toStdString(); + + std::unique_ptr out = OIIO::ImageOutput::create(working_fn_std); + + if (out) { + qDebug() << "Saving to" << filename; + out->open(working_fn_std, spec); + out->write_image(format_info.oiio_desc, download_buffer_.data()); + out->close(); + + emit CompletedDownload(dep); + } else { + qWarning() << "Failed to open output file:" << filename; + } + + working_--; +} diff --git a/app/render/backend/videorenderworker.h b/app/render/backend/videorenderworker.h index dc017d07f..ed3486731 100644 --- a/app/render/backend/videorenderworker.h +++ b/app/render/backend/videorenderworker.h @@ -1,11 +1,87 @@ #ifndef VIDEORENDERWORKER_H #define VIDEORENDERWORKER_H +#include +#include -class VideoRenderWorker -{ +#include "decodercache.h" +#include "node/dependency.h" +#include "render/videoparams.h" + +class VideoRenderWorker : public QObject { + Q_OBJECT public: - VideoRenderWorker(); + VideoRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); + + Q_DISABLE_COPY_MOVE(VideoRenderWorker) + + bool IsStarted(); + + void SetParameters(const VideoRenderingParams& video_params); + + virtual bool Init(); + + bool IsAvailable(); + +public slots: + void Close(); + + void Render(NodeDependency path); + + void RenderAsSibling(NodeDependency dep); + + void Download(NodeDependency dep, QVariant texture, QString filename); + +signals: + void RequestSibling(NodeDependency path); + + void CompletedFrame(NodeDependency path); + + void CompletedDownload(NodeDependency path); + +protected: + virtual bool InitInternal() = 0; + + virtual void CloseInternal() = 0; + + virtual QVariant FrameToTexture(FramePtr frame) = 0; + + const VideoRenderingParams& video_params(); + + DecoderCache* decoder_cache(); + + Node* ValidateBlock(Node* n, const rational& time); + + virtual void ParametersChangedEvent(){} + + virtual bool OutputIsShader(NodeOutput *output) = 0; + + virtual QVariant RunNodeAsShader(NodeOutput *output) = 0; + + virtual void TextureToBuffer(const QVariant& texture, QByteArray& buffer) = 0; + +private: + void ProcessNode(); + + StreamPtr ResolveStreamFromInput(NodeInput* input); + DecoderPtr ResolveDecoderFromInput(NodeInput* input); + + QList ProcessNodeInputsForTime(Node* n, const TimeRange& time); + + void HashNodeRecursively(QCryptographicHash* hash, Node *n, const rational &time); + + VideoRenderingParams video_params_; + + DecoderCache* decoder_cache_; + + QAtomicInt working_; + + QByteArray download_buffer_; + + bool started_; + +private slots: + }; #endif // VIDEORENDERWORKER_H diff --git a/app/widget/timelinewidget/tool/import.cpp b/app/widget/timelinewidget/tool/import.cpp index 5eb9a9c92..5c5656e22 100644 --- a/app/widget/timelinewidget/tool/import.cpp +++ b/app/widget/timelinewidget/tool/import.cpp @@ -218,14 +218,14 @@ void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event) ClipBlock* clip = new ClipBlock(); VideoInput* media = new VideoInput(); - TransformDistort* transform = new TransformDistort(); - OpacityNode* opacity = new OpacityNode(); + //TransformDistort* transform = new TransformDistort(); + //OpacityNode* opacity = new OpacityNode(); // Set parents to node_memory_manager in case no TimelineOutput receives this signal clip->setParent(&node_memory_manager); media->setParent(&node_memory_manager); - transform->setParent(&node_memory_manager); - opacity->setParent(&node_memory_manager); + //transform->setParent(&node_memory_manager); + //opacity->setParent(&node_memory_manager); StreamPtr footage_stream = ghost->data(TimelineViewGhostItem::kAttachedFootage).value(); media->SetFootage(footage_stream); @@ -233,10 +233,10 @@ void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event) clip->set_length(ghost->Length()); clip->set_block_name(footage_stream->footage()->name()); - NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input()); - NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input()); - NodeParam::ConnectEdge(transform->matrix_output(), media->matrix_input()); - + //NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input()); + //NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input()); + //NodeParam::ConnectEdge(transform->matrix_output(), media->matrix_input()); + NodeParam::ConnectEdge(media->texture_output(), clip->texture_input()); if (event->GetModifiers() & Qt::ControlModifier) { //emit parent()->RequestInsertBlockAtTime(clip, ghost->GetAdjustedIn());