From 7100d00060f20046155d44655a957c815b4031e1 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sun, 29 Mar 2020 03:11:30 +1100 Subject: [PATCH] nodes: made changes to the node system to allow for context sensitive shader code Primarily implemented for the math node so a new shader can be generated and compiled based on what inputs the node is receiving. --- app/node/audio/pan/pan.cpp | 5 ++ app/node/audio/pan/pan.h | 1 + app/node/audio/volume/volume.cpp | 5 ++ app/node/audio/volume/volume.h | 1 + .../block/transition/externaltransition.cpp | 12 +-- .../block/transition/externaltransition.h | 10 +-- app/node/external.cpp | 12 +-- app/node/external.h | 10 +-- app/node/factory.cpp | 3 + app/node/factory.h | 1 + app/node/input/media/video/video.cpp | 8 +- app/node/input/media/video/video.h | 6 +- app/node/math/math.cpp | 90 ++++++++++++++++++- app/node/math/math.h | 19 ++++ app/node/node.cpp | 21 +++-- app/node/node.h | 26 ++++-- app/node/value.cpp | 13 +++ app/node/value.h | 1 + app/render/backend/audio/audioworker.cpp | 2 +- app/render/backend/opengl/openglproxy.cpp | 19 ++-- 20 files changed, 209 insertions(+), 56 deletions(-) diff --git a/app/node/audio/pan/pan.cpp b/app/node/audio/pan/pan.cpp index f8a54f5c5..267817616 100644 --- a/app/node/audio/pan/pan.cpp +++ b/app/node/audio/pan/pan.cpp @@ -37,6 +37,11 @@ QString PanNode::Description() const return tr("Adjust the stereo panning of an audio source."); } +Node::Capabilities PanNode::GetCapabilities(const NodeValueDatabase &) const +{ + return kSampleProcessor; +} + NodeInput *PanNode::ProcessesSamplesFrom() const { return samples_input_; diff --git a/app/node/audio/pan/pan.h b/app/node/audio/pan/pan.h index 3b4a011e3..2aaf1c656 100644 --- a/app/node/audio/pan/pan.h +++ b/app/node/audio/pan/pan.h @@ -15,6 +15,7 @@ public: virtual QString Category() const override; virtual QString Description() const override; + virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; virtual NodeInput* ProcessesSamplesFrom() const override; virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override; diff --git a/app/node/audio/volume/volume.cpp b/app/node/audio/volume/volume.cpp index 4b4ce35a7..2e785e11d 100644 --- a/app/node/audio/volume/volume.cpp +++ b/app/node/audio/volume/volume.cpp @@ -36,6 +36,11 @@ QString VolumeNode::Description() const return tr("Adjusts the volume of an audio source."); } +Node::Capabilities VolumeNode::GetCapabilities(const NodeValueDatabase &) const +{ + return kSampleProcessor; +} + NodeInput *VolumeNode::ProcessesSamplesFrom() const { return samples_input_; diff --git a/app/node/audio/volume/volume.h b/app/node/audio/volume/volume.h index 133924e0c..e0f2546c1 100644 --- a/app/node/audio/volume/volume.h +++ b/app/node/audio/volume/volume.h @@ -15,6 +15,7 @@ public: virtual QString Category() const override; virtual QString Description() const override; + virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; virtual NodeInput* ProcessesSamplesFrom() const override; virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override; diff --git a/app/node/block/transition/externaltransition.cpp b/app/node/block/transition/externaltransition.cpp index 107b47f55..f4ed62290 100644 --- a/app/node/block/transition/externaltransition.cpp +++ b/app/node/block/transition/externaltransition.cpp @@ -38,27 +38,27 @@ void ExternalTransition::Retranslate() meta_.Retranslate(); } -bool ExternalTransition::IsAccelerated() const +Node::Capabilities ExternalTransition::GetCapabilities(const NodeValueDatabase &) const { - return true; + return kShader; } -QString ExternalTransition::AcceleratedCodeVertex() const +QString ExternalTransition::ShaderVertexCode(const NodeValueDatabase &) const { return meta_.vert_code(); } -QString ExternalTransition::AcceleratedCodeFragment() const +QString ExternalTransition::ShaderFragmentCode(const NodeValueDatabase&) const { return meta_.frag_code(); } -int ExternalTransition::AcceleratedCodeIterations() const +int ExternalTransition::ShaderIterations() const { return meta_.iterations(); } -NodeInput *ExternalTransition::AcceleratedCodeIterativeInput() const +NodeInput *ExternalTransition::ShaderIterativeInput() const { return meta_.iteration_input(); } diff --git a/app/node/block/transition/externaltransition.h b/app/node/block/transition/externaltransition.h index 153e04b99..c520e0ee9 100644 --- a/app/node/block/transition/externaltransition.h +++ b/app/node/block/transition/externaltransition.h @@ -19,11 +19,11 @@ public: virtual void Retranslate() override; - virtual bool IsAccelerated() const override; - virtual QString AcceleratedCodeVertex() const override; - virtual QString AcceleratedCodeFragment() const override; - virtual int AcceleratedCodeIterations() const override; - virtual NodeInput* AcceleratedCodeIterativeInput() const override; + virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; + virtual QString ShaderVertexCode(const NodeValueDatabase&) const override; + virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override; + virtual int ShaderIterations() const override; + virtual NodeInput* ShaderIterativeInput() const override; private: NodeMetaReader meta_; diff --git a/app/node/external.cpp b/app/node/external.cpp index 7aa96cdd6..bb1c946b1 100644 --- a/app/node/external.cpp +++ b/app/node/external.cpp @@ -40,27 +40,27 @@ void ExternalNode::Retranslate() meta_.Retranslate(); } -bool ExternalNode::IsAccelerated() const +Node::Capabilities ExternalNode::GetCapabilities(const NodeValueDatabase &) const { - return true; + return kShader; } -QString ExternalNode::AcceleratedCodeVertex() const +QString ExternalNode::ShaderVertexCode(const NodeValueDatabase&) const { return meta_.vert_code(); } -QString ExternalNode::AcceleratedCodeFragment() const +QString ExternalNode::ShaderFragmentCode(const NodeValueDatabase&) const { return meta_.frag_code(); } -int ExternalNode::AcceleratedCodeIterations() const +int ExternalNode::ShaderIterations() const { return meta_.iterations(); } -NodeInput *ExternalNode::AcceleratedCodeIterativeInput() const +NodeInput *ExternalNode::ShaderIterativeInput() const { return meta_.iteration_input(); } diff --git a/app/node/external.h b/app/node/external.h index d3a2b15fd..b1a4604ff 100644 --- a/app/node/external.h +++ b/app/node/external.h @@ -23,11 +23,11 @@ public: virtual void Retranslate() override; - virtual bool IsAccelerated() const override; - virtual QString AcceleratedCodeVertex() const override; - virtual QString AcceleratedCodeFragment() const override; - virtual int AcceleratedCodeIterations() const override; - virtual NodeInput* AcceleratedCodeIterativeInput() const override; + virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; + virtual QString ShaderVertexCode(const NodeValueDatabase&) const override; + virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override; + virtual int ShaderIterations() const override; + virtual NodeInput* ShaderIterativeInput() const override; private: NodeMetaReader meta_; diff --git a/app/node/factory.cpp b/app/node/factory.cpp index d0d7b12aa..b5e561e50 100644 --- a/app/node/factory.cpp +++ b/app/node/factory.cpp @@ -8,6 +8,7 @@ #include "distort/transform/transform.h" #include "input/media/video/video.h" #include "input/media/audio/audio.h" +#include "math/math.h" #include "output/track/track.h" #include "output/viewer/viewer.h" #include "external.h" @@ -135,6 +136,8 @@ Node *NodeFactory::CreateInternal(const NodeFactory::InternalID &id) return new VolumeNode(); case kAudioPanning: return new PanNode(); + case kMath: + return new MathNode(); case kInternalNodeCount: break; diff --git a/app/node/factory.h b/app/node/factory.h index aa92ad7a4..55f204e4d 100644 --- a/app/node/factory.h +++ b/app/node/factory.h @@ -19,6 +19,7 @@ public: kTrackOutput, kAudioVolume, kAudioPanning, + kMath, // Count value kInternalNodeCount diff --git a/app/node/input/media/video/video.cpp b/app/node/input/media/video/video.cpp index 8533ed25f..91ed96907 100644 --- a/app/node/input/media/video/video.cpp +++ b/app/node/input/media/video/video.cpp @@ -45,17 +45,17 @@ NodeInput *VideoInput::matrix_input() const return matrix_input_; } -bool VideoInput::IsAccelerated() const +Node::Capabilities VideoInput::GetCapabilities(const NodeValueDatabase &) const { - return true; + return kShader; } -QString VideoInput::AcceleratedCodeVertex() const +QString VideoInput::ShaderVertexCode(const NodeValueDatabase&) const { return ReadFileAsString(":/shaders/videoinput.vert"); } -QString VideoInput::AcceleratedCodeFragment() const +QString VideoInput::ShaderFragmentCode(const NodeValueDatabase&) const { return ReadFileAsString(":/shaders/videoinput.frag"); } diff --git a/app/node/input/media/video/video.h b/app/node/input/media/video/video.h index 5040f17f2..bbc946b8e 100644 --- a/app/node/input/media/video/video.h +++ b/app/node/input/media/video/video.h @@ -20,9 +20,9 @@ public: NodeInput* matrix_input() const; - virtual bool IsAccelerated() const override; - virtual QString AcceleratedCodeVertex() const override; - virtual QString AcceleratedCodeFragment() const override; + virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; + virtual QString ShaderVertexCode(const NodeValueDatabase&) const override; + virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override; virtual void Retranslate() override; diff --git a/app/node/math/math.cpp b/app/node/math/math.cpp index b31fb021b..3345df20c 100644 --- a/app/node/math/math.cpp +++ b/app/node/math/math.cpp @@ -42,7 +42,91 @@ void MathNode::Retranslate() { Node::Retranslate(); - method_in_->set_name(tr("")); - param_a_in_->set_name(tr("")); - param_b_in_->set_name(tr("")); + method_in_->set_name(tr("Method")); + param_a_in_->set_name(tr("Value")); + param_b_in_->set_name(tr("Value")); +} + +Node::Capabilities MathNode::GetCapabilities(const NodeValueDatabase &input) const +{ + if (input[param_a_in_].Has(NodeParam::kTexture) || input[param_b_in_].Has(NodeParam::kTexture)) { + return kShader; + } else if (input[param_a_in_].Has(NodeParam::kSamples) || input[param_b_in_].Has(NodeParam::kSamples)) { + return kSampleProcessor; + } else { + return kNormal; + } +} + +QString MathNode::ShaderID(const NodeValueDatabase &input) const +{ + // FIXME: Hardcoded ADD operation + QString method = QString::number(0); + + QString type_a = QString::number(GuessTypeFromTable(input[param_a_in_])); + QString type_b = QString::number(GuessTypeFromTable(input[param_b_in_])); + + return id().append(method).append(type_a).append(type_b); +} + +QString MathNode::ShaderFragmentCode(const NodeValueDatabase &input) const +{ + NodeParam::DataType type_a = GuessTypeFromTable(input[param_a_in_]); + NodeParam::DataType type_b = GuessTypeFromTable(input[param_b_in_]); + + return QStringLiteral("#version 110\n" + "\n" + "varying vec2 ove_texcoord;\n" + "\n" + "uniform %1 %3;\n" + "uniform %2 %4;\n" + "\n" + "void main(void) {\n" + " gl_FragColor = %5 + %6;\n" + "}\n").arg(GetUniformTypeFromType(type_a), + GetUniformTypeFromType(type_b), + param_a_in_->id(), + param_b_in_->id(), + GetVariableCall(param_a_in_->id(), type_a), + GetVariableCall(param_b_in_->id(), type_b)); +} + +NodeValue MathNode::InputValueFromTable(NodeInput *input, const NodeValueTable &table) const +{ + if (input->IsConnected() + && (input == param_a_in_ || input == param_b_in_) + && table.Has(NodeParam::kTexture)) { + return table.GetWithMeta(NodeParam::kTexture); + } + + return Node::InputValueFromTable(input, table); +} + +NodeParam::DataType MathNode::GuessTypeFromTable(const NodeValueTable &table) +{ + if (table.Has(NodeParam::kTexture)) { + return NodeParam::kTexture; + } else if (table.Has(NodeParam::kSamples)) { + return NodeParam::kSamples; + } else { + return NodeParam::kFloat; + } +} + +QString MathNode::GetUniformTypeFromType(const NodeParam::DataType &type) +{ + if (type == NodeParam::kTexture) { + return QStringLiteral("sampler2D"); + } + + return QStringLiteral("float"); +} + +QString MathNode::GetVariableCall(const QString &input_id, const NodeParam::DataType &type) +{ + if (type == NodeParam::kTexture) { + return QStringLiteral("texture2D(%1, ove_texcoord)").arg(input_id); + } + + return input_id; } diff --git a/app/node/math/math.h b/app/node/math/math.h index edae76cfc..ca8cdead3 100644 --- a/app/node/math/math.h +++ b/app/node/math/math.h @@ -17,7 +17,26 @@ public: virtual void Retranslate() override; + virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; + virtual QString ShaderID(const NodeValueDatabase&) const override; + virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override; + + virtual NodeValue InputValueFromTable(NodeInput* input, const NodeValueTable& table) const override; + private: + enum Operation { + kOpAdd, + kOpSubtrack, + kOpMultiply, + kOpDivide + }; + + static NodeParam::DataType GuessTypeFromTable(const NodeValueTable& table); + + static QString GetUniformTypeFromType(const NodeParam::DataType& type); + + static QString GetVariableCall(const QString& input_id, const NodeParam::DataType& type); + NodeInput* method_in_; NodeInput* param_a_in_; diff --git a/app/node/node.cpp b/app/node/node.cpp index 9d8d048de..86d6ddddf 100644 --- a/app/node/node.cpp +++ b/app/node/node.cpp @@ -434,27 +434,32 @@ QList Node::GetImmediateDependencies() const return node_list; } -bool Node::IsAccelerated() const +Node::Capabilities Node::GetCapabilities(const NodeValueDatabase &) const { - return false; + return kNormal; } -QString Node::AcceleratedCodeVertex() const +QString Node::ShaderID(const NodeValueDatabase &) const +{ + return id(); +} + +QString Node::ShaderVertexCode(const NodeValueDatabase &) const { return QString(); } -QString Node::AcceleratedCodeFragment() const +QString Node::ShaderFragmentCode(const NodeValueDatabase&) const { return QString(); } -int Node::AcceleratedCodeIterations() const +int Node::ShaderIterations() const { return 1; } -NodeInput *Node::AcceleratedCodeIterativeInput() const +NodeInput *Node::ShaderIterativeInput() const { return nullptr; } @@ -601,7 +606,7 @@ NodeOutput *Node::output() const return output_; } -QVariant Node::InputValueFromTable(NodeInput *input, const NodeValueTable &table) const +NodeValue Node::InputValueFromTable(NodeInput *input, const NodeValueTable &table) const { NodeParam::DataType find_data_type = input->data_type(); @@ -611,7 +616,7 @@ QVariant Node::InputValueFromTable(NodeInput *input, const NodeValueTable &table } // Try to get a value from it - return table.Get(find_data_type); + return table.GetWithMeta(find_data_type); } const QPointF &Node::GetPosition() diff --git a/app/node/node.h b/app/node/node.h index c698ac2d3..3439cd1b5 100644 --- a/app/node/node.h +++ b/app/node/node.h @@ -52,6 +52,12 @@ class Node : public QObject { Q_OBJECT public: + enum Capabilities { + kNormal = 0x0, + kShader = 0x1, + kSampleProcessor = 0x2 + }; + Node(); virtual ~Node() override; @@ -143,29 +149,37 @@ public: */ QList GetImmediateDependencies() const; - virtual bool IsAccelerated() const; + /** + * @brief Return accelerated capabilities of this node (if any) + */ + virtual Capabilities GetCapabilities(const NodeValueDatabase&) const; + + /** + * @brief Generate a unique identifier for the shader code (if a node can produce multiple) + */ + virtual QString ShaderID(const NodeValueDatabase&) const; /** * @brief Generate hardware accelerated code for this Node */ - virtual QString AcceleratedCodeVertex() const; + virtual QString ShaderVertexCode(const NodeValueDatabase&) const; /** * @brief Generate hardware accelerated code for this Node */ - virtual QString AcceleratedCodeFragment() const; + virtual QString ShaderFragmentCode(const NodeValueDatabase&) const; /** * @brief Number of iterations to run the accelerated code * * Some code is faster if it's merely repeated on a resulting texture rather than run once on the same buffer. */ - virtual int AcceleratedCodeIterations() const; + virtual int ShaderIterations() const; /** * @brief Parameter that should receive the buffer on an iteration past the first */ - virtual NodeInput* AcceleratedCodeIterativeInput() const; + virtual NodeInput* ShaderIterativeInput() const; /** * @brief Return whether this node processes samples or not @@ -323,7 +337,7 @@ public: NodeOutput* output() const; - virtual QVariant InputValueFromTable(NodeInput* input, const NodeValueTable& table) const; + virtual NodeValue InputValueFromTable(NodeInput* input, const NodeValueTable& table) const; const QPointF& GetPosition(); diff --git a/app/node/value.cpp b/app/node/value.cpp index 0ef541256..4d8e5290d 100644 --- a/app/node/value.cpp +++ b/app/node/value.cpp @@ -117,6 +117,19 @@ int NodeValueTable::Count() const return values_.size(); } +bool NodeValueTable::Has(const NodeParam::DataType &type) const +{ + for (int i=values_.size() - 1;i>=0;i--) { + const NodeValue& v = values_.at(i); + + if (v.type() & type) { + return true; + } + } + + return false; +} + bool NodeValueTable::isEmpty() const { return values_.isEmpty(); diff --git a/app/node/value.h b/app/node/value.h index 2d82276ad..f54805689 100644 --- a/app/node/value.h +++ b/app/node/value.h @@ -35,6 +35,7 @@ public: void Prepend(const NodeParam::DataType& type, const QVariant& data, const QString& tag = QString()); const NodeValue& At(int index) const; int Count() const; + bool Has(const NodeParam::DataType& type) const; bool isEmpty() const; diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index 5b6de78d7..e32637e34 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -26,7 +26,7 @@ void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params_in, NodeValueTable *output_params) { // Check if node processes samples - if (!node->ProcessesSamplesFrom()) { + if (!(node->GetCapabilities(input_params_in) & Node::kSampleProcessor)) { return; } diff --git a/app/render/backend/opengl/openglproxy.cpp b/app/render/backend/opengl/openglproxy.cpp index d486fbcd8..e78a02708 100644 --- a/app/render/backend/opengl/openglproxy.cpp +++ b/app/render/backend/opengl/openglproxy.cpp @@ -185,17 +185,17 @@ void OpenGLProxy::Close() void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable *output_params) { - if (!node->IsAccelerated()) { + if (!(node->GetCapabilities(input_params) & Node::kShader)) { return; } - OpenGLShaderPtr shader = shader_cache_.Get(node->id()); + OpenGLShaderPtr shader = shader_cache_.Get(node->ShaderID(input_params)); if (!shader) { // Since we have shader code, compile it now - QString frag_code = node->AcceleratedCodeFragment(); - QString vert_code = node->AcceleratedCodeVertex(); + QString frag_code = node->ShaderFragmentCode(input_params); + QString vert_code = node->ShaderVertexCode(input_params); if (frag_code.isEmpty()) { frag_code = OpenGLShader::CodeDefaultFragment(); @@ -220,7 +220,7 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c GLuint iterative_input = 0; // If this node requires multiple iterations, get a texture for it too - if (node->AcceleratedCodeIterations() > 1 && node->AcceleratedCodeIterativeInput()) { + if (node->ShaderIterations() > 1 && node->ShaderIterativeInput()) { dst_refs.append(texture_cache_.Get(ctx_, video_params_)); } @@ -242,9 +242,10 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c // Get value from database at this input const NodeValueTable& input_data = input_params[input]; - QVariant value = node->InputValueFromTable(input, input_data); + NodeValue meta_value = node->InputValueFromTable(input, input_data); + const QVariant& value = meta_value.data(); - switch (input->data_type()) { + switch (meta_value.type()) { case NodeInput::kInt: shader->setUniformValue(variable_location, value.toInt()); break; @@ -301,7 +302,7 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c } // If this texture binding is the iterative input, set it here - if (input == node->AcceleratedCodeIterativeInput()) { + if (input == node->ShaderIterativeInput()) { iterative_input = input_texture_count; } @@ -351,7 +352,7 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c // Some nodes use multiple iterations for optimization OpenGLTextureCache::ReferencePtr output_tex; - for (int iteration=0;iterationAcceleratedCodeIterations();iteration++) { + for (int iteration=0;iterationShaderIterations();iteration++) { // If this is not the first iteration, set the parameter that will receive the last iteration's texture OpenGLTextureCache::ReferencePtr source_tex = dst_refs.at((iteration+1)%dst_refs.size()); OpenGLTextureCache::ReferencePtr destination_tex = dst_refs.at(iteration%dst_refs.size());