nodes: made changes to the node system to allow for context sensitive shader code

Primarily implemented for the math node so a new shader can be generated and
compiled based on what inputs the node is receiving.

app/node/audio/pan/pan.cpp

@@ -37,6 +37,11 @@ QString PanNode::Description() const
   return tr("Adjust the stereo panning of an audio source.");
 }
 
+Node::Capabilities PanNode::GetCapabilities(const NodeValueDatabase &) const
+{
+  return kSampleProcessor;
+}
+
 NodeInput *PanNode::ProcessesSamplesFrom() const
 {
   return samples_input_;

app/node/audio/pan/pan.h

@@ -15,6 +15,7 @@ public:
   virtual QString Category() const override;
   virtual QString Description() const override;
 
+  virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
   virtual NodeInput* ProcessesSamplesFrom() const override;
   virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override;
 

app/node/audio/volume/volume.cpp

@@ -36,6 +36,11 @@ QString VolumeNode::Description() const
   return tr("Adjusts the volume of an audio source.");
 }
 
+Node::Capabilities VolumeNode::GetCapabilities(const NodeValueDatabase &) const
+{
+  return kSampleProcessor;
+}
+
 NodeInput *VolumeNode::ProcessesSamplesFrom() const
 {
   return samples_input_;

app/node/audio/volume/volume.h

@@ -15,6 +15,7 @@ public:
   virtual QString Category() const override;
   virtual QString Description() const override;
 
+  virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
   virtual NodeInput* ProcessesSamplesFrom() const override;
   virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override;
 

app/node/block/transition/externaltransition.cpp

@@ -38,27 +38,27 @@ void ExternalTransition::Retranslate()
   meta_.Retranslate();
 }
 
-bool ExternalTransition::IsAccelerated() const
+Node::Capabilities ExternalTransition::GetCapabilities(const NodeValueDatabase &) const
 {
-  return true;
+  return kShader;
 }
 
-QString ExternalTransition::AcceleratedCodeVertex() const
+QString ExternalTransition::ShaderVertexCode(const NodeValueDatabase &) const
 {
   return meta_.vert_code();
 }
 
-QString ExternalTransition::AcceleratedCodeFragment() const
+QString ExternalTransition::ShaderFragmentCode(const NodeValueDatabase&) const
 {
   return meta_.frag_code();
 }
 
-int ExternalTransition::AcceleratedCodeIterations() const
+int ExternalTransition::ShaderIterations() const
 {
   return meta_.iterations();
 }
 
-NodeInput *ExternalTransition::AcceleratedCodeIterativeInput() const
+NodeInput *ExternalTransition::ShaderIterativeInput() const
 {
   return meta_.iteration_input();
 }

app/node/block/transition/externaltransition.h

@@ -19,11 +19,11 @@ public:
 
   virtual void Retranslate() override;
 
-  virtual bool IsAccelerated() const override;
-  virtual QString AcceleratedCodeVertex() const override;
-  virtual QString AcceleratedCodeFragment() const override;
-  virtual int AcceleratedCodeIterations() const override;
-  virtual NodeInput* AcceleratedCodeIterativeInput() const override;
+  virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
+  virtual QString ShaderVertexCode(const NodeValueDatabase&) const override;
+  virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override;
+  virtual int ShaderIterations() const override;
+  virtual NodeInput* ShaderIterativeInput() const override;
 
 private:
   NodeMetaReader meta_;

app/node/external.cpp

@@ -40,27 +40,27 @@ void ExternalNode::Retranslate()
   meta_.Retranslate();
 }
 
-bool ExternalNode::IsAccelerated() const
+Node::Capabilities ExternalNode::GetCapabilities(const NodeValueDatabase &) const
 {
-  return true;
+  return kShader;
 }
 
-QString ExternalNode::AcceleratedCodeVertex() const
+QString ExternalNode::ShaderVertexCode(const NodeValueDatabase&) const
 {
   return meta_.vert_code();
 }
 
-QString ExternalNode::AcceleratedCodeFragment() const
+QString ExternalNode::ShaderFragmentCode(const NodeValueDatabase&) const
 {
   return meta_.frag_code();
 }
 
-int ExternalNode::AcceleratedCodeIterations() const
+int ExternalNode::ShaderIterations() const
 {
   return meta_.iterations();
 }
 
-NodeInput *ExternalNode::AcceleratedCodeIterativeInput() const
+NodeInput *ExternalNode::ShaderIterativeInput() const
 {
   return meta_.iteration_input();
 }

app/node/external.h

@@ -23,11 +23,11 @@ public:
 
   virtual void Retranslate() override;
 
-  virtual bool IsAccelerated() const override;
-  virtual QString AcceleratedCodeVertex() const override;
-  virtual QString AcceleratedCodeFragment() const override;
-  virtual int AcceleratedCodeIterations() const override;
-  virtual NodeInput* AcceleratedCodeIterativeInput() const override;
+  virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
+  virtual QString ShaderVertexCode(const NodeValueDatabase&) const override;
+  virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override;
+  virtual int ShaderIterations() const override;
+  virtual NodeInput* ShaderIterativeInput() const override;
 
 private:
   NodeMetaReader meta_;

app/node/factory.cpp

@@ -8,6 +8,7 @@
 #include "distort/transform/transform.h"
 #include "input/media/video/video.h"
 #include "input/media/audio/audio.h"
+#include "math/math.h"
 #include "output/track/track.h"
 #include "output/viewer/viewer.h"
 #include "external.h"
@@ -135,6 +136,8 @@ Node *NodeFactory::CreateInternal(const NodeFactory::InternalID &id)
     return new VolumeNode();
   case kAudioPanning:
     return new PanNode();
+  case kMath:
+    return new MathNode();
 
   case kInternalNodeCount:
     break;

app/node/factory.h

@@ -19,6 +19,7 @@ public:
     kTrackOutput,
     kAudioVolume,
     kAudioPanning,
+    kMath,
 
     // Count value
     kInternalNodeCount

app/node/input/media/video/video.cpp

@@ -45,17 +45,17 @@ NodeInput *VideoInput::matrix_input() const
   return matrix_input_;
 }
 
-bool VideoInput::IsAccelerated() const
+Node::Capabilities VideoInput::GetCapabilities(const NodeValueDatabase &) const
 {
-  return true;
+  return kShader;
 }
 
-QString VideoInput::AcceleratedCodeVertex() const
+QString VideoInput::ShaderVertexCode(const NodeValueDatabase&) const
 {
   return ReadFileAsString(":/shaders/videoinput.vert");
 }
 
-QString VideoInput::AcceleratedCodeFragment() const
+QString VideoInput::ShaderFragmentCode(const NodeValueDatabase&) const
 {
   return ReadFileAsString(":/shaders/videoinput.frag");
 }

app/node/input/media/video/video.h

@@ -20,9 +20,9 @@ public:
 
   NodeInput* matrix_input() const;
 
-  virtual bool IsAccelerated() const override;
-  virtual QString AcceleratedCodeVertex() const override;
-  virtual QString AcceleratedCodeFragment() const override;
+  virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
+  virtual QString ShaderVertexCode(const NodeValueDatabase&) const override;
+  virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override;
 
   virtual void Retranslate() override;
 

app/node/math/math.cpp

@@ -42,7 +42,91 @@ void MathNode::Retranslate()
 {
   Node::Retranslate();
 
-  method_in_->set_name(tr(""));
-  param_a_in_->set_name(tr(""));
-  param_b_in_->set_name(tr(""));
+  method_in_->set_name(tr("Method"));
+  param_a_in_->set_name(tr("Value"));
+  param_b_in_->set_name(tr("Value"));
+}
+
+Node::Capabilities MathNode::GetCapabilities(const NodeValueDatabase &input) const
+{
+  if (input[param_a_in_].Has(NodeParam::kTexture) || input[param_b_in_].Has(NodeParam::kTexture)) {
+    return kShader;
+  } else if (input[param_a_in_].Has(NodeParam::kSamples) || input[param_b_in_].Has(NodeParam::kSamples)) {
+    return kSampleProcessor;
+  } else {
+    return kNormal;
+  }
+}
+
+QString MathNode::ShaderID(const NodeValueDatabase &input) const
+{
+  // FIXME: Hardcoded ADD operation
+  QString method = QString::number(0);
+
+  QString type_a = QString::number(GuessTypeFromTable(input[param_a_in_]));
+  QString type_b = QString::number(GuessTypeFromTable(input[param_b_in_]));
+
+  return id().append(method).append(type_a).append(type_b);
+}
+
+QString MathNode::ShaderFragmentCode(const NodeValueDatabase &input) const
+{
+  NodeParam::DataType type_a = GuessTypeFromTable(input[param_a_in_]);
+  NodeParam::DataType type_b = GuessTypeFromTable(input[param_b_in_]);
+
+  return QStringLiteral("#version 110\n"
+                        "\n"
+                        "varying vec2 ove_texcoord;\n"
+                        "\n"
+                        "uniform %1 %3;\n"
+                        "uniform %2 %4;\n"
+                        "\n"
+                        "void main(void) {\n"
+                        "  gl_FragColor = %5 + %6;\n"
+                        "}\n").arg(GetUniformTypeFromType(type_a),
+                                   GetUniformTypeFromType(type_b),
+                                   param_a_in_->id(),
+                                   param_b_in_->id(),
+                                   GetVariableCall(param_a_in_->id(), type_a),
+                                   GetVariableCall(param_b_in_->id(), type_b));
+}
+
+NodeValue MathNode::InputValueFromTable(NodeInput *input, const NodeValueTable &table) const
+{
+  if (input->IsConnected()
+      && (input == param_a_in_ || input == param_b_in_)
+      && table.Has(NodeParam::kTexture)) {
+    return table.GetWithMeta(NodeParam::kTexture);
+  }
+
+  return Node::InputValueFromTable(input, table);
+}
+
+NodeParam::DataType MathNode::GuessTypeFromTable(const NodeValueTable &table)
+{
+  if (table.Has(NodeParam::kTexture)) {
+    return NodeParam::kTexture;
+  } else if (table.Has(NodeParam::kSamples)) {
+    return NodeParam::kSamples;
+  } else {
+    return NodeParam::kFloat;
+  }
+}
+
+QString MathNode::GetUniformTypeFromType(const NodeParam::DataType &type)
+{
+  if (type == NodeParam::kTexture) {
+    return QStringLiteral("sampler2D");
+  }
+
+  return QStringLiteral("float");
+}
+
+QString MathNode::GetVariableCall(const QString &input_id, const NodeParam::DataType &type)
+{
+  if (type == NodeParam::kTexture) {
+    return QStringLiteral("texture2D(%1, ove_texcoord)").arg(input_id);
+  }
+
+  return input_id;
 }

app/node/math/math.h

@@ -17,7 +17,26 @@ public:
 
   virtual void Retranslate() override;
 
+  virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override;
+  virtual QString ShaderID(const NodeValueDatabase&) const override;
+  virtual QString ShaderFragmentCode(const NodeValueDatabase&) const override;
+
+  virtual NodeValue InputValueFromTable(NodeInput* input, const NodeValueTable& table) const override;
+
 private:
+  enum Operation {
+    kOpAdd,
+    kOpSubtrack,
+    kOpMultiply,
+    kOpDivide
+  };
+
+  static NodeParam::DataType GuessTypeFromTable(const NodeValueTable& table);
+
+  static QString GetUniformTypeFromType(const NodeParam::DataType& type);
+
+  static QString GetVariableCall(const QString& input_id, const NodeParam::DataType& type);
+
   NodeInput* method_in_;
 
   NodeInput* param_a_in_;

app/node/node.cpp

@@ -434,27 +434,32 @@ QList<Node *> Node::GetImmediateDependencies() const
   return node_list;
 }
 
-bool Node::IsAccelerated() const
+Node::Capabilities Node::GetCapabilities(const NodeValueDatabase &) const
 {
-  return false;
+  return kNormal;
 }
 
-QString Node::AcceleratedCodeVertex() const
+QString Node::ShaderID(const NodeValueDatabase &) const
+{
+  return id();
+}
+
+QString Node::ShaderVertexCode(const NodeValueDatabase &) const
 {
   return QString();
 }
 
-QString Node::AcceleratedCodeFragment() const
+QString Node::ShaderFragmentCode(const NodeValueDatabase&) const
 {
   return QString();
 }
 
-int Node::AcceleratedCodeIterations() const
+int Node::ShaderIterations() const
 {
   return 1;
 }
 
-NodeInput *Node::AcceleratedCodeIterativeInput() const
+NodeInput *Node::ShaderIterativeInput() const
 {
   return nullptr;
 }
@@ -601,7 +606,7 @@ NodeOutput *Node::output() const
   return output_;
 }
 
-QVariant Node::InputValueFromTable(NodeInput *input, const NodeValueTable &table) const
+NodeValue Node::InputValueFromTable(NodeInput *input, const NodeValueTable &table) const
 {
   NodeParam::DataType find_data_type = input->data_type();
 
@@ -611,7 +616,7 @@ QVariant Node::InputValueFromTable(NodeInput *input, const NodeValueTable &table
   }
 
   // Try to get a value from it
-  return table.Get(find_data_type);
+  return table.GetWithMeta(find_data_type);
 }
 
 const QPointF &Node::GetPosition()

app/node/node.h

@@ -52,6 +52,12 @@ class Node : public QObject
 {
   Q_OBJECT
 public:
+  enum Capabilities {
+    kNormal = 0x0,
+    kShader = 0x1,
+    kSampleProcessor = 0x2
+  };
+
   Node();
 
   virtual ~Node() override;
@@ -143,29 +149,37 @@ public:
    */
   QList<Node*> GetImmediateDependencies() const;
 
-  virtual bool IsAccelerated() const;
+  /**
+   * @brief Return accelerated capabilities of this node (if any)
+   */
+  virtual Capabilities GetCapabilities(const NodeValueDatabase&) const;
+
+  /**
+   * @brief Generate a unique identifier for the shader code (if a node can produce multiple)
+   */
+  virtual QString ShaderID(const NodeValueDatabase&) const;
 
   /**
    * @brief Generate hardware accelerated code for this Node
    */
-  virtual QString AcceleratedCodeVertex() const;
+  virtual QString ShaderVertexCode(const NodeValueDatabase&) const;
 
   /**
    * @brief Generate hardware accelerated code for this Node
    */
-  virtual QString AcceleratedCodeFragment() const;
+  virtual QString ShaderFragmentCode(const NodeValueDatabase&) const;
 
   /**
    * @brief Number of iterations to run the accelerated code
    *
    * Some code is faster if it's merely repeated on a resulting texture rather than run once on the same buffer.
    */
-  virtual int AcceleratedCodeIterations() const;
+  virtual int ShaderIterations() const;
 
   /**
    * @brief Parameter that should receive the buffer on an iteration past the first
    */
-  virtual NodeInput* AcceleratedCodeIterativeInput() const;
+  virtual NodeInput* ShaderIterativeInput() const;
 
   /**
    * @brief Return whether this node processes samples or not
@@ -323,7 +337,7 @@ public:
 
   NodeOutput* output() const;
 
-  virtual QVariant InputValueFromTable(NodeInput* input, const NodeValueTable& table) const;
+  virtual NodeValue InputValueFromTable(NodeInput* input, const NodeValueTable& table) const;
 
   const QPointF& GetPosition();
 

app/node/value.cpp

@@ -117,6 +117,19 @@ int NodeValueTable::Count() const
   return values_.size();
 }
 
+bool NodeValueTable::Has(const NodeParam::DataType &type) const
+{
+  for (int i=values_.size() - 1;i>=0;i--) {
+    const NodeValue& v = values_.at(i);
+
+    if (v.type() & type) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 bool NodeValueTable::isEmpty() const
 {
   return values_.isEmpty();

app/node/value.h

@@ -35,6 +35,7 @@ public:
   void Prepend(const NodeParam::DataType& type, const QVariant& data, const QString& tag = QString());
   const NodeValue& At(int index) const;
   int Count() const;
+  bool Has(const NodeParam::DataType& type) const;
 
   bool isEmpty() const;
 

app/render/backend/audio/audioworker.cpp

@@ -26,7 +26,7 @@ void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR
 void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params_in, NodeValueTable *output_params)
 {
   // Check if node processes samples
-  if (!node->ProcessesSamplesFrom()) {
+  if (!(node->GetCapabilities(input_params_in) & Node::kSampleProcessor)) {
     return;
   }
 

app/render/backend/opengl/openglproxy.cpp

@@ -185,17 +185,17 @@ void OpenGLProxy::Close()
 
 void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable *output_params)
 {
-  if (!node->IsAccelerated()) {
+  if (!(node->GetCapabilities(input_params) & Node::kShader)) {
     return;
   }
 
-  OpenGLShaderPtr shader = shader_cache_.Get(node->id());
+  OpenGLShaderPtr shader = shader_cache_.Get(node->ShaderID(input_params));
 
   if (!shader) {
     // Since we have shader code, compile it now
 
-    QString frag_code = node->AcceleratedCodeFragment();
-    QString vert_code = node->AcceleratedCodeVertex();
+    QString frag_code = node->ShaderFragmentCode(input_params);
+    QString vert_code = node->ShaderVertexCode(input_params);
 
     if (frag_code.isEmpty()) {
       frag_code = OpenGLShader::CodeDefaultFragment();
@@ -220,7 +220,7 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c
   GLuint iterative_input = 0;
 
   // If this node requires multiple iterations, get a texture for it too
-  if (node->AcceleratedCodeIterations() > 1 && node->AcceleratedCodeIterativeInput()) {
+  if (node->ShaderIterations() > 1 && node->ShaderIterativeInput()) {
     dst_refs.append(texture_cache_.Get(ctx_, video_params_));
   }
 
@@ -242,9 +242,10 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c
         // Get value from database at this input
         const NodeValueTable& input_data = input_params[input];
 
-        QVariant value = node->InputValueFromTable(input, input_data);
+        NodeValue meta_value = node->InputValueFromTable(input, input_data);
+        const QVariant& value = meta_value.data();
 
-        switch (input->data_type()) {
+        switch (meta_value.type()) {
         case NodeInput::kInt:
           shader->setUniformValue(variable_location, value.toInt());
           break;
@@ -301,7 +302,7 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c
           }
 
           // If this texture binding is the iterative input, set it here
-          if (input == node->AcceleratedCodeIterativeInput()) {
+          if (input == node->ShaderIterativeInput()) {
             iterative_input = input_texture_count;
           }
 
@@ -351,7 +352,7 @@ void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, c
   // Some nodes use multiple iterations for optimization
   OpenGLTextureCache::ReferencePtr output_tex;
 
-  for (int iteration=0;iteration<node->AcceleratedCodeIterations();iteration++) {
+  for (int iteration=0;iteration<node->ShaderIterations();iteration++) {
     // If this is not the first iteration, set the parameter that will receive the last iteration's texture
     OpenGLTextureCache::ReferencePtr source_tex = dst_refs.at((iteration+1)%dst_refs.size());
     OpenGLTextureCache::ReferencePtr destination_tex = dst_refs.at(iteration%dst_refs.size());