rewrote hashing

app/node/CMakeLists.txt

@@ -34,6 +34,8 @@ set(OLIVE_SOURCES
   node/globals.h
   node/graph.cpp
   node/graph.h
+  node/hashtraverser.cpp
+  node/hashtraverser.h
   node/inputdragger.cpp
   node/inputdragger.h
   node/inputimmediate.cpp

app/node/hashtraverser.cpp

@@ -0,0 +1,172 @@
+/***
+
+  Olive - Non-Linear Video Editor
+  Copyright (C) 2021 Olive Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#include "hashtraverser.h"
+
+#include <QUuid>
+
+#include "common/filefunctions.h"
+
+namespace olive {
+
+#define super NodeTraverser
+
+HashTraverser::HashTraverser() :
+  hash_(QCryptographicHash::Sha1) // Appears to be the fastest hashing algorithm
+{
+}
+
+QByteArray HashTraverser::GetHash(const Node *node, const Node::ValueHint &hint, const VideoParams &params, const TimeRange &range)
+{
+  // Reset hash
+  hash_.reset();
+  texture_ids_.clear();
+
+  // Set params throughout traverser
+  SetCacheVideoParams(params);
+
+  // Embed video parameters into this hash
+  Hash(params.effective_width());
+  Hash(params.effective_height());
+  Hash(params.format());
+  Hash(params.interlacing());
+  //Hash(reference);
+
+  // Our overrides will generate a hash from this
+  NodeValueTable table = GenerateTable(node, hint, range);
+  NodeValue final_value = GenerateRowValueElement(hint, NodeValue::kTexture, &table);
+  qDebug() << "finished with result" << final_value.data();
+  HashNodeValue(final_value);
+
+  // Return the hash
+  return hash_.result();
+}
+
+TexturePtr HashTraverser::ProcessVideoFootage(const FootageJob &stream, const rational &input_time)
+{
+  Hash(FileFunctions::GetUniqueFileIdentifier(stream.filename()));
+  Hash(stream.loop_mode());
+  Hash(stream.video_params().stream_index());
+  Hash(stream.video_params().colorspace());
+  Hash(stream.video_params().premultiplied_alpha());
+  Hash(GetCacheVideoParams().divider());
+  Hash(stream.video_params().video_type() == VideoParams::kVideoTypeStill ? 0 : input_time);
+  Hash(stream.video_params().video_type());
+
+  TexturePtr texture = super::ProcessVideoFootage(stream, input_time);
+  texture_ids_.insert(texture.get(), hash_.result());
+  return texture;
+}
+
+SampleBufferPtr HashTraverser::ProcessAudioFootage(const FootageJob &stream, const TimeRange &input_time)
+{
+  Hash(stream.audio_params().stream_index());
+
+  SampleBufferPtr buf = super::ProcessAudioFootage(stream, input_time);
+  texture_ids_.insert(buf.get(), hash_.result());
+  return buf;
+}
+
+TexturePtr HashTraverser::ProcessShader(const Node *node, const TimeRange &range, const ShaderJob &job)
+{
+  HashGenerateJob(node, &job);
+
+  Hash(job.GetShaderID());
+  Hash(job.GetIterativeInput());
+  Hash(job.GetIterationCount());
+
+  for (auto it=job.GetInterpolationMap().cbegin(); it!=job.GetInterpolationMap().cend(); it++) {
+    Hash(it.key());
+    Hash(it.value());
+  }
+
+  TexturePtr texture = super::ProcessShader(node, range, job);
+  texture_ids_.insert(texture.get(), hash_.result());
+  return texture;
+}
+
+SampleBufferPtr HashTraverser::ProcessSamples(const Node *node, const TimeRange &range, const SampleJob &job)
+{
+  SampleBufferPtr buf = super::ProcessSamples(node, range, job);
+  texture_ids_.insert(buf.get(), hash_.result());
+  return buf;
+}
+
+TexturePtr HashTraverser::ProcessFrameGeneration(const Node *node, const GenerateJob &job)
+{
+  HashGenerateJob(node, &job);
+
+  TexturePtr texture = super::ProcessFrameGeneration(node, job);
+  texture_ids_.insert(texture.get(), hash_.result());
+  return texture;
+}
+
+void HashTraverser::HashGenerateJob(const Node *node, const GenerateJob *job)
+{
+  Hash(node->id());
+  Hash(job->GetAlphaChannelRequired());
+
+  for (auto it=job->GetValues().cbegin(); it!=job->GetValues().cend(); it++) {
+    Hash(it.key());
+    HashNodeValue(it.value());
+  }
+}
+
+void HashTraverser::Hash(const QByteArray &array)
+{
+  hash_.addData(array);
+}
+
+void HashTraverser::Hash(const QString &string)
+{
+  hash_.addData(string.toUtf8());
+}
+
+void HashTraverser::HashNodeValue(const NodeValue &value)
+{
+  NodeValue::Type value_type = value.type();
+
+  if (value_type == NodeValue::kSamples || value_type == NodeValue::kTexture) {
+    QByteArray id_for_buffer;
+    if (value_type == NodeValue::kTexture) {
+      TexturePtr texture = value.data().value<TexturePtr>();
+      id_for_buffer = texture_ids_.value(texture.get());
+    } else {
+      SampleBufferPtr samples = value.data().value<SampleBufferPtr>();
+      id_for_buffer = texture_ids_.value(samples.get());
+    }
+
+    if (id_for_buffer.isEmpty()) {
+      qWarning() << "Found ID-less buffer while hashing, collisions are likely to occur";
+    } else {
+      Hash(id_for_buffer);
+    }
+  } else {
+    Hash(NodeValue::ValueToBytes(value));
+  }
+}
+
+template<typename T>
+void HashTraverser::Hash(T value)
+{
+  hash_.addData(reinterpret_cast<const char*>(&value), sizeof(value));
+}
+
+}

app/node/hashtraverser.h

@@ -0,0 +1,68 @@
+/***
+
+  Olive - Non-Linear Video Editor
+  Copyright (C) 2021 Olive Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#ifndef HASHTRAVERSER_H
+#define HASHTRAVERSER_H
+
+#include "traverser.h"
+
+namespace olive {
+
+class HashTraverser : public NodeTraverser
+{
+public:
+  HashTraverser();
+
+  QByteArray GetHash(const Node *node, const Node::ValueHint &hint, const VideoParams &params, const TimeRange &range);
+
+protected:
+  virtual TexturePtr ProcessVideoFootage(const FootageJob &stream, const rational &input_time) override;
+
+  virtual SampleBufferPtr ProcessAudioFootage(const FootageJob &stream, const TimeRange &input_time);
+
+  virtual TexturePtr ProcessShader(const Node *node, const TimeRange &range, const ShaderJob& job) override;
+
+  virtual SampleBufferPtr ProcessSamples(const Node *node, const TimeRange &range, const SampleJob &job);
+
+  virtual TexturePtr ProcessFrameGeneration(const Node *node, const GenerateJob& job) override;
+
+private:
+  void HashGenerateJob(const Node *node, const GenerateJob *job);
+
+  void HashFootageJob();
+
+  template <typename T>
+  void Hash(T value);
+
+  void Hash(const QByteArray &array);
+
+  void Hash(const QString &string);
+
+  void HashNodeValue(const NodeValue &value);
+
+  QCryptographicHash hash_;
+
+  QHash<void*, QByteArray> texture_ids_;
+
+};
+
+}
+
+#endif // HASHTRAVERSER_H

app/node/traverser.cpp

@@ -84,22 +84,26 @@ NodeValue NodeTraverser::GenerateRowValue(const Node *node, const QString &input
 
 NodeValue NodeTraverser::GenerateRowValueElement(const Node *node, const QString &input, int element, NodeValueTable *table)
 {
-  int value_index = GenerateRowValueElementIndex(node, input, element, table);
-
-  if (value_index == -1) {
-    return NodeValue();
-  } else {
-    return table->TakeAt(value_index);
-  }
+  return GenerateRowValueElement(node->GetValueHintForInput(input, element), node->GetInputDataType(input), table);
 }
 
-int NodeTraverser::GenerateRowValueElementIndex(const Node *node, const QString &input, int element, const NodeValueTable *table)
+NodeValue NodeTraverser::GenerateRowValueElement(const Node::ValueHint &hint, NodeValue::Type preferred_type, NodeValueTable *table)
+{
+  int value_index = GenerateRowValueElementIndex(hint, preferred_type, table);
+
+  if (value_index == -1) {
+    value_index = table->Count() - 1;
+  }
+
+  return table->TakeAt(value_index);
+}
+
+int NodeTraverser::GenerateRowValueElementIndex(const Node::ValueHint &hint, NodeValue::Type preferred_type, const NodeValueTable *table)
 {
-  Node::ValueHint hint = node->GetValueHintForInput(input, element);
   QVector<NodeValue::Type> types = hint.type;
 
   if (types.isEmpty()) {
-    types.append(node->GetInputDataType(input));
+    types.append(preferred_type);
   }
 
   if (hint.index == -1) {
@@ -124,6 +128,11 @@ int NodeTraverser::GenerateRowValueElementIndex(const Node *node, const QString
   }
 }
 
+int NodeTraverser::GenerateRowValueElementIndex(const Node *node, const QString &input, int element, const NodeValueTable *table)
+{
+  return GenerateRowValueElementIndex(node->GetValueHintForInput(input, element), node->GetInputDataType(input), table);
+}
+
 NodeGlobals NodeTraverser::GenerateGlobals(const VideoParams &params, const TimeRange &time)
 {
   return NodeGlobals(QVector2D(params.width(), params.height()), time);
@@ -248,23 +257,23 @@ NodeValueTable NodeTraverser::GenerateBlockTable(const Track *track, const TimeR
   return table;
 }
 
-QVariant NodeTraverser::ProcessVideoFootage(const FootageJob &stream, const rational &input_time)
+TexturePtr NodeTraverser::ProcessVideoFootage(const FootageJob &stream, const rational &input_time)
 {
   Q_UNUSED(input_time)
 
   // Create dummy texture with footage params
-  return QVariant::fromValue(std::make_shared<Texture>(stream.video_params()));
+  return std::make_shared<Texture>(stream.video_params());
 }
 
-QVariant NodeTraverser::ProcessAudioFootage(const FootageJob& stream, const TimeRange &input_time)
+SampleBufferPtr NodeTraverser::ProcessAudioFootage(const FootageJob& stream, const TimeRange &input_time)
 {
   Q_UNUSED(stream)
   Q_UNUSED(input_time)
 
-  return QVariant::fromValue(SampleBuffer::Create());
+  return SampleBuffer::Create();
 }
 
-QVariant NodeTraverser::ProcessShader(const Node *node, const TimeRange &range, const ShaderJob &job)
+TexturePtr NodeTraverser::ProcessShader(const Node *node, const TimeRange &range, const ShaderJob &job)
 {
   Q_UNUSED(node)
   Q_UNUSED(range)
@@ -273,19 +282,19 @@ QVariant NodeTraverser::ProcessShader(const Node *node, const TimeRange &range,
   // Create dummy texture with sequence params
   VideoParams tex_params = video_params_;
   tex_params.set_channel_count(GetChannelCountFromJob(job));
-  return QVariant::fromValue(std::make_shared<Texture>(tex_params));
+  return std::make_shared<Texture>(tex_params);
 }
 
-QVariant NodeTraverser::ProcessSamples(const Node *node, const TimeRange &range, const SampleJob &job)
+SampleBufferPtr NodeTraverser::ProcessSamples(const Node *node, const TimeRange &range, const SampleJob &job)
 {
   Q_UNUSED(node)
   Q_UNUSED(range)
   Q_UNUSED(job)
 
-  return QVariant::fromValue(SampleBuffer::Create());
+  return SampleBuffer::Create();
 }
 
-QVariant NodeTraverser::ProcessFrameGeneration(const Node *node, const GenerateJob &job)
+TexturePtr NodeTraverser::ProcessFrameGeneration(const Node *node, const GenerateJob &job)
 {
   Q_UNUSED(node)
   Q_UNUSED(job)
@@ -293,20 +302,20 @@ QVariant NodeTraverser::ProcessFrameGeneration(const Node *node, const GenerateJ
   // Create dummy texture with sequence params
   VideoParams tex_params = video_params_;
   tex_params.set_channel_count(GetChannelCountFromJob(job));
-  return QVariant::fromValue(std::make_shared<Texture>(tex_params));
+  return std::make_shared<Texture>(tex_params);
 }
 
-void NodeTraverser::SaveCachedTexture(const QByteArray &hash, const QVariant &texture)
+void NodeTraverser::SaveCachedTexture(const QByteArray &hash, TexturePtr texture)
 {
   Q_UNUSED(hash)
   Q_UNUSED(texture)
 }
 
-QVariant NodeTraverser::GetCachedTexture(const QByteArray& hash)
+TexturePtr NodeTraverser::GetCachedTexture(const QByteArray& hash)
 {
   Q_UNUSED(hash)
 
-  return QVariant();
+  return nullptr;
 }
 
 QVector2D NodeTraverser::GenerateResolution() const
@@ -320,18 +329,18 @@ void NodeTraverser::PostProcessTable(const Node *node, const Node::ValueHint &hi
   QByteArray cached_node_hash;
 
   // Convert footage to image/sample buffers
-  if (CanCacheFrames() && node->GetCacheTextures()) {
+  /*if (CanCacheFrames() && node->GetCacheTextures()) {
     // This node is set to cache the result, see if we can retrieved a previously cached version
     cached_node_hash = RenderManager::Hash(node, hint, GetCacheVideoParams(), range.in());
 
-    QVariant cached_frame = GetCachedTexture(cached_node_hash);
-    if (!cached_frame.isNull()) {
-      output_params.Push(NodeValue::kTexture, cached_frame, node);
+    TexturePtr cached_frame = GetCachedTexture(cached_node_hash);
+    if (cached_frame) {
+      output_params.Push(NodeValue::kTexture, QVariant::fromValue(cached_frame), node);
 
       // No more to do here
       got_cached_frame = true;
     }
-  }
+  }*/
 
   // Strip out any jobs or footage
   QList<NodeValue> footage_jobs_to_run;
@@ -369,31 +378,19 @@ void NodeTraverser::PostProcessTable(const Node *node, const Node::ValueHint &hi
         rational footage_time = Footage::AdjustTimeByLoopMode(range.in(), job.loop_mode(), job.length(), job.video_params().video_type(), job.video_params().frame_rate_as_time_base());
 
         if (!footage_time.isNaN()) {
-          QVariant value = ProcessVideoFootage(job, footage_time);
-
-          if (!value.isNull()) {
-            output_params.Push(NodeValue::kTexture, value, node);
-          }
+          output_params.Push(NodeValue::kTexture, QVariant::fromValue(ProcessVideoFootage(job, footage_time)), node);
         }
       }
     }
 
     // Run shaders
     foreach (const NodeValue& v, shader_jobs_to_run) {
-      QVariant value = ProcessShader(node, range, v.data().value<ShaderJob>());
-
-      if (!value.isNull()) {
-        output_params.Push(NodeValue::kTexture, value, node);
-      }
+      output_params.Push(NodeValue::kTexture, QVariant::fromValue(ProcessShader(node, range, v.data().value<ShaderJob>())), node);
     }
 
     // Run generate jobs
     foreach (const NodeValue& v, generate_jobs_to_run) {
-      QVariant value = ProcessFrameGeneration(node, v.data().value<GenerateJob>());
-
-      if (!value.isNull()) {
-        output_params.Push(NodeValue::kTexture, value, node);
-      }
+      output_params.Push(NodeValue::kTexture, QVariant::fromValue(ProcessFrameGeneration(node, v.data().value<GenerateJob>())), node);
     }
   }
 
@@ -403,26 +400,18 @@ void NodeTraverser::PostProcessTable(const Node *node, const Node::ValueHint &hi
     FootageJob job = v.data().value<FootageJob>();
 
     if (job.type() == Track::kAudio) {
-      QVariant value = ProcessAudioFootage(job, range);
-
-      if (!value.isNull()) {
-        output_params.Push(NodeValue::kSamples, value, node);
-      }
+      output_params.Push(NodeValue::kSamples, QVariant::fromValue(ProcessAudioFootage(job, range)), node);
     }
   }
 
   // Run any accelerated shader jobs
   foreach (const NodeValue& v, sample_jobs_to_run) {
-    QVariant value = ProcessSamples(node, range, v.data().value<SampleJob>());
-
-    if (!value.isNull()) {
-      output_params.Push(NodeValue::kSamples, value, node);
-    }
+    output_params.Push(NodeValue::kSamples, QVariant::fromValue(ProcessSamples(node, range, v.data().value<SampleJob>())), node);
   }
 
   if (CanCacheFrames() && node->GetCacheTextures() && !got_cached_frame) {
     // Save cached texture
-    SaveCachedTexture(cached_node_hash, output_params.Get(NodeValue::kTexture));
+    SaveCachedTexture(cached_node_hash, output_params.Get(NodeValue::kTexture).value<TexturePtr>());
   }
 }
 

app/node/traverser.h

@@ -45,6 +45,8 @@ public:
 
   NodeValue GenerateRowValue(const Node *node, const QString &input, NodeValueTable *table);
   NodeValue GenerateRowValueElement(const Node *node, const QString &input, int element, NodeValueTable *table);
+  NodeValue GenerateRowValueElement(const Node::ValueHint &hint, NodeValue::Type preferred_type, NodeValueTable *table);
+  int GenerateRowValueElementIndex(const Node::ValueHint &hint, NodeValue::Type preferred_type, const NodeValueTable *table);
   int GenerateRowValueElementIndex(const Node *node, const QString &input, int element, const NodeValueTable *table);
 
   static NodeGlobals GenerateGlobals(const VideoParams &params, const TimeRange &time);
@@ -70,19 +72,19 @@ protected:
 
   virtual NodeValueTable GenerateBlockTable(const Track *track, const TimeRange& range);
 
-  virtual QVariant ProcessVideoFootage(const FootageJob &stream, const rational &input_time);
+  virtual TexturePtr ProcessVideoFootage(const FootageJob &stream, const rational &input_time);
 
-  virtual QVariant ProcessAudioFootage(const FootageJob &stream, const TimeRange &input_time);
+  virtual SampleBufferPtr ProcessAudioFootage(const FootageJob &stream, const TimeRange &input_time);
 
-  virtual QVariant ProcessShader(const Node *node, const TimeRange &range, const ShaderJob& job);
+  virtual TexturePtr ProcessShader(const Node *node, const TimeRange &range, const ShaderJob& job);
 
-  virtual QVariant ProcessSamples(const Node *node, const TimeRange &range, const SampleJob &job);
+  virtual SampleBufferPtr ProcessSamples(const Node *node, const TimeRange &range, const SampleJob &job);
 
-  virtual QVariant ProcessFrameGeneration(const Node *node, const GenerateJob& job);
+  virtual TexturePtr ProcessFrameGeneration(const Node *node, const GenerateJob& job);
 
-  virtual QVariant GetCachedTexture(const QByteArray& hash);
+  virtual TexturePtr GetCachedTexture(const QByteArray& hash);
 
-  virtual void SaveCachedTexture(const QByteArray& hash, const QVariant& texture);
+  virtual void SaveCachedTexture(const QByteArray& hash, TexturePtr texture);
 
   virtual bool CanCacheFrames()
   {

app/node/value.h

@@ -260,6 +260,10 @@ public:
    * @brief Convert a value from a NodeParam into bytes
    */
   static QByteArray ValueToBytes(Type type, const QVariant& value);
+  static QByteArray ValueToBytes(const NodeValue &value)
+  {
+    return ValueToBytes(value.type(), value.data());
+  }
 
   static QVector<QVariant> split_normal_value_into_track_values(Type type, const QVariant &value);