From e6c47b14f45c1d45c12a5610989fc5799d37d627 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 30 Dec 2019 20:14:45 +1100 Subject: [PATCH] implemented audio keyframing This mostly builds on the keyframing we already set up for video, but the audio renderer will now appropriately updated keyframe inputs per sample in accordance with keyframe values. --- app/render/backend/audio/audioworker.cpp | 26 +++++++++-- app/render/backend/audio/audioworker.h | 2 +- app/render/backend/opengl/openglworker.cpp | 4 +- app/render/backend/opengl/openglworker.h | 2 +- app/render/backend/renderworker.cpp | 54 ++++++++++++++-------- app/render/backend/renderworker.h | 6 ++- 6 files changed, 66 insertions(+), 28 deletions(-) diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index 09d8effad..235573be4 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -12,15 +12,18 @@ void AudioWorker::FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable table->Push(NodeParam::kSamples, frame->ToByteArray()); } -void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase *input_params, NodeValueTable *output_params) +void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params_in, NodeValueTable *output_params) { // Check if node processes samples if (!node->ProcessesSamplesFrom()) { return; } + // Copy database so we can make some temporary modifications to it + NodeValueDatabase input_params = input_params_in; + // Try to find the sample buffer in the table - QVariant samples_var = (*input_params)[node->ProcessesSamplesFrom()].Get(NodeParam::kSamples); + QVariant samples_var = input_params[node->ProcessesSamplesFrom()].Get(NodeParam::kSamples); // If there isn't one, there's nothing to do if (samples_var.isNull()) { @@ -34,7 +37,24 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c // FIXME: Hardcoded float sample format for (int i=0;iProcessSamples(input_params, + // Calculate the exact rational time at this sample + int sample_out_of_channel = i / audio_params().channel_count(); + double sample_to_second = static_cast(sample_out_of_channel) / static_cast(audio_params().sample_rate()); + + rational this_sample_time = rational::fromDouble(range.in().toDouble() + sample_to_second); + + // Update all non-sample and non-footage inputs + foreach (NodeParam* param, node->parameters()) { + if (param->type() == NodeParam::kInput + && param != node->ProcessesSamplesFrom()) { + NodeInput* input = static_cast(param); + + input_params.Insert(input, ProcessInput(input, + TimeRange(this_sample_time, this_sample_time))); + } + } + + node->ProcessSamples(&input_params, audio_params(), reinterpret_cast(input_buffer.constData()), reinterpret_cast(output_buffer.data()), diff --git a/app/render/backend/audio/audioworker.h b/app/render/backend/audio/audioworker.h index 5e81bccf9..eff2bc5a5 100644 --- a/app/render/backend/audio/audioworker.h +++ b/app/render/backend/audio/audioworker.h @@ -11,7 +11,7 @@ public: protected: virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override; - virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, const NodeValueDatabase *input_params, NodeValueTable* output_params) override; + virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, const NodeValueDatabase& input_params, NodeValueTable* output_params) override; private: diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index c5057850a..8c065a348 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -138,7 +138,7 @@ void OpenGLWorker::ParametersChangedEvent() } } -void OpenGLWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase *input_params, NodeValueTable *output_params) +void OpenGLWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable *output_params) { OpenGLShaderPtr shader = shader_cache_->Get(node->id()); @@ -173,7 +173,7 @@ void OpenGLWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, NodeInput* input = static_cast(param); // Get value from database at this input - const NodeValueTable& input_data = (*input_params)[input]; + const NodeValueTable& input_data = input_params[input]; QVariant value = node->InputValueFromTable(input, input_data); diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index eb49f7349..336fc4183 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -48,7 +48,7 @@ protected: virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override; - virtual void RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase *input_params, NodeValueTable* output_params) override; + virtual void RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params) override; virtual void TextureToBuffer(const QVariant& texture, QByteArray& buffer) override; diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index 21cf82c5e..250ffebdb 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -42,7 +42,7 @@ NodeValueTable RenderWorker::RenderInternal(const NodeDependency &path) return ProcessNode(path); } -void RenderWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase *input_params, NodeValueTable* output_params) +void RenderWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params) { Q_UNUSED(node) Q_UNUSED(range) @@ -86,24 +86,45 @@ NodeValueTable RenderWorker::ProcessNode(const NodeDependency& dep) // FIXME: Cache certain values here if we've already processed them before + // Generate database of input values of node + NodeValueDatabase database = GenerateDatabase(node, dep.range()); + + // By this point, the node should have all the inputs it needs to render correctly + NodeValueTable table = node->Value(database); + + // Check if we have a shader for this output + RunNodeAccelerated(node, dep.range(), database, &table); + + return table; +} + +NodeValueTable RenderWorker::ProcessInput(const NodeInput *input, const TimeRange& range) +{ + if (input->IsConnected()) { + // Value will equal something from the connected node, follow it + return ProcessNode(NodeDependency(input->get_connected_node(), + range)); + } else { + // Push onto the table the value at this time from the input + QVariant input_value = input->get_value_at_time(range.in()); + + NodeValueTable table; + table.Push(input->data_type(), input_value); + return table; + } +} + +NodeValueDatabase RenderWorker::GenerateDatabase(const Node* node, const TimeRange& range) +{ NodeValueDatabase database; // We need to insert tables into the database for each input foreach (NodeParam* param, node->parameters()) { if (param->type() == NodeParam::kInput) { - NodeValueTable table; NodeInput* input = static_cast(param); - TimeRange input_time = node->InputTimeAdjustment(input, dep.range()); + TimeRange input_time = node->InputTimeAdjustment(input, range); - if (input->IsConnected()) { - // Value will equal something from the connected node, follow it - table = ProcessNode(NodeDependency(input->get_connected_node(), - input_time)); - } else { - // Push onto the table the value at this time from the input - QVariant input_value = input->get_value_at_time(input_time.in()); - table.Push(input->data_type(), input_value); - } + NodeValueTable table = ProcessInput(input, input_time); // Exception for Footage types where we actually retrieve some Footage data from a decoder if (input->data_type() == NodeParam::kFootage) { @@ -126,12 +147,5 @@ NodeValueTable RenderWorker::ProcessNode(const NodeDependency& dep) } } - // By this point, the node should have all the inputs it needs to render correctly - - NodeValueTable table = node->Value(database); - - // Check if we have a shader for this output - RunNodeAccelerated(node, dep.range(), &database, &table); - - return table; + return database; } diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index 659aa7557..d62f08366 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -35,7 +35,7 @@ protected: virtual NodeValueTable RenderInternal(const NodeDependency& path); - virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, const NodeValueDatabase *input_params, NodeValueTable* output_params); + virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, const NodeValueDatabase &input_params, NodeValueTable* output_params); StreamPtr ResolveStreamFromInput(NodeInput* input); DecoderPtr ResolveDecoderFromInput(StreamPtr stream); @@ -48,7 +48,11 @@ protected: virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) = 0; + NodeValueTable ProcessInput(const NodeInput* input, const TimeRange &range); + private: + NodeValueDatabase GenerateDatabase(const Node *node, const TimeRange &range); + bool started_; DecoderCache decoder_cache_;