From e2e8a6859c622e66fa974cb6529515c614a87f50 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Fri, 12 Jun 2020 03:28:35 +1000 Subject: [PATCH] renderer: updated audio to work with minor node overhaul --- app/node/audio/pan/pan.cpp | 25 +++++++++---- app/node/audio/pan/pan.h | 2 +- app/node/audio/volume/volume.cpp | 20 ++++++++--- app/node/audio/volume/volume.h | 2 +- app/node/math/math/math.cpp | 48 +++++++++++++++++-------- app/node/math/math/math.h | 2 +- app/node/node.cpp | 2 +- app/node/node.h | 2 +- app/render/backend/renderworker.cpp | 56 ++++++++++------------------- app/render/shaderinfo.h | 26 +++++++++++--- 10 files changed, 111 insertions(+), 74 deletions(-) diff --git a/app/node/audio/pan/pan.cpp b/app/node/audio/pan/pan.cpp index e97f08f67..dad795d02 100644 --- a/app/node/audio/pan/pan.cpp +++ b/app/node/audio/pan/pan.cpp @@ -62,29 +62,42 @@ QString PanNode::Description() const NodeValueTable PanNode::Value(NodeValueDatabase &value) const { // Create a sample job - SampleJob job(samples_input_); + SampleJob job(samples_input_, value); job.InsertValue(panning_input_, value); // Push it to our table NodeValueTable table = value.Merge(); - if (!qIsNull(job.GetValue(samples_input_).data().toDouble())) { - table.Push(NodeParam::kSampleJob, QVariant::fromValue(job), this); + if (job.HasSamples()) { + float pan_volume = job.GetValue(panning_input_).data().toFloat(); + if (panning_input_->is_static()) { + if (!qIsNull(pan_volume) && job.samples()->audio_params().channel_count() == 2) { + if (pan_volume > 0) { + job.samples()->transform_volume_for_channel(0, 1.0f - pan_volume); + } else { + job.samples()->transform_volume_for_channel(1, 1.0f + pan_volume); + } + } + + table.Push(NodeParam::kSamples, QVariant::fromValue(job.samples()), this); + } else { + table.Push(NodeParam::kSampleJob, QVariant::fromValue(job), this); + } } return table; } -void PanNode::ProcessSamples(NodeValueDatabase &values, const AudioParams ¶ms, const SampleBufferPtr input, SampleBufferPtr output, int index) const +void PanNode::ProcessSamples(NodeValueDatabase &values, const SampleBufferPtr input, SampleBufferPtr output, int index) const { - if (params.channel_count() != 2) { + if (input->audio_params().channel_count() != 2) { // This node currently only works for stereo audio return; } float pan_val = values[panning_input_].Get(NodeParam::kFloat).toFloat(); - for (int i=0;iaudio_params().channel_count();i++) { output->data()[i][index] = input->data()[i][index]; } diff --git a/app/node/audio/pan/pan.h b/app/node/audio/pan/pan.h index 87195d538..df23f9d0f 100644 --- a/app/node/audio/pan/pan.h +++ b/app/node/audio/pan/pan.h @@ -39,7 +39,7 @@ public: virtual NodeValueTable Value(NodeValueDatabase &value) const override; - virtual void ProcessSamples(NodeValueDatabase &values, const AudioParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; + virtual void ProcessSamples(NodeValueDatabase &values, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; virtual void Retranslate() override; diff --git a/app/node/audio/volume/volume.cpp b/app/node/audio/volume/volume.cpp index 5988f3c6c..0180fbfaa 100644 --- a/app/node/audio/volume/volume.cpp +++ b/app/node/audio/volume/volume.cpp @@ -60,23 +60,33 @@ QString VolumeNode::Description() const NodeValueTable VolumeNode::Value(NodeValueDatabase &value) const { - SampleJob job(samples_input_); + SampleJob job(samples_input_, value); job.InsertValue(volume_input_, value); NodeValueTable table = value.Merge(); - if (qFuzzyCompare(job.GetValue(volume_input_).data().toDouble(), 1.0)) { - table.Push(NodeParam::kSampleJob, QVariant::fromValue(job), this); + if (job.HasSamples()) { + float volume_val = job.GetValue(volume_input_).data().toFloat(); + if (volume_input_->is_static()) { + // Volume never changes so we can make some performance optimizations here + if (!qFuzzyCompare(volume_val, 1.0f)) { + job.samples()->transform_volume(volume_val); + } + + table.Push(NodeParam::kSamples, QVariant::fromValue(job.samples()), this); + } else { + table.Push(NodeParam::kSampleJob, QVariant::fromValue(job), this); + } } return table; } -void VolumeNode::ProcessSamples(NodeValueDatabase &values, const AudioParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const +void VolumeNode::ProcessSamples(NodeValueDatabase &values, const SampleBufferPtr input, SampleBufferPtr output, int index) const { float volume_val = values[volume_input_].Get(NodeParam::kFloat).toFloat(); - for (int i=0;iaudio_params().channel_count();i++) { output->data()[i][index] = input->data()[i][index] * volume_val; } } diff --git a/app/node/audio/volume/volume.h b/app/node/audio/volume/volume.h index 3b331205e..7547eeffd 100644 --- a/app/node/audio/volume/volume.h +++ b/app/node/audio/volume/volume.h @@ -39,7 +39,7 @@ public: virtual NodeValueTable Value(NodeValueDatabase &value) const override; - virtual void ProcessSamples(NodeValueDatabase &values, const AudioParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; + virtual void ProcessSamples(NodeValueDatabase &values, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; virtual void Retranslate() override; diff --git a/app/node/math/math/math.cpp b/app/node/math/math/math.cpp index 79e455dfa..6c656b369 100644 --- a/app/node/math/math/math.cpp +++ b/app/node/math/math/math.cpp @@ -308,18 +308,26 @@ NodeValueTable MathNode::Value(NodeValueDatabase &value) const bool operation_is_noop = false; - if (calc.GetMostLikelyPairing() == kPairTextureNumber) { - NodeValue& number_val = val_a.type() == NodeParam::kTexture ? val_b : val_a; + const NodeValue& number_val = val_a.type() == NodeParam::kTexture ? val_b : val_a; + if (calc.GetMostLikelyPairing() == kPairTextureNumber) { if (NumberIsNoOp(GetOperation(), RetrieveNumber(number_val))) { operation_is_noop = true; } + } else if (calc.GetMostLikelyPairing() == kPairTextureMatrix) { + // Only allow matrix multiplication + if (GetOperation() != kOpMultiply + || number_val.data().value().isIdentity()) { + operation_is_noop = true; + } } - if (!operation_is_noop) { - output.Push(NodeParam::kShaderJob, QVariant::fromValue(job), this); - } else { + if (operation_is_noop) { + // Just push texture as-is output.Push(val_a.type() == NodeParam::kTexture ? val_a : val_b); + } else { + // Push shader job + output.Push(NodeParam::kShaderJob, QVariant::fromValue(job), this); } break; } @@ -327,18 +335,28 @@ NodeValueTable MathNode::Value(NodeValueDatabase &value) const case kPairSampleNumber: { // Queue a sample job - SampleJob job(val_a.type() == NodeParam::kSamples ? param_a_in_ : param_b_in_); - - NodeValue& number_val = val_a.type() == NodeParam::kSamples ? val_b : val_a; + const NodeValue& number_val = val_a.type() == NodeParam::kSamples ? val_b : val_a; NodeInput* number_param = val_a.type() == NodeParam::kSamples ? param_b_in_ : param_a_in_; float number = RetrieveNumber(number_val); - if (!NumberIsNoOp(GetOperation(), number)) { - job.InsertValue(number_param, NodeValue(NodeParam::kFloat, number, this)); - output.Push(NodeParam::kSampleJob, QVariant::fromValue(job), this); - } else { - output.Push(val_a.type() == NodeParam::kSamples ? val_a : val_b); + SampleJob job(val_a.type() == NodeParam::kSamples ? val_a : val_b); + job.InsertValue(number_param, NodeValue(NodeParam::kFloat, number, this)); + + if (job.HasSamples()) { + if (number_param->is_static()) { + if (!NumberIsNoOp(GetOperation(), number)) { + for (int i=0;iaudio_params().channel_count();i++) { + for (int j=0;jsample_count();j++) { + job.samples()->data()[i][j] = PerformAll(job.samples()->data()[i][j], number); + } + } + } + + output.Push(NodeParam::kSamples, QVariant::fromValue(job.samples()), this); + } else { + output.Push(NodeParam::kSampleJob, QVariant::fromValue(job), this); + } } break; } @@ -351,7 +369,7 @@ NodeValueTable MathNode::Value(NodeValueDatabase &value) const return output; } -void MathNode::ProcessSamples(NodeValueDatabase &values, const AudioParams ¶ms, const SampleBufferPtr input, SampleBufferPtr output, int index) const +void MathNode::ProcessSamples(NodeValueDatabase &values, const SampleBufferPtr input, SampleBufferPtr output, int index) const { // This function is only used for sample+number pairing NodeValue number_val = values[param_a_in_].GetWithMeta(NodeParam::kNumber); @@ -366,7 +384,7 @@ void MathNode::ProcessSamples(NodeValueDatabase &values, const AudioParams ¶ float number_flt = RetrieveNumber(number_val); - for (int i=0;iaudio_params().channel_count();i++) { output->data()[i][index] = PerformAll(input->data()[i][index], number_flt); } } diff --git a/app/node/math/math/math.h b/app/node/math/math/math.h index 35163382e..41934cf31 100644 --- a/app/node/math/math/math.h +++ b/app/node/math/math/math.h @@ -42,7 +42,7 @@ public: virtual ShaderCode GetShaderCode(const QByteArray &shader_id) const override; virtual NodeValueTable Value(NodeValueDatabase &value) const override; - virtual void ProcessSamples(NodeValueDatabase &values, const AudioParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; + virtual void ProcessSamples(NodeValueDatabase &values, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; NodeInput* param_a_in() const; NodeInput* param_b_in() const; diff --git a/app/node/node.cpp b/app/node/node.cpp index 212a88f04..bedbe81fc 100644 --- a/app/node/node.cpp +++ b/app/node/node.cpp @@ -464,7 +464,7 @@ ShaderCode Node::GetShaderCode(const QByteArray &shader_id) const return ShaderCode(QString(), QString()); } -void Node::ProcessSamples(NodeValueDatabase &, const AudioParams&, const SampleBufferPtr, SampleBufferPtr, int) const +void Node::ProcessSamples(NodeValueDatabase &, const SampleBufferPtr, SampleBufferPtr, int) const { } diff --git a/app/node/node.h b/app/node/node.h index 58dbd44ed..54cb0194a 100644 --- a/app/node/node.h +++ b/app/node/node.h @@ -179,7 +179,7 @@ public: /** * @brief If ProcessesSamples() is true, this is the function that will process them. */ - virtual void ProcessSamples(NodeValueDatabase &values, const AudioParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const; + virtual void ProcessSamples(NodeValueDatabase &values, const SampleBufferPtr input, SampleBufferPtr output, int index) const; /** * @brief Returns the input with the specified ID (or nullptr if it doesn't exist) diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index e5a9756fd..4fa7b5852 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -157,7 +157,7 @@ NodeValueTable RenderWorker::GenerateBlockTable(const TrackOutput *track, const if (original_track) { AudioVisualWaveform visual_waveform; visual_waveform.set_channel_count(audio_params_.channel_count()); - visual_waveform.AddSamples(block_range_buffer, audio_params_.sample_rate()); + visual_waveform.OverwriteSamples(block_range_buffer, audio_params_.sample_rate()); original_track->waveform_lock()->lock(); @@ -185,59 +185,32 @@ NodeValueTable RenderWorker::GenerateBlockTable(const TrackOutput *track, const QVariant RenderWorker::ProcessSamples(const Node *node, const TimeRange &range, const SampleJob& job) { - // FIX THIS CODE: - return QVariant(); - /* - NodeInput* sample_input = node->ProcessesSamplesFrom(input_params); - - // Try to find the sample buffer in the table - QVariant samples_var = input_params[sample_input].Get(NodeParam::kSamples); - - // If there isn't one, there's nothing to do - if (samples_var.isNull()) { + if (!job.samples() || !job.samples()->is_allocated()) { return QVariant(); } - SampleBufferPtr input_buffer = samples_var.value(); + SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(job.samples()->audio_params(), job.samples()->sample_count()); + NodeValueDatabase value_db; - if (!input_buffer) { - return QVariant(); - } - - SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(input_buffer->audio_params(), input_buffer->sample_count()); - - int sample_count = input_buffer->sample_count(); - - // FIXME: Hardcoded float sample format - for (int i=0;isample_count();i++) { // Calculate the exact rational time at this sample - int sample_out_of_channel = i / audio_params_.channel_count(); - double sample_to_second = static_cast(sample_out_of_channel) / static_cast(audio_params_.sample_rate()); + double sample_to_second = static_cast(i) / static_cast(audio_params_.sample_rate()); rational this_sample_time = rational::fromDouble(range.in().toDouble() + sample_to_second); // Update all non-sample and non-footage inputs - foreach (NodeParam* param, node->parameters()) { - if (param->type() == NodeParam::kInput - && param != sample_input) { - NodeInput* input = static_cast(param); - - // If the input isn't keyframing, we don't need to update it unless it's connected, in which case it may change - if (input->is_connected() || input->is_keyframing()) { - input_params.Insert(input, ProcessInput(input, TimeRange(this_sample_time, this_sample_time))); - } - } + NodeValueMap::const_iterator j; + for (j=job.GetValues().constBegin(); j!=job.GetValues().constEnd(); j++) { + value_db.Insert(j.key(), ProcessInput(j.key(), TimeRange(this_sample_time, this_sample_time))); } - node->ProcessSamples(input_params, - audio_params_, - input_buffer, + node->ProcessSamples(value_db, + job.samples(), output_buffer, i); } return QVariant::fromValue(output_buffer); - */ } QVariant RenderWorker::GetCachedFrame(const Node* node, const rational& time) @@ -250,6 +223,13 @@ QVariant RenderWorker::GetCachedFrame(const Node* node, const rational& time) if (QFileInfo::exists(fn)) { FramePtr f = FrameHashCache::LoadCacheFrame(hash); + // The cached frame won't load with the correct divider by default, so we enforce it here + f->set_video_params(VideoParams(f->width() * video_params_.divider(), + f->height() * video_params_.divider(), + f->video_params().time_base(), + f->video_params().format(), + video_params_.divider())); + return CachedFrameToTexture(f); } } diff --git a/app/render/shaderinfo.h b/app/render/shaderinfo.h index 642080ea1..d9138da60 100644 --- a/app/render/shaderinfo.h +++ b/app/render/shaderinfo.h @@ -1,6 +1,7 @@ #ifndef SHADERINFO_H #define SHADERINFO_H +#include "codec/samplebuffer.h" #include "node/input.h" #include "node/inputarray.h" #include "node/value.h" @@ -53,18 +54,33 @@ private: class SampleJob : public AcceleratedJob { public: - SampleJob(NodeInput* from = nullptr) : - from_(from) + SampleJob() { + samples_ = nullptr; } - NodeInput* from() const + SampleJob(const NodeValue& value) { - return from_; + samples_ = value.data().value(); + } + + SampleJob(NodeInput* from, NodeValueDatabase& db) + { + samples_ = db[from].Take(NodeParam::kSamples).value(); + } + + SampleBufferPtr samples() const + { + return samples_; + } + + bool HasSamples() const + { + return samples_ && samples_->is_allocated(); } private: - NodeInput* from_; + SampleBufferPtr samples_; };