diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index 6efe27768..5d190af1f 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -22,6 +22,8 @@ set(OLIVE_SOURCES audio/audiovisualwaveform.h audio/packedprocessor.cpp audio/packedprocessor.h + audio/planarprocessor.cpp + audio/planarprocessor.h audio/tempoprocessor.cpp audio/tempoprocessor.h PARENT_SCOPE diff --git a/app/audio/packedprocessor.cpp b/app/audio/packedprocessor.cpp index 10eb9ea18..f748da5c2 100644 --- a/app/audio/packedprocessor.cpp +++ b/app/audio/packedprocessor.cpp @@ -76,17 +76,12 @@ QByteArray PackedProcessor::Convert(SampleBufferPtr planar) return QByteArray(); } - int nb_channels = planar->audio_params().channel_count(); - QByteArray output(planar->audio_params().samples_to_bytes(nb_samples), Qt::Uninitialized); uint8_t *output_data = reinterpret_cast(output.data()); - QVector input_arrays(nb_channels); - for (int i=0; i(planar->data(i)); - } - - int ret = swr_convert(swr_ctx_, &output_data, nb_samples, input_arrays.data(), nb_samples); + int ret = swr_convert(swr_ctx_, &output_data, nb_samples, + const_cast(reinterpret_cast(planar->to_raw_ptrs())), + nb_samples); if (ret < 0) { char buf[200]; av_strerror(ret, buf, 200); diff --git a/app/audio/planarprocessor.cpp b/app/audio/planarprocessor.cpp new file mode 100644 index 000000000..abd2be85b --- /dev/null +++ b/app/audio/planarprocessor.cpp @@ -0,0 +1,104 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "planarprocessor.h" + +#include "common/ffmpegutils.h" + +namespace olive { + +PlanarProcessor::PlanarProcessor() : + swr_ctx_(nullptr) +{ +} + +PlanarProcessor::~PlanarProcessor() +{ + Close(); +} + +bool PlanarProcessor::Open(const AudioParams ¶ms) +{ + if (IsOpen()) { + return true; + } + + swr_ctx_ = swr_alloc_set_opts(nullptr, + params.channel_layout(), + FFmpegUtils::GetFFmpegSampleFormat(params.format(), true), + params.sample_rate(), + params.channel_layout(), + FFmpegUtils::GetFFmpegSampleFormat(params.format(), false), + params.sample_rate(), + 0, + nullptr); + + if (!swr_ctx_) { + qCritical() << "Failed to allocate resample context"; + return false; + } + + if (swr_init(swr_ctx_) < 0) { + qCritical() << "Failed to init resample context"; + swr_free(&swr_ctx_); + return false; + } + + params_ = params; + + return true; +} + +SampleBufferPtr PlanarProcessor::Convert(const QByteArray &packed) +{ + if (!IsOpen()) { + qCritical() << "Tried to convert while closed"; + return nullptr; + } + + if (packed.isEmpty()) { + return nullptr; + } + + int nb_samples_per_channel = params_.bytes_to_samples(packed.size()); + + SampleBufferPtr output = SampleBuffer::CreateAllocated(params_, nb_samples_per_channel); + + const uint8_t *input = reinterpret_cast(packed.constData()); + int ret = swr_convert(swr_ctx_, + reinterpret_cast(output->to_raw_ptrs()), nb_samples_per_channel, + &input, nb_samples_per_channel); + if (ret < 0) { + char buf[200]; + av_strerror(ret, buf, 200); + qDebug() << "Planar processor failed with error:" << buf << ret; + } + + return output; +} + +void PlanarProcessor::Close() +{ + if (swr_ctx_) { + swr_free(&swr_ctx_); + } +} + +} diff --git a/app/audio/planarprocessor.h b/app/audio/planarprocessor.h new file mode 100644 index 000000000..ad5167a95 --- /dev/null +++ b/app/audio/planarprocessor.h @@ -0,0 +1,62 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef PLANARPROCESSOR_H +#define PLANARPROCESSOR_H + +extern "C" { +#include +} + +#include "codec/samplebuffer.h" +#include "render/audioparams.h" + +namespace olive { + +class PlanarProcessor +{ +public: + PlanarProcessor(); + + ~PlanarProcessor(); + + DISABLE_COPY_MOVE(PlanarProcessor) + + bool Open(const AudioParams ¶ms); + + SampleBufferPtr Convert(const QByteArray &packed); + + void Close(); + + bool IsOpen() const + { + return swr_ctx_; + } + +private: + SwrContext *swr_ctx_; + + AudioParams params_; + +}; + +} + +#endif // PLANARPROCESSOR_H diff --git a/app/audio/tempoprocessor.cpp b/app/audio/tempoprocessor.cpp index 0a0aa777e..f22946227 100644 --- a/app/audio/tempoprocessor.cpp +++ b/app/audio/tempoprocessor.cpp @@ -36,7 +36,6 @@ TempoProcessor::TempoProcessor() : filter_graph_(nullptr), buffersrc_ctx_(nullptr), buffersink_ctx_(nullptr), - processed_frame_(nullptr), open_(false) { } @@ -150,47 +149,42 @@ bool TempoProcessor::Open(const AudioParams ¶ms, const double& speed) return true; } -void TempoProcessor::Push(const char *data, int length) +void TempoProcessor::Push(const QByteArray &packed) { - if (flushed_) { - if (length > 0) { - qCritical() << "Tried to push" << length << "bytes after TempoProcessor was closed"; - } + if (!IsOpen()) { + qWarning() << "Tried to push to closed TempoProcessor"; return; } - AVFrame* src_frame; - - if (length == 0) { - // No audio data, flush the last out of the filter graph - src_frame = nullptr; - flushed_ = true; - } else { - src_frame = av_frame_alloc(); - - if (!src_frame) { - qCritical() << "Failed to allocate source frame"; - return; - } - - // Allocate a buffer for the number of samples we got - src_frame->sample_rate = params_.sample_rate(); - src_frame->format = FFmpegUtils::GetFFmpegSampleFormat(params_.format()); - src_frame->channel_layout = params_.channel_layout(); - src_frame->nb_samples = params_.bytes_to_samples(length); - src_frame->pts = timestamp_; - timestamp_ += src_frame->nb_samples; - - if (av_frame_get_buffer(src_frame, 0) < 0) { - qCritical() << "Failed to allocate buffer for source frame"; - av_frame_free(&src_frame); - return; - } - - // Copy buffer from data array to frame - memcpy(src_frame->data[0], data, static_cast(length)); + if (flushed_) { + qWarning() << "Tried to push to flushed TempoProcessor"; + return; } + AVFrame* src_frame = av_frame_alloc(); + + if (!src_frame) { + qCritical() << "Failed to allocate source frame"; + return; + } + + // Allocate a buffer for the number of samples we got + src_frame->sample_rate = params_.sample_rate(); + src_frame->format = FFmpegUtils::GetFFmpegSampleFormat(params_.format()); + src_frame->channel_layout = params_.channel_layout(); + src_frame->nb_samples = params_.bytes_to_samples(packed.size()); + src_frame->pts = timestamp_; + timestamp_ += src_frame->nb_samples; + + if (av_frame_get_buffer(src_frame, 0) < 0) { + qCritical() << "Failed to allocate buffer for source frame"; + av_frame_free(&src_frame); + return; + } + + // Copy buffer from data array to frame + memcpy(src_frame->data[0], packed, packed.size()); + int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, src_frame, AV_BUFFERSRC_FLAG_KEEP_REF); if (ret < 0) { @@ -202,46 +196,45 @@ void TempoProcessor::Push(const char *data, int length) } } -int TempoProcessor::Pull(char *data, int max_length) +void TempoProcessor::Flush() { - if (!processed_frame_) { - processed_frame_ = av_frame_alloc(); - - // Try to pull samples from the buffersink - int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame_); - + if (!flushed_) { + int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF); if (ret < 0) { - // We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the - // error might be fatal... - if (ret != AVERROR(EAGAIN)) { - qCritical() << "Failed to pull from buffersink" << ret; - } + qCritical() << "Failed to feed buffer source" << ret; + } + flushed_ = true; + } +} - av_frame_free(&processed_frame_); +QByteArray TempoProcessor::Pull() +{ + QByteArray b; + AVFrame *processed_frame = av_frame_alloc(); - return 0; + // Try to pull samples from the buffersink + int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame); + + if (ret < 0) { + // We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the + // error might be fatal... + if (ret != AVERROR(EAGAIN)) { + qCritical() << "Failed to pull from buffersink" << ret; } - processed_frame_byte_index_ = 0; - processed_frame_max_bytes_ = params_.samples_to_bytes(processed_frame_->nb_samples); + av_frame_free(&processed_frame); + return b; } - // Determine how many bytes we should copy into the data array - int copy_length = qMin(max_length, processed_frame_max_bytes_ - processed_frame_byte_index_); + b.resize(params_.samples_to_bytes(processed_frame->nb_samples)); // Copy the bytes - memcpy(data, processed_frame_->data[0] + processed_frame_byte_index_, static_cast(copy_length)); - - // Add the copied amount to the current index - processed_frame_byte_index_ += copy_length; + memcpy(b.data(), processed_frame->data[0], b.size()); // If the index has reached the limit of this processed frame, we can dispose of the frame now - if (processed_frame_byte_index_ == processed_frame_max_bytes_) { - av_frame_free(&processed_frame_); - processed_frame_ = nullptr; - } + av_frame_free(&processed_frame); - return copy_length; + return b; } void TempoProcessor::Close() @@ -253,11 +246,6 @@ void TempoProcessor::Close() filter_graph_ = nullptr; } - if (processed_frame_) { - av_frame_free(&processed_frame_); - processed_frame_ = nullptr; - } - buffersrc_ctx_ = nullptr; buffersink_ctx_ = nullptr; } diff --git a/app/audio/tempoprocessor.h b/app/audio/tempoprocessor.h index bac242751..c56eafc5d 100644 --- a/app/audio/tempoprocessor.h +++ b/app/audio/tempoprocessor.h @@ -52,9 +52,11 @@ public: bool Open(const AudioParams& params, const double &speed); - void Push(const char *data, int length); + void Push(const QByteArray &packed); - int Pull(char* data, int max_length); + void Flush(); + + QByteArray Pull(); void Close(); @@ -67,10 +69,6 @@ private: AVFilterContext* buffersink_ctx_; - AVFrame* processed_frame_; - int processed_frame_byte_index_; - int processed_frame_max_bytes_; - AudioParams params_; int64_t timestamp_; diff --git a/app/dialog/speedduration/speeddurationdialog.cpp b/app/dialog/speedduration/speeddurationdialog.cpp index f5308e8c3..ed4432166 100644 --- a/app/dialog/speedduration/speeddurationdialog.cpp +++ b/app/dialog/speedduration/speeddurationdialog.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "core.h" @@ -39,49 +40,60 @@ SpeedDurationDialog::SpeedDurationDialog(const QVector &clips, cons { setWindowTitle(tr("Speed/Duration")); - QGridLayout *layout = new QGridLayout(this); + QVBoxLayout *layout = new QVBoxLayout(this); - int row = 0; + { + QGroupBox *speed_group = new QGroupBox(); + layout->addWidget(speed_group); - layout->addWidget(new QLabel(tr("Speed:")), row, 0); + QGridLayout *speed_layout = new QGridLayout(speed_group); - speed_slider_ = new FloatSlider(); - speed_slider_->SetDisplayType(FloatSlider::kPercentage); - connect(speed_slider_, &FloatSlider::ValueChanged, this, &SpeedDurationDialog::SpeedChanged); - layout->addWidget(speed_slider_, row, 1); + int row = 0; - row++; + speed_layout->addWidget(new QLabel(tr("Speed:")), row, 0); - layout->addWidget(new QLabel(tr("Duration:")), row, 0); + speed_slider_ = new FloatSlider(); + speed_slider_->SetDisplayType(FloatSlider::kPercentage); + connect(speed_slider_, &FloatSlider::ValueChanged, this, &SpeedDurationDialog::SpeedChanged); + speed_layout->addWidget(speed_slider_, row, 1); - dur_slider_ = new RationalSlider(); - dur_slider_->SetTimebase(timebase); - dur_slider_->SetDisplayType(RationalSlider::kTime); - connect(dur_slider_, &RationalSlider::ValueChanged, this, &SpeedDurationDialog::DurationChanged); - layout->addWidget(dur_slider_, row, 1); + row++; - row++; + speed_layout->addWidget(new QLabel(tr("Duration:")), row, 0); - link_box_ = new QCheckBox(tr("Link Speed and Duration")); - link_box_->setChecked(true); - layout->addWidget(link_box_, row, 0, 1, 2); + dur_slider_ = new RationalSlider(); + dur_slider_->SetTimebase(timebase); + dur_slider_->SetDisplayType(RationalSlider::kTime); + connect(dur_slider_, &RationalSlider::ValueChanged, this, &SpeedDurationDialog::DurationChanged); + speed_layout->addWidget(dur_slider_, row, 1); - row++; + row++; + + link_box_ = new QCheckBox(tr("Link Speed and Duration")); + link_box_->setChecked(true); + speed_layout->addWidget(link_box_, row, 0, 1, 2); + } + + reverse_box_ = new QCheckBox(tr("Reverse")); + layout->addWidget(reverse_box_); + + maintain_audio_pitch_box_ = new QCheckBox(tr("Maintain Audio Pitch")); + layout->addWidget(maintain_audio_pitch_box_); ripple_box_ = new QCheckBox(tr("Ripple Trailing Clips")); - layout->addWidget(ripple_box_, row, 0, 1, 2); - - row++; + layout->addWidget(ripple_box_); QDialogButtonBox *btns = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel); btns->setCenterButtons(true); connect(btns, &QDialogButtonBox::accepted, this, &SpeedDurationDialog::accept); connect(btns, &QDialogButtonBox::rejected, this, &SpeedDurationDialog::reject); - layout->addWidget(btns, row, 0, 1, 2); + layout->addWidget(btns); // Determine which speed value to use start_speed_ = clips.first()->speed(); start_duration_ = clips.first()->length(); + start_reverse_ = clips.first()->reverse(); + start_maintain_audio_pitch_ = clips.first()->maintain_audio_pitch(); for (int i=1; ispeed())) { // Speed differs per clip @@ -91,6 +103,14 @@ SpeedDurationDialog::SpeedDurationDialog(const QVector &clips, cons if (start_duration_ != -1 && clips.at(i)->length() != start_duration_) { start_duration_ = -1; } + + if (clips.at(i)->reverse() != start_reverse_) { + start_reverse_ = -1; + } + + if (clips.at(i)->maintain_audio_pitch() != start_maintain_audio_pitch_) { + start_maintain_audio_pitch_ = -1; + } } if (qIsNaN(start_speed_)) { @@ -104,6 +124,18 @@ SpeedDurationDialog::SpeedDurationDialog(const QVector &clips, cons } else { dur_slider_->SetValue(start_duration_); } + + if (start_reverse_ == -1) { + reverse_box_->setTristate(); + } else { + reverse_box_->setChecked(start_reverse_); + } + + if (start_maintain_audio_pitch_ == -1) { + maintain_audio_pitch_box_->setTristate(); + } else { + maintain_audio_pitch_box_->setChecked(start_maintain_audio_pitch_); + } } void SpeedDurationDialog::accept() @@ -133,6 +165,20 @@ void SpeedDurationDialog::accept() } } + // Set reverse values + if (!reverse_box_->isTristate()) { + foreach (ClipBlock *c, clips_) { + command->add_child(new NodeParamSetStandardValueCommand(NodeKeyframeTrackReference(NodeInput(c, ClipBlock::kReverseInput)), reverse_box_->isChecked())); + } + } + + // Set reverse values + if (!maintain_audio_pitch_box_->isTristate()) { + foreach (ClipBlock *c, clips_) { + command->add_child(new NodeParamSetStandardValueCommand(NodeKeyframeTrackReference(NodeInput(c, ClipBlock::kMaintainAudioPitchInput)), maintain_audio_pitch_box_->isChecked())); + } + } + // Set duration values foreach (ClipBlock *c, clips_) { rational proposed_length = c->length(); diff --git a/app/dialog/speedduration/speeddurationdialog.h b/app/dialog/speedduration/speeddurationdialog.h index 135f77434..be24ec9d6 100644 --- a/app/dialog/speedduration/speeddurationdialog.h +++ b/app/dialog/speedduration/speeddurationdialog.h @@ -56,8 +56,16 @@ private: QCheckBox *link_box_; + QCheckBox *reverse_box_; + + QCheckBox *maintain_audio_pitch_box_; + QCheckBox *ripple_box_; + int start_reverse_; + + int start_maintain_audio_pitch_; + double start_speed_; rational start_duration_; diff --git a/app/node/block/clip/clip.cpp b/app/node/block/clip/clip.cpp index d966098a0..efd46e316 100644 --- a/app/node/block/clip/clip.cpp +++ b/app/node/block/clip/clip.cpp @@ -33,6 +33,7 @@ const QString ClipBlock::kBufferIn = QStringLiteral("buffer_in"); const QString ClipBlock::kMediaInInput = QStringLiteral("media_in_in"); const QString ClipBlock::kSpeedInput = QStringLiteral("speed_in"); const QString ClipBlock::kReverseInput = QStringLiteral("reverse_in"); +const QString ClipBlock::kMaintainAudioPitchInput = QStringLiteral("maintain_audio_pitch_in"); ClipBlock::ClipBlock() : in_transition_(nullptr), @@ -52,6 +53,8 @@ ClipBlock::ClipBlock() : AddInput(kReverseInput, NodeValue::kBoolean, false, InputFlags(kInputFlagNotConnectable | kInputFlagNotKeyframable)); IgnoreHashingFrom(kReverseInput); + AddInput(kMaintainAudioPitchInput, NodeValue::kBoolean, false, InputFlags(kInputFlagNotConnectable | kInputFlagNotKeyframable)); + PrependInput(kBufferIn, NodeValue::kNone, InputFlags(kInputFlagNotKeyframable)); SetValueHintForInput(kBufferIn, ValueHint(NodeValue::kBuffer)); } diff --git a/app/node/block/clip/clip.h b/app/node/block/clip/clip.h index ee8eef2e8..82350d89f 100644 --- a/app/node/block/clip/clip.h +++ b/app/node/block/clip/clip.h @@ -71,6 +71,21 @@ public: return GetStandardValue(kReverseInput).toBool(); } + void set_reverse(bool e) + { + SetStandardValue(kReverseInput, e); + } + + bool maintain_audio_pitch() const + { + return GetStandardValue(kMaintainAudioPitchInput).toBool(); + } + + void set_maintain_audio_pitch(bool e) + { + SetStandardValue(kMaintainAudioPitchInput, e); + } + TransitionBlock* in_transition() { return in_transition_; @@ -110,6 +125,7 @@ public: static const QString kMediaInInput; static const QString kSpeedInput; static const QString kReverseInput; + static const QString kMaintainAudioPitchInput; protected: virtual void LinkChangeEvent() override; diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index 01bd95518..970b2f326 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -25,6 +25,9 @@ #include #include +#include "audio/packedprocessor.h" +#include "audio/planarprocessor.h" +#include "audio/tempoprocessor.h" #include "node/block/clip/clip.h" #include "node/block/transition/transition.h" #include "node/project/project.h" @@ -288,8 +291,35 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim // Just silence, don't think there's any other practical application of 0 speed audio samples_from_this_block->silence(); } else if (!qFuzzyCompare(speed_value, 1.0)) { - // Multiply time - samples_from_this_block->speed(speed_value); + if (clip_cast->maintain_audio_pitch()) { + PackedProcessor packer; + packer.Open(samples_from_this_block->audio_params()); + + QByteArray packed = packer.Convert(samples_from_this_block); + + if (!packed.isEmpty()) { + TempoProcessor tp; + tp.Open(samples_from_this_block->audio_params(), speed_value); + + // FIXME: This is not the best way to do this, the TempoProcessor works best + // when it's given a continuous stream of audio, which is challenging + // in our current "modular" audio system. This should still work reasonably + // well on export (assuming audio is all generated at once on export), but + // users may hear clicks and pops in the audio during preview due to this + // approach. + tp.Push(packed); + tp.Flush(); + packed = tp.Pull(); + tp.Close(); + + PlanarProcessor planar; + planar.Open(samples_from_this_block->audio_params()); + samples_from_this_block = planar.Convert(packed); + } + } else { + // Multiply time + samples_from_this_block->speed(speed_value); + } } if (reversed) { diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index d8827a7c0..206fe9719 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -466,11 +466,8 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() // If the tempo must be adjusted, adjust now if (tempo_processor_.IsOpen()) { - tempo_processor_.Push(pack.data(), pack.size()); - int actual = tempo_processor_.Pull(pack.data(), pack.size()); - if (actual != pack.size()) { - pack.resize(actual); - } + tempo_processor_.Push(pack); + pack = tempo_processor_.Pull(); } // TempoProcessor may have emptied the array