oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implemented maintaining audio pitch and reversing

Browse Source
This commit is contained in:
itsmattkc
2022-02-01 13:39:43 -08:00
parent efa74c72a8
commit 019b3cfe29
12 changed files with 361 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/audio/CMakeLists.txt
+2
View File
@@ -22,6 +22,8 @@ set(OLIVE_SOURCES
audio/audiovisualwaveform.h
audio/packedprocessor.cpp
audio/packedprocessor.h
audio/planarprocessor.cpp
audio/planarprocessor.h
audio/tempoprocessor.cpp
audio/tempoprocessor.h
PARENT_SCOPE
app/audio/packedprocessor.cpp
+3 -8
View File
@@ -76,17 +76,12 @@ QByteArray PackedProcessor::Convert(SampleBufferPtr planar)
return QByteArray();
}
int nb_channels = planar->audio_params().channel_count();
QByteArray output(planar->audio_params().samples_to_bytes(nb_samples), Qt::Uninitialized);
uint8_t *output_data = reinterpret_cast<uint8_t*>(output.data());
QVector<const uint8_t*> input_arrays(nb_channels);
for (int i=0; i<nb_channels; i++) {
input_arrays[i] = reinterpret_cast<const uint8_t*>(planar->data(i));
}
int ret = swr_convert(swr_ctx_, &output_data, nb_samples, input_arrays.data(), nb_samples);
int ret = swr_convert(swr_ctx_, &output_data, nb_samples,
const_cast<const uint8_t**>(reinterpret_cast<uint8_t**>(planar->to_raw_ptrs())),
nb_samples);
if (ret < 0) {
char buf[200];
av_strerror(ret, buf, 200);
app/audio/planarprocessor.cpp
+104
View File
@@ -0,0 +1,104 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "planarprocessor.h"
#include "common/ffmpegutils.h"
namespace olive {
PlanarProcessor::PlanarProcessor() :
swr_ctx_(nullptr)
{
}
PlanarProcessor::~PlanarProcessor()
{
Close();
}
bool PlanarProcessor::Open(const AudioParams &params)
{
if (IsOpen()) {
return true;
}
swr_ctx_ = swr_alloc_set_opts(nullptr,
params.channel_layout(),
FFmpegUtils::GetFFmpegSampleFormat(params.format(), true),
params.sample_rate(),
params.channel_layout(),
FFmpegUtils::GetFFmpegSampleFormat(params.format(), false),
params.sample_rate(),
0,
nullptr);
if (!swr_ctx_) {
qCritical() << "Failed to allocate resample context";
return false;
}
if (swr_init(swr_ctx_) < 0) {
qCritical() << "Failed to init resample context";
swr_free(&swr_ctx_);
return false;
}
params_ = params;
return true;
}
SampleBufferPtr PlanarProcessor::Convert(const QByteArray &packed)
{
if (!IsOpen()) {
qCritical() << "Tried to convert while closed";
return nullptr;
}
if (packed.isEmpty()) {
return nullptr;
}
int nb_samples_per_channel = params_.bytes_to_samples(packed.size());
SampleBufferPtr output = SampleBuffer::CreateAllocated(params_, nb_samples_per_channel);
const uint8_t *input = reinterpret_cast<const uint8_t*>(packed.constData());
int ret = swr_convert(swr_ctx_,
reinterpret_cast<uint8_t**>(output->to_raw_ptrs()), nb_samples_per_channel,
&input, nb_samples_per_channel);
if (ret < 0) {
char buf[200];
av_strerror(ret, buf, 200);
qDebug() << "Planar processor failed with error:" << buf << ret;
}
return output;
}
void PlanarProcessor::Close()
{
if (swr_ctx_) {
swr_free(&swr_ctx_);
}
}
}
app/audio/planarprocessor.h
+62
View File
@@ -0,0 +1,62 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef PLANARPROCESSOR_H
#define PLANARPROCESSOR_H
extern "C" {
#include <libswresample/swresample.h>
}
#include "codec/samplebuffer.h"
#include "render/audioparams.h"
namespace olive {
class PlanarProcessor
{
public:
PlanarProcessor();
~PlanarProcessor();
DISABLE_COPY_MOVE(PlanarProcessor)
bool Open(const AudioParams &params);
SampleBufferPtr Convert(const QByteArray &packed);
void Close();
bool IsOpen() const
{
return swr_ctx_;
}
private:
SwrContext *swr_ctx_;
AudioParams params_;
};
}
#endif // PLANARPROCESSOR_H
app/audio/tempoprocessor.cpp
+56 -68
View File
@@ -36,7 +36,6 @@ TempoProcessor::TempoProcessor() :
filter_graph_(nullptr),
buffersrc_ctx_(nullptr),
buffersink_ctx_(nullptr),
processed_frame_(nullptr),
open_(false)
{
}
@@ -150,47 +149,42 @@ bool TempoProcessor::Open(const AudioParams &params, const double& speed)
return true;
}
void TempoProcessor::Push(const char *data, int length)
void TempoProcessor::Push(const QByteArray &packed)
{
if (flushed_) {
if (length > 0) {
qCritical() << "Tried to push" << length << "bytes after TempoProcessor was closed";
}
if (!IsOpen()) {
qWarning() << "Tried to push to closed TempoProcessor";
return;
}
AVFrame* src_frame;
if (length == 0) {
// No audio data, flush the last out of the filter graph
src_frame = nullptr;
flushed_ = true;
} else {
src_frame = av_frame_alloc();
if (!src_frame) {
qCritical() << "Failed to allocate source frame";
return;
}
// Allocate a buffer for the number of samples we got
src_frame->sample_rate = params_.sample_rate();
src_frame->format = FFmpegUtils::GetFFmpegSampleFormat(params_.format());
src_frame->channel_layout = params_.channel_layout();
src_frame->nb_samples = params_.bytes_to_samples(length);
src_frame->pts = timestamp_;
timestamp_ += src_frame->nb_samples;
if (av_frame_get_buffer(src_frame, 0) < 0) {
qCritical() << "Failed to allocate buffer for source frame";
av_frame_free(&src_frame);
return;
}
// Copy buffer from data array to frame
memcpy(src_frame->data[0], data, static_cast<size_t>(length));
if (flushed_) {
qWarning() << "Tried to push to flushed TempoProcessor";
return;
}
AVFrame* src_frame = av_frame_alloc();
if (!src_frame) {
qCritical() << "Failed to allocate source frame";
return;
}
// Allocate a buffer for the number of samples we got
src_frame->sample_rate = params_.sample_rate();
src_frame->format = FFmpegUtils::GetFFmpegSampleFormat(params_.format());
src_frame->channel_layout = params_.channel_layout();
src_frame->nb_samples = params_.bytes_to_samples(packed.size());
src_frame->pts = timestamp_;
timestamp_ += src_frame->nb_samples;
if (av_frame_get_buffer(src_frame, 0) < 0) {
qCritical() << "Failed to allocate buffer for source frame";
av_frame_free(&src_frame);
return;
}
// Copy buffer from data array to frame
memcpy(src_frame->data[0], packed, packed.size());
int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, src_frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0) {
@@ -202,46 +196,45 @@ void TempoProcessor::Push(const char *data, int length)
}
}
int TempoProcessor::Pull(char *data, int max_length)
void TempoProcessor::Flush()
{
if (!processed_frame_) {
processed_frame_ = av_frame_alloc();
// Try to pull samples from the buffersink
int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame_);
if (!flushed_) {
int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0) {
// We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the
// error might be fatal...
if (ret != AVERROR(EAGAIN)) {
qCritical() << "Failed to pull from buffersink" << ret;
}
qCritical() << "Failed to feed buffer source" << ret;
}
flushed_ = true;
}
}
av_frame_free(&processed_frame_);
QByteArray TempoProcessor::Pull()
{
QByteArray b;
AVFrame *processed_frame = av_frame_alloc();
return 0;
// Try to pull samples from the buffersink
int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame);
if (ret < 0) {
// We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the
// error might be fatal...
if (ret != AVERROR(EAGAIN)) {
qCritical() << "Failed to pull from buffersink" << ret;
}
processed_frame_byte_index_ = 0;
processed_frame_max_bytes_ = params_.samples_to_bytes(processed_frame_->nb_samples);
av_frame_free(&processed_frame);
return b;
}
// Determine how many bytes we should copy into the data array
int copy_length = qMin(max_length, processed_frame_max_bytes_ - processed_frame_byte_index_);
b.resize(params_.samples_to_bytes(processed_frame->nb_samples));
// Copy the bytes
memcpy(data, processed_frame_->data[0] + processed_frame_byte_index_, static_cast<size_t>(copy_length));
// Add the copied amount to the current index
processed_frame_byte_index_ += copy_length;
memcpy(b.data(), processed_frame->data[0], b.size());
// If the index has reached the limit of this processed frame, we can dispose of the frame now
if (processed_frame_byte_index_ == processed_frame_max_bytes_) {
av_frame_free(&processed_frame_);
processed_frame_ = nullptr;
}
av_frame_free(&processed_frame);
return copy_length;
return b;
}
void TempoProcessor::Close()
@@ -253,11 +246,6 @@ void TempoProcessor::Close()
filter_graph_ = nullptr;
}
if (processed_frame_) {
av_frame_free(&processed_frame_);
processed_frame_ = nullptr;
}
buffersrc_ctx_ = nullptr;
buffersink_ctx_ = nullptr;
}
app/audio/tempoprocessor.h
+4 -6
View File
@@ -52,9 +52,11 @@ public:
bool Open(const AudioParams& params, const double &speed);
void Push(const char *data, int length);
void Push(const QByteArray &packed);
int Pull(char* data, int max_length);
void Flush();
QByteArray Pull();
void Close();
@@ -67,10 +69,6 @@ private:
AVFilterContext* buffersink_ctx_;
AVFrame* processed_frame_;
int processed_frame_byte_index_;
int processed_frame_max_bytes_;
AudioParams params_;
int64_t timestamp_;
app/dialog/speedduration/speeddurationdialog.cpp
+69 -23
View File
@@ -22,6 +22,7 @@
#include <QDialogButtonBox>
#include <QGridLayout>
#include <QGroupBox>
#include <QMessageBox>
#include "core.h"
@@ -39,49 +40,60 @@ SpeedDurationDialog::SpeedDurationDialog(const QVector<ClipBlock *> &clips, cons
{
setWindowTitle(tr("Speed/Duration"));
QGridLayout *layout = new QGridLayout(this);
QVBoxLayout *layout = new QVBoxLayout(this);
int row = 0;
{
QGroupBox *speed_group = new QGroupBox();
layout->addWidget(speed_group);
layout->addWidget(new QLabel(tr("Speed:")), row, 0);
QGridLayout *speed_layout = new QGridLayout(speed_group);
speed_slider_ = new FloatSlider();
speed_slider_->SetDisplayType(FloatSlider::kPercentage);
connect(speed_slider_, &FloatSlider::ValueChanged, this, &SpeedDurationDialog::SpeedChanged);
layout->addWidget(speed_slider_, row, 1);
int row = 0;
row++;
speed_layout->addWidget(new QLabel(tr("Speed:")), row, 0);
layout->addWidget(new QLabel(tr("Duration:")), row, 0);
speed_slider_ = new FloatSlider();
speed_slider_->SetDisplayType(FloatSlider::kPercentage);
connect(speed_slider_, &FloatSlider::ValueChanged, this, &SpeedDurationDialog::SpeedChanged);
speed_layout->addWidget(speed_slider_, row, 1);
dur_slider_ = new RationalSlider();
dur_slider_->SetTimebase(timebase);
dur_slider_->SetDisplayType(RationalSlider::kTime);
connect(dur_slider_, &RationalSlider::ValueChanged, this, &SpeedDurationDialog::DurationChanged);
layout->addWidget(dur_slider_, row, 1);
row++;
row++;
speed_layout->addWidget(new QLabel(tr("Duration:")), row, 0);
link_box_ = new QCheckBox(tr("Link Speed and Duration"));
link_box_->setChecked(true);
layout->addWidget(link_box_, row, 0, 1, 2);
dur_slider_ = new RationalSlider();
dur_slider_->SetTimebase(timebase);
dur_slider_->SetDisplayType(RationalSlider::kTime);
connect(dur_slider_, &RationalSlider::ValueChanged, this, &SpeedDurationDialog::DurationChanged);
speed_layout->addWidget(dur_slider_, row, 1);
row++;
row++;
link_box_ = new QCheckBox(tr("Link Speed and Duration"));
link_box_->setChecked(true);
speed_layout->addWidget(link_box_, row, 0, 1, 2);
}
reverse_box_ = new QCheckBox(tr("Reverse"));
layout->addWidget(reverse_box_);
maintain_audio_pitch_box_ = new QCheckBox(tr("Maintain Audio Pitch"));
layout->addWidget(maintain_audio_pitch_box_);
ripple_box_ = new QCheckBox(tr("Ripple Trailing Clips"));
layout->addWidget(ripple_box_, row, 0, 1, 2);
row++;
layout->addWidget(ripple_box_);
QDialogButtonBox *btns = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel);
btns->setCenterButtons(true);
connect(btns, &QDialogButtonBox::accepted, this, &SpeedDurationDialog::accept);
connect(btns, &QDialogButtonBox::rejected, this, &SpeedDurationDialog::reject);
layout->addWidget(btns, row, 0, 1, 2);
layout->addWidget(btns);
// Determine which speed value to use
start_speed_ = clips.first()->speed();
start_duration_ = clips.first()->length();
start_reverse_ = clips.first()->reverse();
start_maintain_audio_pitch_ = clips.first()->maintain_audio_pitch();
for (int i=1; i<clips.size(); i++) {
if (!qIsNaN(start_speed_) && !qFuzzyCompare(start_speed_, clips.at(i)->speed())) {
// Speed differs per clip
@@ -91,6 +103,14 @@ SpeedDurationDialog::SpeedDurationDialog(const QVector<ClipBlock *> &clips, cons
if (start_duration_ != -1 && clips.at(i)->length() != start_duration_) {
start_duration_ = -1;
}
if (clips.at(i)->reverse() != start_reverse_) {
start_reverse_ = -1;
}
if (clips.at(i)->maintain_audio_pitch() != start_maintain_audio_pitch_) {
start_maintain_audio_pitch_ = -1;
}
}
if (qIsNaN(start_speed_)) {
@@ -104,6 +124,18 @@ SpeedDurationDialog::SpeedDurationDialog(const QVector<ClipBlock *> &clips, cons
} else {
dur_slider_->SetValue(start_duration_);
}
if (start_reverse_ == -1) {
reverse_box_->setTristate();
} else {
reverse_box_->setChecked(start_reverse_);
}
if (start_maintain_audio_pitch_ == -1) {
maintain_audio_pitch_box_->setTristate();
} else {
maintain_audio_pitch_box_->setChecked(start_maintain_audio_pitch_);
}
}
void SpeedDurationDialog::accept()
@@ -133,6 +165,20 @@ void SpeedDurationDialog::accept()
}
}
// Set reverse values
if (!reverse_box_->isTristate()) {
foreach (ClipBlock *c, clips_) {
command->add_child(new NodeParamSetStandardValueCommand(NodeKeyframeTrackReference(NodeInput(c, ClipBlock::kReverseInput)), reverse_box_->isChecked()));
}
}
// Set reverse values
if (!maintain_audio_pitch_box_->isTristate()) {
foreach (ClipBlock *c, clips_) {
command->add_child(new NodeParamSetStandardValueCommand(NodeKeyframeTrackReference(NodeInput(c, ClipBlock::kMaintainAudioPitchInput)), maintain_audio_pitch_box_->isChecked()));
}
}
// Set duration values
foreach (ClipBlock *c, clips_) {
rational proposed_length = c->length();
app/dialog/speedduration/speeddurationdialog.h
+8
View File
@@ -56,8 +56,16 @@ private:
QCheckBox *link_box_;
QCheckBox *reverse_box_;
QCheckBox *maintain_audio_pitch_box_;
QCheckBox *ripple_box_;
int start_reverse_;
int start_maintain_audio_pitch_;
double start_speed_;
rational start_duration_;
app/node/block/clip/clip.cpp
+3
View File
@@ -33,6 +33,7 @@ const QString ClipBlock::kBufferIn = QStringLiteral("buffer_in");
const QString ClipBlock::kMediaInInput = QStringLiteral("media_in_in");
const QString ClipBlock::kSpeedInput = QStringLiteral("speed_in");
const QString ClipBlock::kReverseInput = QStringLiteral("reverse_in");
const QString ClipBlock::kMaintainAudioPitchInput = QStringLiteral("maintain_audio_pitch_in");
ClipBlock::ClipBlock() :
in_transition_(nullptr),
@@ -52,6 +53,8 @@ ClipBlock::ClipBlock() :
AddInput(kReverseInput, NodeValue::kBoolean, false, InputFlags(kInputFlagNotConnectable | kInputFlagNotKeyframable));
IgnoreHashingFrom(kReverseInput);
AddInput(kMaintainAudioPitchInput, NodeValue::kBoolean, false, InputFlags(kInputFlagNotConnectable | kInputFlagNotKeyframable));
PrependInput(kBufferIn, NodeValue::kNone, InputFlags(kInputFlagNotKeyframable));
SetValueHintForInput(kBufferIn, ValueHint(NodeValue::kBuffer));
}
app/node/block/clip/clip.h
+16
View File
@@ -71,6 +71,21 @@ public:
return GetStandardValue(kReverseInput).toBool();
}
void set_reverse(bool e)
{
SetStandardValue(kReverseInput, e);
}
bool maintain_audio_pitch() const
{
return GetStandardValue(kMaintainAudioPitchInput).toBool();
}
void set_maintain_audio_pitch(bool e)
{
SetStandardValue(kMaintainAudioPitchInput, e);
}
TransitionBlock* in_transition()
{
return in_transition_;
@@ -110,6 +125,7 @@ public:
static const QString kMediaInInput;
static const QString kSpeedInput;
static const QString kReverseInput;
static const QString kMaintainAudioPitchInput;
protected:
virtual void LinkChangeEvent() override;
app/render/renderprocessor.cpp
+32 -2
View File
@@ -25,6 +25,9 @@
#include <QVector3D>
#include <QVector4D>
#include "audio/packedprocessor.h"
#include "audio/planarprocessor.h"
#include "audio/tempoprocessor.h"
#include "node/block/clip/clip.h"
#include "node/block/transition/transition.h"
#include "node/project/project.h"
@@ -288,8 +291,35 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim
// Just silence, don't think there's any other practical application of 0 speed audio
samples_from_this_block->silence();
} else if (!qFuzzyCompare(speed_value, 1.0)) {
// Multiply time
samples_from_this_block->speed(speed_value);
if (clip_cast->maintain_audio_pitch()) {
PackedProcessor packer;
packer.Open(samples_from_this_block->audio_params());
QByteArray packed = packer.Convert(samples_from_this_block);
if (!packed.isEmpty()) {
TempoProcessor tp;
tp.Open(samples_from_this_block->audio_params(), speed_value);
// FIXME: This is not the best way to do this, the TempoProcessor works best
// when it's given a continuous stream of audio, which is challenging
// in our current "modular" audio system. This should still work reasonably
// well on export (assuming audio is all generated at once on export), but
// users may hear clicks and pops in the audio during preview due to this
// approach.
tp.Push(packed);
tp.Flush();
packed = tp.Pull();
tp.Close();
PlanarProcessor planar;
planar.Open(samples_from_this_block->audio_params());
samples_from_this_block = planar.Convert(packed);
}
} else {
// Multiply time
samples_from_this_block->speed(speed_value);
}
}
if (reversed) {
app/widget/viewer/viewer.cpp
+2 -5
View File
@@ -466,11 +466,8 @@ void ViewerWidget::ReceivedAudioBufferForPlayback()
// If the tempo must be adjusted, adjust now
if (tempo_processor_.IsOpen()) {
tempo_processor_.Push(pack.data(), pack.size());
int actual = tempo_processor_.Pull(pack.data(), pack.size());
if (actual != pack.size()) {
pack.resize(actual);
}
tempo_processor_.Push(pack);
pack = tempo_processor_.Pull();
}
// TempoProcessor may have emptied the array