oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

audio: merged all processors into single versatile object

Browse Source
This commit is contained in:
itsmattkc
2022-05-05 15:28:32 -07:00
parent 8ba1b231a3
commit 9ec42c792a
14 changed files with 426 additions and 686 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/audio/CMakeLists.txt
+2 -6
View File
@@ -18,13 +18,9 @@ set(OLIVE_SOURCES
${OLIVE_SOURCES}
audio/audiomanager.cpp
audio/audiomanager.h
audio/audioprocessor.cpp
audio/audioprocessor.h
audio/audiovisualwaveform.cpp
audio/audiovisualwaveform.h
audio/packedprocessor.cpp
audio/packedprocessor.h
audio/planarprocessor.cpp
audio/planarprocessor.h
audio/tempoprocessor.cpp
audio/tempoprocessor.h
PARENT_SCOPE
)
app/audio/audiomanager.cpp
-1
View File
@@ -26,7 +26,6 @@
#include <QApplication>
#include "audio/packedprocessor.h"
#include "config/config.h"
namespace olive {
app/audio/audiomanager.h
+1
View File
@@ -104,6 +104,7 @@ private:
PaDeviceIndex input_device_;
PaStream *input_stream_;
FFmpegEncoder *input_encoder_;
};
app/audio/audioprocessor.cpp
+302
View File
@@ -0,0 +1,302 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "audioprocessor.h"
extern "C" {
#include <libavfilter/buffersrc.h>
#include <libavfilter/buffersink.h>
}
#include "common/ffmpegutils.h"
namespace olive {
AudioProcessor::AudioProcessor()
{
filter_graph_ = nullptr;
in_frame_ = nullptr;
out_frame_ = nullptr;
}
AudioProcessor::~AudioProcessor()
{
Close();
}
bool AudioProcessor::Open(const AudioParams &from, const AudioParams &to, double tempo)
{
if (filter_graph_) {
qWarning() << "Tried to open a processor that was already open";
return false;
}
filter_graph_ = avfilter_graph_alloc();
if (!filter_graph_) {
qCritical() << "Failed to allocate filter graph";
return false;
}
from_fmt_ = FFmpegUtils::GetFFmpegSampleFormat(from.format());
to_fmt_ = FFmpegUtils::GetFFmpegSampleFormat(to.format());
// Set up audio buffer args
char filter_args[200];
snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64,
1,
from.sample_rate(),
from.sample_rate(),
from_fmt_,
from.channel_layout());
int r;
// Create buffersrc (input)
r = avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_);
if (r < 0) {
qCritical() << "Failed to create buffersrc:" << r;
Close();
return false;
}
// Store "previous" filter for linking
AVFilterContext *previous_filter = buffersrc_ctx_;
// Create tempo
bool create_tempo;
if ((create_tempo = !qFuzzyCompare(tempo, 1.0))) {
// Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside
// those boundaries, we need to daisychain more than one together.
double base = (tempo > 1.0) ? 2.0 : 0.5;
double speed_log = log(tempo) / log(base);
// This is the number of how many 0.5 or 2.0 tempos we need to daisychain
int whole = qFloor(speed_log);
// Set speed_log to the remainder
speed_log -= whole;
for (int i=0;i<=whole;i++) {
double filter_tempo = (i == whole) ? qPow(base, speed_log) : base;
if (qFuzzyCompare(filter_tempo, 1.0)) {
// This filter would do nothing
continue;
}
previous_filter = CreateTempoFilter(filter_graph_,
previous_filter,
filter_tempo);
if (!previous_filter) {
qCritical() << "Failed to create audio tempo filter";
Close();
return false;
}
}
}
// Create conversion filter
if (from.sample_rate() != to.sample_rate() || from.channel_layout() != to.channel_layout() || from.format() != to.format()
|| (to.FormatIsPlanar() && create_tempo)) { // Tempo processor automatically converts to packed,
// so if the desired output is planar, it'll need
// to be converted
snprintf(filter_args, 200, "sample_fmts=%s:sample_rates=%d:channel_layouts=0x%" PRIx64,
av_get_sample_fmt_name(to_fmt_),
to.sample_rate(),
to.channel_layout());
AVFilterContext *c;
r = avfilter_graph_create_filter(&c, avfilter_get_by_name("aformat"), "fmt", filter_args, nullptr, filter_graph_);
if (r < 0) {
qCritical() << "Failed to create format conversion filter:" << r << filter_args;
Close();
return false;
}
r = avfilter_link(previous_filter, 0, c, 0);
if (r < 0) {
qCritical() << "Failed to link filters:" << r;
Close();
return false;
}
previous_filter = c;
}
// Create buffersink (output)
r = avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_);
if (r < 0) {
qCritical() << "Failed to create buffersink:" << r;
Close();
return false;
}
r = avfilter_link(previous_filter, 0, buffersink_ctx_, 0);
if (r < 0) {
qCritical() << "Failed to link filters:" << r;
Close();
return false;
}
r = avfilter_graph_config(filter_graph_, nullptr);
if (r < 0) {
qCritical() << "Failed to configure graph:" << r;
Close();
return false;
}
in_frame_ = av_frame_alloc();
if (in_frame_) {
in_frame_->sample_rate = from.sample_rate();
in_frame_->format = from_fmt_;
in_frame_->channel_layout = from.channel_layout();
in_frame_->channels = from.channel_count();
in_frame_->pts = 0;
} else {
qCritical() << "Failed to allocate input frame";
Close();
return false;
}
out_frame_ = av_frame_alloc();
if (!out_frame_) {
qCritical() << "Failed to allocate output frame";
Close();
return false;
}
from_ = from;
to_ = to;
return true;
}
void AudioProcessor::Close()
{
if (filter_graph_) {
avfilter_graph_free(&filter_graph_);
filter_graph_ = nullptr;
buffersrc_ctx_ = nullptr;
buffersink_ctx_ = nullptr;
}
if (in_frame_) {
av_frame_free(&in_frame_);
in_frame_ = nullptr;
}
if (out_frame_) {
av_frame_free(&out_frame_);
out_frame_ = nullptr;
}
}
int AudioProcessor::Convert(float **in, int nb_in_samples, AudioProcessor::Buffer *output)
{
if (!IsOpen()) {
qCritical() << "Tried to convert on closed processor";
return -1;
}
int r = 0;
if (in && nb_in_samples) {
// Set frame parameters
in_frame_->nb_samples = nb_in_samples;
for (int i=0; i<from_.channel_count(); i++) {
in_frame_->data[i] = reinterpret_cast<uint8_t*>(in[i]);
in_frame_->linesize[i] = from_.samples_to_bytes(nb_in_samples);
}
r = av_buffersrc_add_frame_flags(buffersrc_ctx_, in_frame_, AV_BUFFERSRC_FLAG_KEEP_REF);
if (r < 0) {
qCritical() << "Failed to add frame to buffersrc:" << r;
return r;
}
}
if (output) {
int nb_channels = to_.channel_count();
if (to_.FormatIsPacked()) {
nb_channels = 1;
}
AudioProcessor::Buffer &result = *output;
result.resize(nb_channels);
int byte_offset = 0;
while (true) {
av_frame_unref(out_frame_);
r = av_buffersink_get_frame(buffersink_ctx_, out_frame_);
if (r < 0) {
if (r == AVERROR(EAGAIN)) {
r = 0;
} else {
// Handle unexpected error
qCritical() << "Failed to pull from buffersink:" << r;
}
break;
}
int nb_bytes = out_frame_->nb_samples * to_.bytes_per_sample_per_channel();
if (to_.FormatIsPacked()) {
nb_bytes *= to_.channel_count();
}
for (int i=0; i<nb_channels; i++) {
result[i].resize(byte_offset + nb_bytes);
memcpy(result[i].data() + byte_offset, out_frame_->data[i], nb_bytes);
}
byte_offset += nb_bytes;
}
av_frame_unref(out_frame_);
}
return r;
}
void AudioProcessor::Flush()
{
int r = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF);
if (r < 0) {
qCritical() << "Failed to flush:" << r;
}
}
AVFilterContext *AudioProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo)
{
// Set up tempo param, which is taken as a C string
char speed_param[20];
snprintf(speed_param, 20, "%f", tempo);
AVFilterContext* tempo_ctx = nullptr;
if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0
&& avfilter_link(link, 0, tempo_ctx, 0) == 0) {
return tempo_ctx;
}
return nullptr;
}
}
app/audio/tempoprocessor.h → app/audio/audioprocessor.h
+21 -29
View File
@@ -18,14 +18,8 @@
***/
#ifndef TEMPOPROCESSOR_H
#define TEMPOPROCESSOR_H
#ifdef __MINGW32__
#ifndef __USE_MINGW_ANSI_STDIO
#define __USE_MINGW_ANSI_STDIO
#endif
#endif
#ifndef AUDIOPROCESSOR_H
#define AUDIOPROCESSOR_H
#include <inttypes.h>
@@ -37,29 +31,26 @@ extern "C" {
namespace olive {
class TempoProcessor
class AudioProcessor
{
public:
TempoProcessor();
AudioProcessor();
~TempoProcessor();
~AudioProcessor();
DISABLE_COPY_MOVE(TempoProcessor)
DISABLE_COPY_MOVE(AudioProcessor)
bool IsOpen() const;
const double& GetSpeed() const;
bool Open(const AudioParams& params, const double &speed);
void Push(const QByteArray &packed);
void Flush();
QByteArray Pull();
bool Open(const AudioParams &from, const AudioParams &to, double tempo = 1.0);
void Close();
bool IsOpen() const { return filter_graph_; }
using Buffer = QVector<QByteArray>;
int Convert(float **in, int nb_in_samples, AudioProcessor::Buffer *output);
void Flush();
private:
static AVFilterContext* CreateTempoFilter(AVFilterGraph *graph, AVFilterContext *link, const double& tempo);
@@ -69,17 +60,18 @@ private:
AVFilterContext* buffersink_ctx_;
AudioParams params_;
AudioParams from_;
AVSampleFormat from_fmt_;
int64_t timestamp_;
AudioParams to_;
AVSampleFormat to_fmt_;
double speed_;
AVFrame *in_frame_;
bool open_;
AVFrame *out_frame_;
bool flushed_;
};
}
#endif // TEMPOPROCESSOR_H
#endif // AUDIOPROCESSOR_H
app/audio/packedprocessor.cpp
-101
View File
@@ -1,101 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "packedprocessor.h"
#include "common/ffmpegutils.h"
namespace olive {
PackedProcessor::PackedProcessor() :
swr_ctx_(nullptr)
{
}
PackedProcessor::~PackedProcessor()
{
Close();
}
bool PackedProcessor::Open(const AudioParams &params)
{
if (IsOpen()) {
return true;
}
swr_ctx_ = swr_alloc_set_opts(nullptr,
params.channel_layout(),
FFmpegUtils::GetFFmpegSampleFormat(AudioParams::GetPackedEquivalent(params.format())),
params.sample_rate(),
params.channel_layout(),
FFmpegUtils::GetFFmpegSampleFormat(params.format()),
params.sample_rate(),
0,
nullptr);
if (!swr_ctx_) {
qCritical() << "Failed to allocate resample context";
return false;
}
if (swr_init(swr_ctx_) < 0) {
qCritical() << "Failed to init resample context";
swr_free(&swr_ctx_);
return false;
}
return true;
}
QByteArray PackedProcessor::Convert(SampleBufferPtr planar)
{
if (!IsOpen()) {
qCritical() << "Tried to convert while closed";
return QByteArray();
}
int nb_samples = planar->sample_count();
if (nb_samples == 0) {
return QByteArray();
}
QByteArray output(planar->audio_params().samples_to_bytes(nb_samples), Qt::Uninitialized);
uint8_t *output_data = reinterpret_cast<uint8_t*>(output.data());
int ret = swr_convert(swr_ctx_, &output_data, nb_samples,
const_cast<const uint8_t**>(reinterpret_cast<uint8_t**>(planar->to_raw_ptrs())),
nb_samples);
if (ret < 0) {
char buf[200];
av_strerror(ret, buf, 200);
qDebug() << "Packed processor failed with error:" << buf << ret;
}
return output;
}
void PackedProcessor::Close()
{
if (swr_ctx_) {
swr_free(&swr_ctx_);
}
}
}
app/audio/packedprocessor.h
-60
View File
@@ -1,60 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef PACKEDPROCESSOR_H
#define PACKEDPROCESSOR_H
extern "C" {
#include <libswresample/swresample.h>
}
#include "codec/samplebuffer.h"
#include "render/audioparams.h"
namespace olive {
class PackedProcessor
{
public:
PackedProcessor();
~PackedProcessor();
DISABLE_COPY_MOVE(PackedProcessor)
bool Open(const AudioParams &params);
QByteArray Convert(SampleBufferPtr planar);
void Close();
bool IsOpen() const
{
return swr_ctx_;
}
private:
SwrContext *swr_ctx_;
};
}
#endif // PACKEDPROCESSOR_H
app/audio/planarprocessor.cpp
-104
View File
@@ -1,104 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "planarprocessor.h"
#include "common/ffmpegutils.h"
namespace olive {
PlanarProcessor::PlanarProcessor() :
swr_ctx_(nullptr)
{
}
PlanarProcessor::~PlanarProcessor()
{
Close();
}
bool PlanarProcessor::Open(const AudioParams &params)
{
if (IsOpen()) {
return true;
}
swr_ctx_ = swr_alloc_set_opts(nullptr,
params.channel_layout(),
FFmpegUtils::GetFFmpegSampleFormat(AudioParams::GetPlanarEquivalent(params.format())),
params.sample_rate(),
params.channel_layout(),
FFmpegUtils::GetFFmpegSampleFormat(params.format()),
params.sample_rate(),
0,
nullptr);
if (!swr_ctx_) {
qCritical() << "Failed to allocate resample context";
return false;
}
if (swr_init(swr_ctx_) < 0) {
qCritical() << "Failed to init resample context";
swr_free(&swr_ctx_);
return false;
}
params_ = params;
return true;
}
SampleBufferPtr PlanarProcessor::Convert(const QByteArray &packed)
{
if (!IsOpen()) {
qCritical() << "Tried to convert while closed";
return nullptr;
}
if (packed.isEmpty()) {
return nullptr;
}
int nb_samples_per_channel = params_.bytes_to_samples(packed.size());
SampleBufferPtr output = SampleBuffer::CreateAllocated(params_, nb_samples_per_channel);
const uint8_t *input = reinterpret_cast<const uint8_t*>(packed.constData());
int ret = swr_convert(swr_ctx_,
reinterpret_cast<uint8_t**>(output->to_raw_ptrs()), nb_samples_per_channel,
&input, nb_samples_per_channel);
if (ret < 0) {
char buf[200];
av_strerror(ret, buf, 200);
qDebug() << "Planar processor failed with error:" << buf << ret;
}
return output;
}
void PlanarProcessor::Close()
{
if (swr_ctx_) {
swr_free(&swr_ctx_);
}
}
}
app/audio/planarprocessor.h
-62
View File
@@ -1,62 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef PLANARPROCESSOR_H
#define PLANARPROCESSOR_H
extern "C" {
#include <libswresample/swresample.h>
}
#include "codec/samplebuffer.h"
#include "render/audioparams.h"
namespace olive {
class PlanarProcessor
{
public:
PlanarProcessor();
~PlanarProcessor();
DISABLE_COPY_MOVE(PlanarProcessor)
bool Open(const AudioParams &params);
SampleBufferPtr Convert(const QByteArray &packed);
void Close();
bool IsOpen() const
{
return swr_ctx_;
}
private:
SwrContext *swr_ctx_;
AudioParams params_;
};
}
#endif // PLANARPROCESSOR_H
app/audio/tempoprocessor.cpp
-269
View File
@@ -1,269 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2021 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "tempoprocessor.h"
extern "C" {
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/opt.h>
}
#include <QDebug>
#include "common/ffmpegutils.h"
namespace olive {
TempoProcessor::TempoProcessor() :
filter_graph_(nullptr),
buffersrc_ctx_(nullptr),
buffersink_ctx_(nullptr),
open_(false)
{
}
TempoProcessor::~TempoProcessor()
{
Close();
}
bool TempoProcessor::IsOpen() const
{
return open_;
}
const double &TempoProcessor::GetSpeed() const
{
return speed_;
}
bool TempoProcessor::Open(const AudioParams &params, const double& speed)
{
if (open_) {
return true;
}
params_ = params;
speed_ = speed;
// Create AVFilterGraph instance
filter_graph_ = avfilter_graph_alloc();
if (!filter_graph_) {
qCritical() << "Failed to create AVFilterGraph";
Close();
return false;
}
// Set up audio buffer args
char filter_args[200];
snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64,
1,
params_.sample_rate(),
params_.sample_rate(),
FFmpegUtils::GetFFmpegSampleFormat(params_.format()),
params.channel_layout());
// Create buffer and buffersink
if (avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_) < 0) {
qCritical() << "Failed to create audio buffer source";
Close();
return false;
}
if (avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_) < 0) {
qCritical() << "Failed to create audio buffer sink";
Close();
return false;
}
// Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside
// those boundaries, we need to daisychain more than one together.
double base = (speed_ > 1.0) ? 2.0 : 0.5;
double speed_log = log(speed_) / log(base);
// This is the number of how many 0.5 or 2.0 tempos we need to daisychain
int whole = qFloor(speed_log);
// Set speed_log to the remainder
speed_log -= whole;
AVFilterContext* previous_filter = buffersrc_ctx_;
for (int i=0;i<=whole;i++) {
double filter_tempo = (i == whole) ? qPow(base, speed_log) : base;
if (qFuzzyCompare(filter_tempo, 1.0)) {
// This filter would do nothing
continue;
}
previous_filter = CreateTempoFilter(filter_graph_,
previous_filter,
filter_tempo);
if (!previous_filter) {
qCritical() << "Failed to create audio tempo filter";
Close();
return false;
}
}
// Link the last filter to the buffersink
if (avfilter_link(previous_filter, 0, buffersink_ctx_, 0) != 0) {
qCritical() << "Failed to link final filter and buffer sink";
Close();
return false;
}
// Config graph
if (avfilter_graph_config(filter_graph_, nullptr) < 0) {
qCritical() << "Failed to configure filter graph";
Close();
return false;
}
timestamp_ = 0;
open_ = true;
flushed_ = false;
return true;
}
void TempoProcessor::Push(const QByteArray &packed)
{
if (!IsOpen()) {
qWarning() << "Tried to push to closed TempoProcessor";
return;
}
if (flushed_) {
qWarning() << "Tried to push to flushed TempoProcessor";
return;
}
AVFrame* src_frame = av_frame_alloc();
if (!src_frame) {
qCritical() << "Failed to allocate source frame";
return;
}
// Allocate a buffer for the number of samples we got
src_frame->sample_rate = params_.sample_rate();
src_frame->format = FFmpegUtils::GetFFmpegSampleFormat(params_.format());
src_frame->channel_layout = params_.channel_layout();
src_frame->nb_samples = params_.bytes_to_samples(packed.size());
src_frame->pts = timestamp_;
timestamp_ += src_frame->nb_samples;
if (av_frame_get_buffer(src_frame, 0) < 0) {
qCritical() << "Failed to allocate buffer for source frame";
av_frame_free(&src_frame);
return;
}
// Copy buffer from data array to frame
memcpy(src_frame->data[0], packed, packed.size());
int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, src_frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0) {
qCritical() << "Failed to feed buffer source" << ret;
}
if (src_frame) {
av_frame_free(&src_frame);
}
}
void TempoProcessor::Flush()
{
if (!flushed_) {
int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0) {
qCritical() << "Failed to feed buffer source" << ret;
}
flushed_ = true;
}
}
QByteArray TempoProcessor::Pull()
{
QByteArray b;
AVFrame *processed_frame = av_frame_alloc();
// Try to pull samples from the buffersink
int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame);
if (ret < 0) {
// We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the
// error might be fatal...
if (ret != AVERROR(EAGAIN)) {
qCritical() << "Failed to pull from buffersink" << ret;
}
av_frame_free(&processed_frame);
return b;
}
b.resize(params_.samples_to_bytes(processed_frame->nb_samples));
// Copy the bytes
memcpy(b.data(), processed_frame->data[0], b.size());
// If the index has reached the limit of this processed frame, we can dispose of the frame now
av_frame_free(&processed_frame);
return b;
}
void TempoProcessor::Close()
{
open_ = false;
if (filter_graph_) {
avfilter_graph_free(&filter_graph_);
filter_graph_ = nullptr;
}
buffersrc_ctx_ = nullptr;
buffersink_ctx_ = nullptr;
}
AVFilterContext *TempoProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo)
{
// Set up tempo param, which is taken as a C string
char speed_param[20];
snprintf(speed_param, 20, "%f", tempo);
AVFilterContext* tempo_ctx = nullptr;
if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0
&& avfilter_link(link, 0, tempo_ctx, 0) == 0) {
return tempo_ctx;
}
return nullptr;
}
}
app/render/audioparams.h
+20
View File
@@ -186,6 +186,26 @@ public:
duration_ = duration;
}
static bool FormatIsPacked(Format f)
{
return f >= kPackedStart && f < kPackedEnd;
}
bool FormatIsPacked() const
{
return FormatIsPacked(format_);
}
static bool FormatIsPlanar(Format f)
{
return f >= kPlanarStart && f < kPlanarEnd;
}
bool FormatIsPlanar() const
{
return FormatIsPlanar(format_);
}
qint64 time_to_bytes(const double& time) const;
qint64 time_to_bytes(const rational& time) const;
qint64 time_to_bytes_per_channel(const double& time) const;
app/render/renderprocessor.cpp
+28 -18
View File
@@ -25,9 +25,7 @@
#include <QVector3D>
#include <QVector4D>
#include "audio/packedprocessor.h"
#include "audio/planarprocessor.h"
#include "audio/tempoprocessor.h"
#include "audio/audioprocessor.h"
#include "node/block/clip/clip.h"
#include "node/block/transition/transition.h"
#include "node/project/project.h"
@@ -300,14 +298,10 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim
samples_from_this_block->silence();
} else if (!qFuzzyCompare(speed_value, 1.0)) {
if (clip_cast->maintain_audio_pitch()) {
PackedProcessor packer;
packer.Open(samples_from_this_block->audio_params());
AudioProcessor processor;
QByteArray packed = packer.Convert(samples_from_this_block);
if (!packed.isEmpty()) {
TempoProcessor tp;
tp.Open(samples_from_this_block->audio_params(), speed_value);
if (processor.Open(samples_from_this_block->audio_params(), samples_from_this_block->audio_params(), speed_value)) {
AudioProcessor::Buffer out;
// FIXME: This is not the best way to do this, the TempoProcessor works best
// when it's given a continuous stream of audio, which is challenging
@@ -315,15 +309,31 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim
// well on export (assuming audio is all generated at once on export), but
// users may hear clicks and pops in the audio during preview due to this
// approach.
tp.Push(packed);
tp.Flush();
packed = tp.Pull();
tp.Close();
int r = processor.Convert(samples_from_this_block->to_raw_ptrs(), samples_from_this_block->sample_count(), nullptr);
if (!packed.isEmpty()) {
PlanarProcessor planar;
planar.Open(samples_from_this_block->audio_params());
samples_from_this_block = planar.Convert(packed);
if (r < 0) {
qCritical() << "Failed to change tempo of audio:" << r;
} else {
processor.Flush();
processor.Convert(nullptr, 0, &out);
if (!out.empty()) {
int nb_samples = out.front().size() * samples_from_this_block->audio_params().bytes_per_sample_per_channel();
if (nb_samples) {
SampleBufferPtr new_samples = SampleBuffer::Create();
new_samples->set_audio_params(samples_from_this_block->audio_params());
new_samples->set_sample_count(nb_samples);
new_samples->allocate();
for (int i=0; i<out.size(); i++) {
memcpy(new_samples->data(i), out[i].data(), out[i].size());
}
samples_from_this_block = new_samples;
}
}
}
}
} else {
app/widget/viewer/viewer.cpp
+47 -32
View File
@@ -210,8 +210,7 @@ void ViewerWidget::ConnectNodeEvent(ViewerOutput *n)
last_length_ = 0;
LengthChangedSlot(n->GetLength());
AudioParams ap = n->GetAudioParams();
packed_processor_.Open(ap);
UpdateAudioProcessor();
ColorManager* color_manager = n->project()->color_manager();
@@ -245,7 +244,7 @@ void ViewerWidget::DisconnectNodeEvent(ViewerOutput *n)
disconnect(n->video_frame_cache(), &FrameHashCache::Shifted, this, &ViewerWidget::ViewerShiftedRange);
disconnect(n, &ViewerOutput::TextureInputChanged, this, &ViewerWidget::UpdateStack);
packed_processor_.Close();
CloseAudioProcessor();
SetDisplayImage(QVariant());
@@ -456,6 +455,21 @@ void ViewerWidget::DisarmRecording()
record_armed_ = false;
}
void ViewerWidget::UpdateAudioProcessor()
{
audio_processor_.Close();
AudioParams ap = GetConnectedNode()->GetAudioParams();
AudioParams packed = ap;
packed.set_format(AudioParams::GetPackedEquivalent(ap.format()));
audio_processor_.Open(ap, packed, (playback_speed_ == 0) ? 1 : std::abs(playback_speed_));
}
void ViewerWidget::CloseAudioProcessor()
{
audio_processor_.Close();
}
void ViewerWidget::QueueNextAudioBuffer()
{
rational queue_end = audio_playback_queue_time_ + (kAudioPlaybackInterval * playback_speed_);
@@ -493,23 +507,23 @@ void ViewerWidget::ReceivedAudioBufferForPlayback()
}
// Convert to packed data for audio output
QByteArray pack = packed_processor_.Convert(samples);
// If the tempo must be adjusted, adjust now
if (tempo_processor_.IsOpen()) {
tempo_processor_.Push(pack);
pack = tempo_processor_.Pull();
}
AudioProcessor::Buffer buf;
int r = audio_processor_.Convert(samples->to_raw_ptrs(), samples->sample_count(), &buf);
// TempoProcessor may have emptied the array
if (!pack.isEmpty()) {
if (prequeuing_audio_) {
// Add to prequeued audio buffer
prequeued_audio_.append(pack);
} else {
// Push directly to audio manager
AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack);
if (r >= 0) {
if (!buf.empty()) {
const QByteArray &pack = buf.at(0);
if (prequeuing_audio_) {
// Add to prequeued audio buffer
prequeued_audio_.append(pack);
} else {
// Push directly to audio manager
AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack);
}
}
} else {
qCritical() << "Failed to process audio for playback:" << r;
}
}
}
@@ -540,10 +554,19 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing()
samples->transform_volume_for_sample(samples->sample_count() - i - 1, amt);
}*/
QByteArray packed = packed_processor_.Convert(samples);
AudioManager::instance()->ClearBufferedOutput();
AudioManager::instance()->PushToOutput(samples->audio_params(), packed);
AudioMonitor::PushBytesOnAll(packed);
AudioProcessor::Buffer buf;
int r = audio_processor_.Convert(samples->to_raw_ptrs(), samples->sample_count(), &buf);
if (r >= 0) {
if (!buf.empty()) {
const QByteArray &packed = buf.at(0);
AudioManager::instance()->ClearBufferedOutput();
AudioManager::instance()->PushToOutput(samples->audio_params(), packed);
AudioMonitor::PushBytesOnAll(packed);
}
} else {
qCritical() << "Failed to process audio for scrubbing:" << r;
}
}
}
}
@@ -675,9 +698,7 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only)
AudioManager::instance()->SetOutputNotifyInterval(ap.time_to_bytes(kAudioPlaybackInterval));
connect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer);
if (std::abs(playback_speed_) > 1) {
tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_));
}
UpdateAudioProcessor();
static const int prequeue_count = 2;
prequeuing_audio_ = prequeue_count; // Queue two buffers ahead of time
@@ -721,9 +742,7 @@ void ViewerWidget::PauseInternal()
disconnect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer);
qDeleteAll(audio_playback_queue_);
audio_playback_queue_.clear();
if (tempo_processor_.IsOpen()) {
tempo_processor_.Close();
}
UpdateAudioProcessor();
foreach (ViewerWidget* viewer, instances_) {
viewer->auto_cacher_.SetAudioPaused(false);
@@ -1434,11 +1453,7 @@ void ViewerWidget::UpdateRendererVideoParameters()
void ViewerWidget::UpdateRendererAudioParameters()
{
packed_processor_.Close();
AudioParams ap = GetConnectedNode()->GetAudioParams();
packed_processor_.Open(ap);
UpdateAudioProcessor();
}
void ViewerWidget::SetZoomFromMenu(QAction *action)
app/widget/viewer/viewer.h
+5 -4
View File
@@ -28,8 +28,7 @@
#include <QTimer>
#include <QWidget>
#include "audio/packedprocessor.h"
#include "audio/tempoprocessor.h"
#include "audio/audioprocessor.h"
#include "audiowaveformview.h"
#include "common/rational.h"
#include "node/output/viewer/viewer.h"
@@ -212,6 +211,9 @@ private:
void DisarmRecording();
void UpdateAudioProcessor();
void CloseAudioProcessor();
QStackedWidget* stack_;
ViewerSizer* sizer_;
@@ -255,8 +257,7 @@ private:
std::list<RenderTicketWatcher*> audio_playback_queue_;
rational audio_playback_queue_time_;
PackedProcessor packed_processor_;
TempoProcessor tempo_processor_;
AudioProcessor audio_processor_;
QByteArray prequeued_audio_;
static const rational kAudioPlaybackInterval;