diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index 5d190af1f..08273c9af 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -18,13 +18,9 @@ set(OLIVE_SOURCES ${OLIVE_SOURCES} audio/audiomanager.cpp audio/audiomanager.h + audio/audioprocessor.cpp + audio/audioprocessor.h audio/audiovisualwaveform.cpp audio/audiovisualwaveform.h - audio/packedprocessor.cpp - audio/packedprocessor.h - audio/planarprocessor.cpp - audio/planarprocessor.h - audio/tempoprocessor.cpp - audio/tempoprocessor.h PARENT_SCOPE ) diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index f22319f42..fe791b3d5 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -26,7 +26,6 @@ #include -#include "audio/packedprocessor.h" #include "config/config.h" namespace olive { diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index c9400316c..20d1be75b 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -104,6 +104,7 @@ private: PaDeviceIndex input_device_; PaStream *input_stream_; + FFmpegEncoder *input_encoder_; }; diff --git a/app/audio/audioprocessor.cpp b/app/audio/audioprocessor.cpp new file mode 100644 index 000000000..b9c95bb6d --- /dev/null +++ b/app/audio/audioprocessor.cpp @@ -0,0 +1,302 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "audioprocessor.h" + +extern "C" { +#include +#include +} + +#include "common/ffmpegutils.h" + +namespace olive { + +AudioProcessor::AudioProcessor() +{ + filter_graph_ = nullptr; + in_frame_ = nullptr; + out_frame_ = nullptr; +} + +AudioProcessor::~AudioProcessor() +{ + Close(); +} + +bool AudioProcessor::Open(const AudioParams &from, const AudioParams &to, double tempo) +{ + if (filter_graph_) { + qWarning() << "Tried to open a processor that was already open"; + return false; + } + + filter_graph_ = avfilter_graph_alloc(); + if (!filter_graph_) { + qCritical() << "Failed to allocate filter graph"; + return false; + } + + from_fmt_ = FFmpegUtils::GetFFmpegSampleFormat(from.format()); + to_fmt_ = FFmpegUtils::GetFFmpegSampleFormat(to.format()); + + // Set up audio buffer args + char filter_args[200]; + snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64, + 1, + from.sample_rate(), + from.sample_rate(), + from_fmt_, + from.channel_layout()); + + int r; + + // Create buffersrc (input) + r = avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_); + if (r < 0) { + qCritical() << "Failed to create buffersrc:" << r; + Close(); + return false; + } + + // Store "previous" filter for linking + AVFilterContext *previous_filter = buffersrc_ctx_; + + // Create tempo + bool create_tempo; + if ((create_tempo = !qFuzzyCompare(tempo, 1.0))) { + // Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside + // those boundaries, we need to daisychain more than one together. + double base = (tempo > 1.0) ? 2.0 : 0.5; + double speed_log = log(tempo) / log(base); + + // This is the number of how many 0.5 or 2.0 tempos we need to daisychain + int whole = qFloor(speed_log); + + // Set speed_log to the remainder + speed_log -= whole; + + for (int i=0;i<=whole;i++) { + double filter_tempo = (i == whole) ? qPow(base, speed_log) : base; + + if (qFuzzyCompare(filter_tempo, 1.0)) { + // This filter would do nothing + continue; + } + + previous_filter = CreateTempoFilter(filter_graph_, + previous_filter, + filter_tempo); + + if (!previous_filter) { + qCritical() << "Failed to create audio tempo filter"; + Close(); + return false; + } + } + } + + // Create conversion filter + if (from.sample_rate() != to.sample_rate() || from.channel_layout() != to.channel_layout() || from.format() != to.format() + || (to.FormatIsPlanar() && create_tempo)) { // Tempo processor automatically converts to packed, + // so if the desired output is planar, it'll need + // to be converted + snprintf(filter_args, 200, "sample_fmts=%s:sample_rates=%d:channel_layouts=0x%" PRIx64, + av_get_sample_fmt_name(to_fmt_), + to.sample_rate(), + to.channel_layout()); + + AVFilterContext *c; + r = avfilter_graph_create_filter(&c, avfilter_get_by_name("aformat"), "fmt", filter_args, nullptr, filter_graph_); + if (r < 0) { + qCritical() << "Failed to create format conversion filter:" << r << filter_args; + Close(); + return false; + } + + r = avfilter_link(previous_filter, 0, c, 0); + if (r < 0) { + qCritical() << "Failed to link filters:" << r; + Close(); + return false; + } + + previous_filter = c; + } + + // Create buffersink (output) + r = avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_); + if (r < 0) { + qCritical() << "Failed to create buffersink:" << r; + Close(); + return false; + } + + r = avfilter_link(previous_filter, 0, buffersink_ctx_, 0); + if (r < 0) { + qCritical() << "Failed to link filters:" << r; + Close(); + return false; + } + + r = avfilter_graph_config(filter_graph_, nullptr); + if (r < 0) { + qCritical() << "Failed to configure graph:" << r; + Close(); + return false; + } + + in_frame_ = av_frame_alloc(); + if (in_frame_) { + in_frame_->sample_rate = from.sample_rate(); + in_frame_->format = from_fmt_; + in_frame_->channel_layout = from.channel_layout(); + in_frame_->channels = from.channel_count(); + in_frame_->pts = 0; + } else { + qCritical() << "Failed to allocate input frame"; + Close(); + return false; + } + + out_frame_ = av_frame_alloc(); + if (!out_frame_) { + qCritical() << "Failed to allocate output frame"; + Close(); + return false; + } + + from_ = from; + to_ = to; + + return true; +} + +void AudioProcessor::Close() +{ + if (filter_graph_) { + avfilter_graph_free(&filter_graph_); + filter_graph_ = nullptr; + buffersrc_ctx_ = nullptr; + buffersink_ctx_ = nullptr; + } + + if (in_frame_) { + av_frame_free(&in_frame_); + in_frame_ = nullptr; + } + + if (out_frame_) { + av_frame_free(&out_frame_); + out_frame_ = nullptr; + } +} + +int AudioProcessor::Convert(float **in, int nb_in_samples, AudioProcessor::Buffer *output) +{ + if (!IsOpen()) { + qCritical() << "Tried to convert on closed processor"; + return -1; + } + + int r = 0; + + if (in && nb_in_samples) { + // Set frame parameters + in_frame_->nb_samples = nb_in_samples; + for (int i=0; idata[i] = reinterpret_cast(in[i]); + in_frame_->linesize[i] = from_.samples_to_bytes(nb_in_samples); + } + + r = av_buffersrc_add_frame_flags(buffersrc_ctx_, in_frame_, AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + qCritical() << "Failed to add frame to buffersrc:" << r; + return r; + } + } + + if (output) { + int nb_channels = to_.channel_count(); + + if (to_.FormatIsPacked()) { + nb_channels = 1; + } + + AudioProcessor::Buffer &result = *output; + result.resize(nb_channels); + + int byte_offset = 0; + + while (true) { + av_frame_unref(out_frame_); + r = av_buffersink_get_frame(buffersink_ctx_, out_frame_); + if (r < 0) { + if (r == AVERROR(EAGAIN)) { + r = 0; + } else { + // Handle unexpected error + qCritical() << "Failed to pull from buffersink:" << r; + } + break; + } + + int nb_bytes = out_frame_->nb_samples * to_.bytes_per_sample_per_channel(); + if (to_.FormatIsPacked()) { + nb_bytes *= to_.channel_count(); + } + + for (int i=0; idata[i], nb_bytes); + } + byte_offset += nb_bytes; + } + av_frame_unref(out_frame_); + } + + return r; +} + +void AudioProcessor::Flush() +{ + int r = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + qCritical() << "Failed to flush:" << r; + } +} + +AVFilterContext *AudioProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo) +{ + // Set up tempo param, which is taken as a C string + char speed_param[20]; + snprintf(speed_param, 20, "%f", tempo); + + AVFilterContext* tempo_ctx = nullptr; + + if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0 + && avfilter_link(link, 0, tempo_ctx, 0) == 0) { + return tempo_ctx; + } + + return nullptr; +} + +} diff --git a/app/audio/tempoprocessor.h b/app/audio/audioprocessor.h similarity index 66% rename from app/audio/tempoprocessor.h rename to app/audio/audioprocessor.h index c56eafc5d..534cca9b1 100644 --- a/app/audio/tempoprocessor.h +++ b/app/audio/audioprocessor.h @@ -18,14 +18,8 @@ ***/ -#ifndef TEMPOPROCESSOR_H -#define TEMPOPROCESSOR_H - -#ifdef __MINGW32__ -#ifndef __USE_MINGW_ANSI_STDIO -#define __USE_MINGW_ANSI_STDIO -#endif -#endif +#ifndef AUDIOPROCESSOR_H +#define AUDIOPROCESSOR_H #include @@ -37,29 +31,26 @@ extern "C" { namespace olive { -class TempoProcessor +class AudioProcessor { public: - TempoProcessor(); + AudioProcessor(); - ~TempoProcessor(); + ~AudioProcessor(); - DISABLE_COPY_MOVE(TempoProcessor) + DISABLE_COPY_MOVE(AudioProcessor) - bool IsOpen() const; - - const double& GetSpeed() const; - - bool Open(const AudioParams& params, const double &speed); - - void Push(const QByteArray &packed); - - void Flush(); - - QByteArray Pull(); + bool Open(const AudioParams &from, const AudioParams &to, double tempo = 1.0); void Close(); + bool IsOpen() const { return filter_graph_; } + + using Buffer = QVector; + int Convert(float **in, int nb_in_samples, AudioProcessor::Buffer *output); + + void Flush(); + private: static AVFilterContext* CreateTempoFilter(AVFilterGraph *graph, AVFilterContext *link, const double& tempo); @@ -69,17 +60,18 @@ private: AVFilterContext* buffersink_ctx_; - AudioParams params_; + AudioParams from_; + AVSampleFormat from_fmt_; - int64_t timestamp_; + AudioParams to_; + AVSampleFormat to_fmt_; - double speed_; + AVFrame *in_frame_; - bool open_; + AVFrame *out_frame_; - bool flushed_; }; } -#endif // TEMPOPROCESSOR_H +#endif // AUDIOPROCESSOR_H diff --git a/app/audio/packedprocessor.cpp b/app/audio/packedprocessor.cpp deleted file mode 100644 index 91fd25a5c..000000000 --- a/app/audio/packedprocessor.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "packedprocessor.h" - -#include "common/ffmpegutils.h" - -namespace olive { - -PackedProcessor::PackedProcessor() : - swr_ctx_(nullptr) -{ -} - -PackedProcessor::~PackedProcessor() -{ - Close(); -} - -bool PackedProcessor::Open(const AudioParams ¶ms) -{ - if (IsOpen()) { - return true; - } - - swr_ctx_ = swr_alloc_set_opts(nullptr, - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(AudioParams::GetPackedEquivalent(params.format())), - params.sample_rate(), - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(params.format()), - params.sample_rate(), - 0, - nullptr); - - if (!swr_ctx_) { - qCritical() << "Failed to allocate resample context"; - return false; - } - - if (swr_init(swr_ctx_) < 0) { - qCritical() << "Failed to init resample context"; - swr_free(&swr_ctx_); - return false; - } - - return true; -} - -QByteArray PackedProcessor::Convert(SampleBufferPtr planar) -{ - if (!IsOpen()) { - qCritical() << "Tried to convert while closed"; - return QByteArray(); - } - - int nb_samples = planar->sample_count(); - if (nb_samples == 0) { - return QByteArray(); - } - - QByteArray output(planar->audio_params().samples_to_bytes(nb_samples), Qt::Uninitialized); - uint8_t *output_data = reinterpret_cast(output.data()); - - int ret = swr_convert(swr_ctx_, &output_data, nb_samples, - const_cast(reinterpret_cast(planar->to_raw_ptrs())), - nb_samples); - if (ret < 0) { - char buf[200]; - av_strerror(ret, buf, 200); - qDebug() << "Packed processor failed with error:" << buf << ret; - } - - return output; -} - -void PackedProcessor::Close() -{ - if (swr_ctx_) { - swr_free(&swr_ctx_); - } -} - -} diff --git a/app/audio/packedprocessor.h b/app/audio/packedprocessor.h deleted file mode 100644 index 8b1bb59c5..000000000 --- a/app/audio/packedprocessor.h +++ /dev/null @@ -1,60 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef PACKEDPROCESSOR_H -#define PACKEDPROCESSOR_H - -extern "C" { -#include -} - -#include "codec/samplebuffer.h" -#include "render/audioparams.h" - -namespace olive { - -class PackedProcessor -{ -public: - PackedProcessor(); - - ~PackedProcessor(); - - DISABLE_COPY_MOVE(PackedProcessor) - - bool Open(const AudioParams ¶ms); - - QByteArray Convert(SampleBufferPtr planar); - - void Close(); - - bool IsOpen() const - { - return swr_ctx_; - } - -private: - SwrContext *swr_ctx_; - -}; - -} - -#endif // PACKEDPROCESSOR_H diff --git a/app/audio/planarprocessor.cpp b/app/audio/planarprocessor.cpp deleted file mode 100644 index 0d5222b76..000000000 --- a/app/audio/planarprocessor.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "planarprocessor.h" - -#include "common/ffmpegutils.h" - -namespace olive { - -PlanarProcessor::PlanarProcessor() : - swr_ctx_(nullptr) -{ -} - -PlanarProcessor::~PlanarProcessor() -{ - Close(); -} - -bool PlanarProcessor::Open(const AudioParams ¶ms) -{ - if (IsOpen()) { - return true; - } - - swr_ctx_ = swr_alloc_set_opts(nullptr, - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(AudioParams::GetPlanarEquivalent(params.format())), - params.sample_rate(), - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(params.format()), - params.sample_rate(), - 0, - nullptr); - - if (!swr_ctx_) { - qCritical() << "Failed to allocate resample context"; - return false; - } - - if (swr_init(swr_ctx_) < 0) { - qCritical() << "Failed to init resample context"; - swr_free(&swr_ctx_); - return false; - } - - params_ = params; - - return true; -} - -SampleBufferPtr PlanarProcessor::Convert(const QByteArray &packed) -{ - if (!IsOpen()) { - qCritical() << "Tried to convert while closed"; - return nullptr; - } - - if (packed.isEmpty()) { - return nullptr; - } - - int nb_samples_per_channel = params_.bytes_to_samples(packed.size()); - - SampleBufferPtr output = SampleBuffer::CreateAllocated(params_, nb_samples_per_channel); - - const uint8_t *input = reinterpret_cast(packed.constData()); - int ret = swr_convert(swr_ctx_, - reinterpret_cast(output->to_raw_ptrs()), nb_samples_per_channel, - &input, nb_samples_per_channel); - if (ret < 0) { - char buf[200]; - av_strerror(ret, buf, 200); - qDebug() << "Planar processor failed with error:" << buf << ret; - } - - return output; -} - -void PlanarProcessor::Close() -{ - if (swr_ctx_) { - swr_free(&swr_ctx_); - } -} - -} diff --git a/app/audio/planarprocessor.h b/app/audio/planarprocessor.h deleted file mode 100644 index ad5167a95..000000000 --- a/app/audio/planarprocessor.h +++ /dev/null @@ -1,62 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef PLANARPROCESSOR_H -#define PLANARPROCESSOR_H - -extern "C" { -#include -} - -#include "codec/samplebuffer.h" -#include "render/audioparams.h" - -namespace olive { - -class PlanarProcessor -{ -public: - PlanarProcessor(); - - ~PlanarProcessor(); - - DISABLE_COPY_MOVE(PlanarProcessor) - - bool Open(const AudioParams ¶ms); - - SampleBufferPtr Convert(const QByteArray &packed); - - void Close(); - - bool IsOpen() const - { - return swr_ctx_; - } - -private: - SwrContext *swr_ctx_; - - AudioParams params_; - -}; - -} - -#endif // PLANARPROCESSOR_H diff --git a/app/audio/tempoprocessor.cpp b/app/audio/tempoprocessor.cpp deleted file mode 100644 index f22946227..000000000 --- a/app/audio/tempoprocessor.cpp +++ /dev/null @@ -1,269 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "tempoprocessor.h" - -extern "C" { -#include -#include -#include -} - -#include - -#include "common/ffmpegutils.h" - -namespace olive { - -TempoProcessor::TempoProcessor() : - filter_graph_(nullptr), - buffersrc_ctx_(nullptr), - buffersink_ctx_(nullptr), - open_(false) -{ -} - -TempoProcessor::~TempoProcessor() -{ - Close(); -} - -bool TempoProcessor::IsOpen() const -{ - return open_; -} - -const double &TempoProcessor::GetSpeed() const -{ - return speed_; -} - -bool TempoProcessor::Open(const AudioParams ¶ms, const double& speed) -{ - if (open_) { - return true; - } - - params_ = params; - speed_ = speed; - - // Create AVFilterGraph instance - filter_graph_ = avfilter_graph_alloc(); - if (!filter_graph_) { - qCritical() << "Failed to create AVFilterGraph"; - Close(); - return false; - } - - // Set up audio buffer args - char filter_args[200]; - snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64, - 1, - params_.sample_rate(), - params_.sample_rate(), - FFmpegUtils::GetFFmpegSampleFormat(params_.format()), - params.channel_layout()); - - // Create buffer and buffersink - if (avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_) < 0) { - qCritical() << "Failed to create audio buffer source"; - Close(); - return false; - } - - if (avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_) < 0) { - qCritical() << "Failed to create audio buffer sink"; - Close(); - return false; - } - - // Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside - // those boundaries, we need to daisychain more than one together. - double base = (speed_ > 1.0) ? 2.0 : 0.5; - double speed_log = log(speed_) / log(base); - - // This is the number of how many 0.5 or 2.0 tempos we need to daisychain - int whole = qFloor(speed_log); - - // Set speed_log to the remainder - speed_log -= whole; - - AVFilterContext* previous_filter = buffersrc_ctx_; - - for (int i=0;i<=whole;i++) { - double filter_tempo = (i == whole) ? qPow(base, speed_log) : base; - - if (qFuzzyCompare(filter_tempo, 1.0)) { - // This filter would do nothing - continue; - } - - previous_filter = CreateTempoFilter(filter_graph_, - previous_filter, - filter_tempo); - - if (!previous_filter) { - qCritical() << "Failed to create audio tempo filter"; - Close(); - return false; - } - } - - // Link the last filter to the buffersink - if (avfilter_link(previous_filter, 0, buffersink_ctx_, 0) != 0) { - qCritical() << "Failed to link final filter and buffer sink"; - Close(); - return false; - } - - // Config graph - if (avfilter_graph_config(filter_graph_, nullptr) < 0) { - qCritical() << "Failed to configure filter graph"; - Close(); - return false; - } - - timestamp_ = 0; - - open_ = true; - - flushed_ = false; - - return true; -} - -void TempoProcessor::Push(const QByteArray &packed) -{ - if (!IsOpen()) { - qWarning() << "Tried to push to closed TempoProcessor"; - return; - } - - if (flushed_) { - qWarning() << "Tried to push to flushed TempoProcessor"; - return; - } - - AVFrame* src_frame = av_frame_alloc(); - - if (!src_frame) { - qCritical() << "Failed to allocate source frame"; - return; - } - - // Allocate a buffer for the number of samples we got - src_frame->sample_rate = params_.sample_rate(); - src_frame->format = FFmpegUtils::GetFFmpegSampleFormat(params_.format()); - src_frame->channel_layout = params_.channel_layout(); - src_frame->nb_samples = params_.bytes_to_samples(packed.size()); - src_frame->pts = timestamp_; - timestamp_ += src_frame->nb_samples; - - if (av_frame_get_buffer(src_frame, 0) < 0) { - qCritical() << "Failed to allocate buffer for source frame"; - av_frame_free(&src_frame); - return; - } - - // Copy buffer from data array to frame - memcpy(src_frame->data[0], packed, packed.size()); - - int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, src_frame, AV_BUFFERSRC_FLAG_KEEP_REF); - - if (ret < 0) { - qCritical() << "Failed to feed buffer source" << ret; - } - - if (src_frame) { - av_frame_free(&src_frame); - } -} - -void TempoProcessor::Flush() -{ - if (!flushed_) { - int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF); - if (ret < 0) { - qCritical() << "Failed to feed buffer source" << ret; - } - flushed_ = true; - } -} - -QByteArray TempoProcessor::Pull() -{ - QByteArray b; - AVFrame *processed_frame = av_frame_alloc(); - - // Try to pull samples from the buffersink - int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame); - - if (ret < 0) { - // We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the - // error might be fatal... - if (ret != AVERROR(EAGAIN)) { - qCritical() << "Failed to pull from buffersink" << ret; - } - - av_frame_free(&processed_frame); - return b; - } - - b.resize(params_.samples_to_bytes(processed_frame->nb_samples)); - - // Copy the bytes - memcpy(b.data(), processed_frame->data[0], b.size()); - - // If the index has reached the limit of this processed frame, we can dispose of the frame now - av_frame_free(&processed_frame); - - return b; -} - -void TempoProcessor::Close() -{ - open_ = false; - - if (filter_graph_) { - avfilter_graph_free(&filter_graph_); - filter_graph_ = nullptr; - } - - buffersrc_ctx_ = nullptr; - buffersink_ctx_ = nullptr; -} - -AVFilterContext *TempoProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo) -{ - // Set up tempo param, which is taken as a C string - char speed_param[20]; - snprintf(speed_param, 20, "%f", tempo); - - AVFilterContext* tempo_ctx = nullptr; - - if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0 - && avfilter_link(link, 0, tempo_ctx, 0) == 0) { - return tempo_ctx; - } - - return nullptr; -} - -} diff --git a/app/render/audioparams.h b/app/render/audioparams.h index 9213fd872..cb4c591b7 100644 --- a/app/render/audioparams.h +++ b/app/render/audioparams.h @@ -186,6 +186,26 @@ public: duration_ = duration; } + static bool FormatIsPacked(Format f) + { + return f >= kPackedStart && f < kPackedEnd; + } + + bool FormatIsPacked() const + { + return FormatIsPacked(format_); + } + + static bool FormatIsPlanar(Format f) + { + return f >= kPlanarStart && f < kPlanarEnd; + } + + bool FormatIsPlanar() const + { + return FormatIsPlanar(format_); + } + qint64 time_to_bytes(const double& time) const; qint64 time_to_bytes(const rational& time) const; qint64 time_to_bytes_per_channel(const double& time) const; diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index dc4556f8d..7e3876f8d 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -25,9 +25,7 @@ #include #include -#include "audio/packedprocessor.h" -#include "audio/planarprocessor.h" -#include "audio/tempoprocessor.h" +#include "audio/audioprocessor.h" #include "node/block/clip/clip.h" #include "node/block/transition/transition.h" #include "node/project/project.h" @@ -300,14 +298,10 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim samples_from_this_block->silence(); } else if (!qFuzzyCompare(speed_value, 1.0)) { if (clip_cast->maintain_audio_pitch()) { - PackedProcessor packer; - packer.Open(samples_from_this_block->audio_params()); + AudioProcessor processor; - QByteArray packed = packer.Convert(samples_from_this_block); - - if (!packed.isEmpty()) { - TempoProcessor tp; - tp.Open(samples_from_this_block->audio_params(), speed_value); + if (processor.Open(samples_from_this_block->audio_params(), samples_from_this_block->audio_params(), speed_value)) { + AudioProcessor::Buffer out; // FIXME: This is not the best way to do this, the TempoProcessor works best // when it's given a continuous stream of audio, which is challenging @@ -315,15 +309,31 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim // well on export (assuming audio is all generated at once on export), but // users may hear clicks and pops in the audio during preview due to this // approach. - tp.Push(packed); - tp.Flush(); - packed = tp.Pull(); - tp.Close(); + int r = processor.Convert(samples_from_this_block->to_raw_ptrs(), samples_from_this_block->sample_count(), nullptr); - if (!packed.isEmpty()) { - PlanarProcessor planar; - planar.Open(samples_from_this_block->audio_params()); - samples_from_this_block = planar.Convert(packed); + if (r < 0) { + qCritical() << "Failed to change tempo of audio:" << r; + } else { + processor.Flush(); + + processor.Convert(nullptr, 0, &out); + + if (!out.empty()) { + int nb_samples = out.front().size() * samples_from_this_block->audio_params().bytes_per_sample_per_channel(); + + if (nb_samples) { + SampleBufferPtr new_samples = SampleBuffer::Create(); + new_samples->set_audio_params(samples_from_this_block->audio_params()); + new_samples->set_sample_count(nb_samples); + new_samples->allocate(); + + for (int i=0; idata(i), out[i].data(), out[i].size()); + } + + samples_from_this_block = new_samples; + } + } } } } else { diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index d404fb5ed..012e7650e 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -210,8 +210,7 @@ void ViewerWidget::ConnectNodeEvent(ViewerOutput *n) last_length_ = 0; LengthChangedSlot(n->GetLength()); - AudioParams ap = n->GetAudioParams(); - packed_processor_.Open(ap); + UpdateAudioProcessor(); ColorManager* color_manager = n->project()->color_manager(); @@ -245,7 +244,7 @@ void ViewerWidget::DisconnectNodeEvent(ViewerOutput *n) disconnect(n->video_frame_cache(), &FrameHashCache::Shifted, this, &ViewerWidget::ViewerShiftedRange); disconnect(n, &ViewerOutput::TextureInputChanged, this, &ViewerWidget::UpdateStack); - packed_processor_.Close(); + CloseAudioProcessor(); SetDisplayImage(QVariant()); @@ -456,6 +455,21 @@ void ViewerWidget::DisarmRecording() record_armed_ = false; } +void ViewerWidget::UpdateAudioProcessor() +{ + audio_processor_.Close(); + + AudioParams ap = GetConnectedNode()->GetAudioParams(); + AudioParams packed = ap; + packed.set_format(AudioParams::GetPackedEquivalent(ap.format())); + audio_processor_.Open(ap, packed, (playback_speed_ == 0) ? 1 : std::abs(playback_speed_)); +} + +void ViewerWidget::CloseAudioProcessor() +{ + audio_processor_.Close(); +} + void ViewerWidget::QueueNextAudioBuffer() { rational queue_end = audio_playback_queue_time_ + (kAudioPlaybackInterval * playback_speed_); @@ -493,23 +507,23 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() } // Convert to packed data for audio output - QByteArray pack = packed_processor_.Convert(samples); - - // If the tempo must be adjusted, adjust now - if (tempo_processor_.IsOpen()) { - tempo_processor_.Push(pack); - pack = tempo_processor_.Pull(); - } + AudioProcessor::Buffer buf; + int r = audio_processor_.Convert(samples->to_raw_ptrs(), samples->sample_count(), &buf); // TempoProcessor may have emptied the array - if (!pack.isEmpty()) { - if (prequeuing_audio_) { - // Add to prequeued audio buffer - prequeued_audio_.append(pack); - } else { - // Push directly to audio manager - AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack); + if (r >= 0) { + if (!buf.empty()) { + const QByteArray &pack = buf.at(0); + if (prequeuing_audio_) { + // Add to prequeued audio buffer + prequeued_audio_.append(pack); + } else { + // Push directly to audio manager + AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack); + } } + } else { + qCritical() << "Failed to process audio for playback:" << r; } } } @@ -540,10 +554,19 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing() samples->transform_volume_for_sample(samples->sample_count() - i - 1, amt); }*/ - QByteArray packed = packed_processor_.Convert(samples); - AudioManager::instance()->ClearBufferedOutput(); - AudioManager::instance()->PushToOutput(samples->audio_params(), packed); - AudioMonitor::PushBytesOnAll(packed); + AudioProcessor::Buffer buf; + int r = audio_processor_.Convert(samples->to_raw_ptrs(), samples->sample_count(), &buf); + + if (r >= 0) { + if (!buf.empty()) { + const QByteArray &packed = buf.at(0); + AudioManager::instance()->ClearBufferedOutput(); + AudioManager::instance()->PushToOutput(samples->audio_params(), packed); + AudioMonitor::PushBytesOnAll(packed); + } + } else { + qCritical() << "Failed to process audio for scrubbing:" << r; + } } } } @@ -675,9 +698,7 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) AudioManager::instance()->SetOutputNotifyInterval(ap.time_to_bytes(kAudioPlaybackInterval)); connect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer); - if (std::abs(playback_speed_) > 1) { - tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_)); - } + UpdateAudioProcessor(); static const int prequeue_count = 2; prequeuing_audio_ = prequeue_count; // Queue two buffers ahead of time @@ -721,9 +742,7 @@ void ViewerWidget::PauseInternal() disconnect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer); qDeleteAll(audio_playback_queue_); audio_playback_queue_.clear(); - if (tempo_processor_.IsOpen()) { - tempo_processor_.Close(); - } + UpdateAudioProcessor(); foreach (ViewerWidget* viewer, instances_) { viewer->auto_cacher_.SetAudioPaused(false); @@ -1434,11 +1453,7 @@ void ViewerWidget::UpdateRendererVideoParameters() void ViewerWidget::UpdateRendererAudioParameters() { - packed_processor_.Close(); - - AudioParams ap = GetConnectedNode()->GetAudioParams(); - - packed_processor_.Open(ap); + UpdateAudioProcessor(); } void ViewerWidget::SetZoomFromMenu(QAction *action) diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 4bf043af1..be515ff4d 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -28,8 +28,7 @@ #include #include -#include "audio/packedprocessor.h" -#include "audio/tempoprocessor.h" +#include "audio/audioprocessor.h" #include "audiowaveformview.h" #include "common/rational.h" #include "node/output/viewer/viewer.h" @@ -212,6 +211,9 @@ private: void DisarmRecording(); + void UpdateAudioProcessor(); + void CloseAudioProcessor(); + QStackedWidget* stack_; ViewerSizer* sizer_; @@ -255,8 +257,7 @@ private: std::list audio_playback_queue_; rational audio_playback_queue_time_; - PackedProcessor packed_processor_; - TempoProcessor tempo_processor_; + AudioProcessor audio_processor_; QByteArray prequeued_audio_; static const rational kAudioPlaybackInterval;