From 845b4a755954a626587aeaf166757daeef522752 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sat, 25 Sep 2021 13:22:13 -0700 Subject: [PATCH] use ffmpeg for sample packing FFmpeg has a much more optimized algorithm than we do so may as well use it. --- app/audio/CMakeLists.txt | 12 ++-- app/audio/packedprocessor.cpp | 106 ++++++++++++++++++++++++++++++++++ app/audio/packedprocessor.h | 60 +++++++++++++++++++ app/audio/tempoprocessor.cpp | 4 ++ app/audio/tempoprocessor.h | 4 ++ app/codec/samplebuffer.cpp | 19 ------ app/codec/samplebuffer.h | 2 - app/widget/viewer/viewer.cpp | 11 +++- app/widget/viewer/viewer.h | 2 + 9 files changed, 191 insertions(+), 29 deletions(-) create mode 100644 app/audio/packedprocessor.cpp create mode 100644 app/audio/packedprocessor.h diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index dd5a92d48..a03a4d88e 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -16,15 +16,17 @@ set(OLIVE_SOURCES ${OLIVE_SOURCES} - audio/audiomanager.h audio/audiomanager.cpp - audio/audiovisualwaveform.h + audio/audiomanager.h audio/audiovisualwaveform.cpp - audio/outputdeviceproxy.h + audio/audiovisualwaveform.h audio/outputdeviceproxy.cpp - audio/outputmanager.h + audio/outputdeviceproxy.h audio/outputmanager.cpp - audio/tempoprocessor.h + audio/outputmanager.h + audio/packedprocessor.cpp + audio/packedprocessor.h audio/tempoprocessor.cpp + audio/tempoprocessor.h PARENT_SCOPE ) diff --git a/app/audio/packedprocessor.cpp b/app/audio/packedprocessor.cpp new file mode 100644 index 000000000..10eb9ea18 --- /dev/null +++ b/app/audio/packedprocessor.cpp @@ -0,0 +1,106 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "packedprocessor.h" + +#include "common/ffmpegutils.h" + +namespace olive { + +PackedProcessor::PackedProcessor() : + swr_ctx_(nullptr) +{ +} + +PackedProcessor::~PackedProcessor() +{ + Close(); +} + +bool PackedProcessor::Open(const AudioParams ¶ms) +{ + if (IsOpen()) { + return true; + } + + swr_ctx_ = swr_alloc_set_opts(nullptr, + params.channel_layout(), + FFmpegUtils::GetFFmpegSampleFormat(params.format(), false), + params.sample_rate(), + params.channel_layout(), + FFmpegUtils::GetFFmpegSampleFormat(params.format(), true), + params.sample_rate(), + 0, + nullptr); + + if (!swr_ctx_) { + qCritical() << "Failed to allocate resample context"; + return false; + } + + if (swr_init(swr_ctx_) < 0) { + qCritical() << "Failed to init resample context"; + swr_free(&swr_ctx_); + return false; + } + + return true; +} + +QByteArray PackedProcessor::Convert(SampleBufferPtr planar) +{ + if (!IsOpen()) { + qCritical() << "Tried to convert while closed"; + return QByteArray(); + } + + int nb_samples = planar->sample_count(); + if (nb_samples == 0) { + return QByteArray(); + } + + int nb_channels = planar->audio_params().channel_count(); + + QByteArray output(planar->audio_params().samples_to_bytes(nb_samples), Qt::Uninitialized); + uint8_t *output_data = reinterpret_cast(output.data()); + + QVector input_arrays(nb_channels); + for (int i=0; i(planar->data(i)); + } + + int ret = swr_convert(swr_ctx_, &output_data, nb_samples, input_arrays.data(), nb_samples); + if (ret < 0) { + char buf[200]; + av_strerror(ret, buf, 200); + qDebug() << "Packed processor failed with error:" << buf << ret; + } + + return output; +} + +void PackedProcessor::Close() +{ + if (swr_ctx_) { + swr_free(&swr_ctx_); + } +} + +} diff --git a/app/audio/packedprocessor.h b/app/audio/packedprocessor.h new file mode 100644 index 000000000..8b1bb59c5 --- /dev/null +++ b/app/audio/packedprocessor.h @@ -0,0 +1,60 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef PACKEDPROCESSOR_H +#define PACKEDPROCESSOR_H + +extern "C" { +#include +} + +#include "codec/samplebuffer.h" +#include "render/audioparams.h" + +namespace olive { + +class PackedProcessor +{ +public: + PackedProcessor(); + + ~PackedProcessor(); + + DISABLE_COPY_MOVE(PackedProcessor) + + bool Open(const AudioParams ¶ms); + + QByteArray Convert(SampleBufferPtr planar); + + void Close(); + + bool IsOpen() const + { + return swr_ctx_; + } + +private: + SwrContext *swr_ctx_; + +}; + +} + +#endif // PACKEDPROCESSOR_H diff --git a/app/audio/tempoprocessor.cpp b/app/audio/tempoprocessor.cpp index e45f51bec..0a0aa777e 100644 --- a/app/audio/tempoprocessor.cpp +++ b/app/audio/tempoprocessor.cpp @@ -39,7 +39,11 @@ TempoProcessor::TempoProcessor() : processed_frame_(nullptr), open_(false) { +} +TempoProcessor::~TempoProcessor() +{ + Close(); } bool TempoProcessor::IsOpen() const diff --git a/app/audio/tempoprocessor.h b/app/audio/tempoprocessor.h index cdbda48df..bac242751 100644 --- a/app/audio/tempoprocessor.h +++ b/app/audio/tempoprocessor.h @@ -42,6 +42,10 @@ class TempoProcessor public: TempoProcessor(); + ~TempoProcessor(); + + DISABLE_COPY_MOVE(TempoProcessor) + bool IsOpen() const; const double& GetSpeed() const; diff --git a/app/codec/samplebuffer.cpp b/app/codec/samplebuffer.cpp index 5a60c49a2..df00e0f7f 100644 --- a/app/codec/samplebuffer.cpp +++ b/app/codec/samplebuffer.cpp @@ -237,23 +237,4 @@ void SampleBuffer::set(int channel, const float *data, int sample_offset, int sa memcpy(&data_[channel].data()[sample_offset], data, sizeof(float) * sample_length); } -QByteArray SampleBuffer::toPackedData() const -{ - QByteArray packed_data; - - if (is_allocated()) { - packed_data.resize(audio_params_.samples_to_bytes(sample_count_per_channel_)); - - float* output_data = reinterpret_cast(packed_data.data()); - - for (int j=0;jHasResult()) { SampleBufferPtr samples = watcher->Get().value(); if (samples && audio_playback_device_) { - qint64 t = QDateTime::currentMSecsSinceEpoch(); - QByteArray pack = samples->toPackedData(); - qDebug() << "Packing took:" << (QDateTime::currentMSecsSinceEpoch() - t); + if (!packed_processor_.IsOpen()) { + packed_processor_.Open(samples->audio_params()); + } + + // Convert to packed data for audio output + QByteArray pack = packed_processor_.Convert(samples); + audio_playback_device_->Push(pack); if (prequeuing_audio_) { @@ -627,6 +631,7 @@ void ViewerWidget::PauseInternal() audio_playback_device_.reset(nullptr); qDeleteAll(audio_playback_queue_); audio_playback_queue_.clear(); + packed_processor_.Close(); UpdateTextureFromNode(); } diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index a0e91f898..7844bb555 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -28,6 +28,7 @@ #include #include +#include "audio/packedprocessor.h" #include "audiowaveformview.h" #include "common/rational.h" #include "node/output/viewer/viewer.h" @@ -257,6 +258,7 @@ private: std::unique_ptr audio_playback_device_; std::list audio_playback_queue_; rational audio_playback_queue_time_; + PackedProcessor packed_processor_; static QVector instances_;