From 8f7280dd0967630b7666e6449320841a8d393c0f Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Wed, 4 May 2022 15:19:20 -0700 Subject: [PATCH 1/9] clip: add missing retranslate --- app/node/block/clip/clip.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/app/node/block/clip/clip.cpp b/app/node/block/clip/clip.cpp index 16720eafc..c48dee2a4 100644 --- a/app/node/block/clip/clip.cpp +++ b/app/node/block/clip/clip.cpp @@ -284,6 +284,7 @@ void ClipBlock::Retranslate() SetInputName(kMediaInInput, tr("Media In")); SetInputName(kSpeedInput, tr("Speed")); SetInputName(kReverseInput, tr("Reverse")); + SetInputName(kMaintainAudioPitchInput, tr("Maintain Audio Pitch")); } void ClipBlock::Hash(QCryptographicHash &hash, const NodeGlobals &globals, const VideoParams &video_params) const From ac7ff774e07ccb9ccd79ae0d059613b6d31c2a8e Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Wed, 4 May 2022 15:19:34 -0700 Subject: [PATCH 2/9] subtitle: disable flag to not show in param view --- app/node/block/subtitle/subtitle.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/node/block/subtitle/subtitle.cpp b/app/node/block/subtitle/subtitle.cpp index 8c60d8ee4..24b628178 100644 --- a/app/node/block/subtitle/subtitle.cpp +++ b/app/node/block/subtitle/subtitle.cpp @@ -29,6 +29,9 @@ const QString SubtitleBlock::kTextIn = QStringLiteral("text_in"); SubtitleBlock::SubtitleBlock() { AddInput(kTextIn, NodeValue::kText, InputFlags(kInputFlagNotConnectable | kInputFlagNotKeyframable)); + + // Undo block flag that hides in param view + SetFlags(GetFlags() & ~kDontShowInParamView); } QString SubtitleBlock::Name() const From fd692e69107f667b8a9e009104b0afc4144caf16 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 5 May 2022 09:39:52 -0700 Subject: [PATCH 3/9] rendertask: always wait for watchers to finish I don't know why this wasn't written to do this before, perhaps I thought waiting for the thread would achieve this goal. Anyway, in my testing, this ensures render jobs are complete before the task returns. Fixes #1899 Fixes #1900 --- app/task/render/render.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/task/render/render.cpp b/app/task/render/render.cpp index f3cd07d5c..bb7c99a17 100644 --- a/app/task/render/render.cpp +++ b/app/task/render/render.cpp @@ -266,9 +266,14 @@ bool RenderTask::Render(ColorManager* manager, if (IsCancelled() || !result) { // Cancel every watcher we created foreach (RenderTicketWatcher* watcher, running_watchers_) { + watcher->Cancel(); disconnect(watcher, &RenderTicketWatcher::Finished, this, &RenderTask::TicketDone); RenderManager::instance()->RemoveTicket(watcher->GetTicket()); } + + foreach (RenderTicketWatcher* watcher, running_watchers_) { + watcher->WaitForFinished(); + } } watcher_thread.quit(); From 591d02bdd6735d01c9c2230a6d5140a798c1df94 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 5 May 2022 11:47:53 -0700 Subject: [PATCH 4/9] audiomanager: fix crash and return error text if applicable --- app/audio/audiomanager.cpp | 13 ++++++++++--- app/audio/audiomanager.h | 2 +- app/codec/ffmpeg/ffmpegencoder.cpp | 2 +- app/widget/viewer/viewer.cpp | 5 +++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index 8fac9bd65..f22319f42 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -189,7 +189,7 @@ void AudioManager::HardReset() Pa_Initialize(); } -bool AudioManager::StartRecording(const EncodingParams ¶ms) +bool AudioManager::StartRecording(const EncodingParams ¶ms, QString *error_str) { if (input_device_ == paNoDevice) { return false; @@ -203,12 +203,19 @@ bool AudioManager::StartRecording(const EncodingParams ¶ms) PaStreamParameters p = GetPortAudioParams(params.audio_params(), input_device_); - if (Pa_OpenStream(&input_stream_, &p, nullptr, params.audio_params().sample_rate(), paFramesPerBufferUnspecified, paNoFlag, InputCallback, input_encoder_) == paNoError) { - if (Pa_StartStream(input_stream_) == paNoError) { + PaError r = Pa_OpenStream(&input_stream_, &p, nullptr, params.audio_params().sample_rate(), paFramesPerBufferUnspecified, paNoFlag, InputCallback, input_encoder_); + if (r == paNoError) { + //const PaStreamInfo* info = Pa_GetStreamInfo(input_stream_); + r = Pa_StartStream(input_stream_); + if (r == paNoError) { return true; } } + if (error_str) { + *error_str = Pa_GetErrorText(r); + } + StopRecording(); return false; } diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index bf71e2223..c9400316c 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -74,7 +74,7 @@ public: void HardReset(); - bool StartRecording(const EncodingParams ¶ms); + bool StartRecording(const EncodingParams ¶ms, QString *error_str = nullptr); void StopRecording(); diff --git a/app/codec/ffmpeg/ffmpegencoder.cpp b/app/codec/ffmpeg/ffmpegencoder.cpp index 19db89732..8a0462e2d 100644 --- a/app/codec/ffmpeg/ffmpegencoder.cpp +++ b/app/codec/ffmpeg/ffmpegencoder.cpp @@ -285,7 +285,7 @@ bool FFmpegEncoder::WriteAudio(SampleBufferPtr audio) } } - result = WriteAudioData(audio->audio_params(), const_cast(input_data), input_sample_count); + result = WriteAudioData(audio ? audio->audio_params() : params().audio_params(), const_cast(input_data), input_sample_count); if (input_data) { av_freep(&input_data[0]); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 80fe1a444..d404fb5ed 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -1190,12 +1190,13 @@ void ViewerWidget::Play(bool in_to_out_only) encode_param.SetFilename(recording_filename_); encode_param.set_audio_bit_rate(OLIVE_CONFIG("AudioRecordingBitRate").toInt() * 1000); - if (AudioManager::instance()->StartRecording(encode_param)) { + QString error; + if (AudioManager::instance()->StartRecording(encode_param, &error)) { recording_ = true; controls_->SetPauseButtonRecordingState(true); recording_callback_->EnableRecordingOverlay(TimelineCoordinate(recording_range_.in(), recording_track_)); } else { - QMessageBox::critical(this, tr("Audio Recording"), tr("Failed to start audio recording")); + QMessageBox::critical(this, tr("Audio Recording"), tr("Failed to start audio recording: %1").arg(error)); return; } } From 8ba1b231a3b7a9e552c670123f8ab8507b206e21 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 5 May 2022 14:40:56 -0700 Subject: [PATCH 5/9] packaging: add macos mic permission description --- app/packaging/macos/MacOSXBundleInfo.plist.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/packaging/macos/MacOSXBundleInfo.plist.in b/app/packaging/macos/MacOSXBundleInfo.plist.in index 286f53ae2..4b8ea9a1a 100644 --- a/app/packaging/macos/MacOSXBundleInfo.plist.in +++ b/app/packaging/macos/MacOSXBundleInfo.plist.in @@ -28,5 +28,7 @@ ${MACOSX_BUNDLE_COPYRIGHT} NSPrincipalClass NSApplication + NSMicrophoneUsageDescription + This app requires microphone access to record audio tracks. From 9ec42c792adaf93c9daf893dc9055e967caf927e Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 5 May 2022 15:28:32 -0700 Subject: [PATCH 6/9] audio: merged all processors into single versatile object --- app/audio/CMakeLists.txt | 8 +- app/audio/audiomanager.cpp | 1 - app/audio/audiomanager.h | 1 + app/audio/audioprocessor.cpp | 302 ++++++++++++++++++ .../{tempoprocessor.h => audioprocessor.h} | 50 ++- app/audio/packedprocessor.cpp | 101 ------ app/audio/packedprocessor.h | 60 ---- app/audio/planarprocessor.cpp | 104 ------ app/audio/planarprocessor.h | 62 ---- app/audio/tempoprocessor.cpp | 269 ---------------- app/render/audioparams.h | 20 ++ app/render/renderprocessor.cpp | 46 +-- app/widget/viewer/viewer.cpp | 79 +++-- app/widget/viewer/viewer.h | 9 +- 14 files changed, 426 insertions(+), 686 deletions(-) create mode 100644 app/audio/audioprocessor.cpp rename app/audio/{tempoprocessor.h => audioprocessor.h} (66%) delete mode 100644 app/audio/packedprocessor.cpp delete mode 100644 app/audio/packedprocessor.h delete mode 100644 app/audio/planarprocessor.cpp delete mode 100644 app/audio/planarprocessor.h delete mode 100644 app/audio/tempoprocessor.cpp diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index 5d190af1f..08273c9af 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -18,13 +18,9 @@ set(OLIVE_SOURCES ${OLIVE_SOURCES} audio/audiomanager.cpp audio/audiomanager.h + audio/audioprocessor.cpp + audio/audioprocessor.h audio/audiovisualwaveform.cpp audio/audiovisualwaveform.h - audio/packedprocessor.cpp - audio/packedprocessor.h - audio/planarprocessor.cpp - audio/planarprocessor.h - audio/tempoprocessor.cpp - audio/tempoprocessor.h PARENT_SCOPE ) diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index f22319f42..fe791b3d5 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -26,7 +26,6 @@ #include -#include "audio/packedprocessor.h" #include "config/config.h" namespace olive { diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index c9400316c..20d1be75b 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -104,6 +104,7 @@ private: PaDeviceIndex input_device_; PaStream *input_stream_; + FFmpegEncoder *input_encoder_; }; diff --git a/app/audio/audioprocessor.cpp b/app/audio/audioprocessor.cpp new file mode 100644 index 000000000..b9c95bb6d --- /dev/null +++ b/app/audio/audioprocessor.cpp @@ -0,0 +1,302 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2021 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "audioprocessor.h" + +extern "C" { +#include +#include +} + +#include "common/ffmpegutils.h" + +namespace olive { + +AudioProcessor::AudioProcessor() +{ + filter_graph_ = nullptr; + in_frame_ = nullptr; + out_frame_ = nullptr; +} + +AudioProcessor::~AudioProcessor() +{ + Close(); +} + +bool AudioProcessor::Open(const AudioParams &from, const AudioParams &to, double tempo) +{ + if (filter_graph_) { + qWarning() << "Tried to open a processor that was already open"; + return false; + } + + filter_graph_ = avfilter_graph_alloc(); + if (!filter_graph_) { + qCritical() << "Failed to allocate filter graph"; + return false; + } + + from_fmt_ = FFmpegUtils::GetFFmpegSampleFormat(from.format()); + to_fmt_ = FFmpegUtils::GetFFmpegSampleFormat(to.format()); + + // Set up audio buffer args + char filter_args[200]; + snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64, + 1, + from.sample_rate(), + from.sample_rate(), + from_fmt_, + from.channel_layout()); + + int r; + + // Create buffersrc (input) + r = avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_); + if (r < 0) { + qCritical() << "Failed to create buffersrc:" << r; + Close(); + return false; + } + + // Store "previous" filter for linking + AVFilterContext *previous_filter = buffersrc_ctx_; + + // Create tempo + bool create_tempo; + if ((create_tempo = !qFuzzyCompare(tempo, 1.0))) { + // Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside + // those boundaries, we need to daisychain more than one together. + double base = (tempo > 1.0) ? 2.0 : 0.5; + double speed_log = log(tempo) / log(base); + + // This is the number of how many 0.5 or 2.0 tempos we need to daisychain + int whole = qFloor(speed_log); + + // Set speed_log to the remainder + speed_log -= whole; + + for (int i=0;i<=whole;i++) { + double filter_tempo = (i == whole) ? qPow(base, speed_log) : base; + + if (qFuzzyCompare(filter_tempo, 1.0)) { + // This filter would do nothing + continue; + } + + previous_filter = CreateTempoFilter(filter_graph_, + previous_filter, + filter_tempo); + + if (!previous_filter) { + qCritical() << "Failed to create audio tempo filter"; + Close(); + return false; + } + } + } + + // Create conversion filter + if (from.sample_rate() != to.sample_rate() || from.channel_layout() != to.channel_layout() || from.format() != to.format() + || (to.FormatIsPlanar() && create_tempo)) { // Tempo processor automatically converts to packed, + // so if the desired output is planar, it'll need + // to be converted + snprintf(filter_args, 200, "sample_fmts=%s:sample_rates=%d:channel_layouts=0x%" PRIx64, + av_get_sample_fmt_name(to_fmt_), + to.sample_rate(), + to.channel_layout()); + + AVFilterContext *c; + r = avfilter_graph_create_filter(&c, avfilter_get_by_name("aformat"), "fmt", filter_args, nullptr, filter_graph_); + if (r < 0) { + qCritical() << "Failed to create format conversion filter:" << r << filter_args; + Close(); + return false; + } + + r = avfilter_link(previous_filter, 0, c, 0); + if (r < 0) { + qCritical() << "Failed to link filters:" << r; + Close(); + return false; + } + + previous_filter = c; + } + + // Create buffersink (output) + r = avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_); + if (r < 0) { + qCritical() << "Failed to create buffersink:" << r; + Close(); + return false; + } + + r = avfilter_link(previous_filter, 0, buffersink_ctx_, 0); + if (r < 0) { + qCritical() << "Failed to link filters:" << r; + Close(); + return false; + } + + r = avfilter_graph_config(filter_graph_, nullptr); + if (r < 0) { + qCritical() << "Failed to configure graph:" << r; + Close(); + return false; + } + + in_frame_ = av_frame_alloc(); + if (in_frame_) { + in_frame_->sample_rate = from.sample_rate(); + in_frame_->format = from_fmt_; + in_frame_->channel_layout = from.channel_layout(); + in_frame_->channels = from.channel_count(); + in_frame_->pts = 0; + } else { + qCritical() << "Failed to allocate input frame"; + Close(); + return false; + } + + out_frame_ = av_frame_alloc(); + if (!out_frame_) { + qCritical() << "Failed to allocate output frame"; + Close(); + return false; + } + + from_ = from; + to_ = to; + + return true; +} + +void AudioProcessor::Close() +{ + if (filter_graph_) { + avfilter_graph_free(&filter_graph_); + filter_graph_ = nullptr; + buffersrc_ctx_ = nullptr; + buffersink_ctx_ = nullptr; + } + + if (in_frame_) { + av_frame_free(&in_frame_); + in_frame_ = nullptr; + } + + if (out_frame_) { + av_frame_free(&out_frame_); + out_frame_ = nullptr; + } +} + +int AudioProcessor::Convert(float **in, int nb_in_samples, AudioProcessor::Buffer *output) +{ + if (!IsOpen()) { + qCritical() << "Tried to convert on closed processor"; + return -1; + } + + int r = 0; + + if (in && nb_in_samples) { + // Set frame parameters + in_frame_->nb_samples = nb_in_samples; + for (int i=0; idata[i] = reinterpret_cast(in[i]); + in_frame_->linesize[i] = from_.samples_to_bytes(nb_in_samples); + } + + r = av_buffersrc_add_frame_flags(buffersrc_ctx_, in_frame_, AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + qCritical() << "Failed to add frame to buffersrc:" << r; + return r; + } + } + + if (output) { + int nb_channels = to_.channel_count(); + + if (to_.FormatIsPacked()) { + nb_channels = 1; + } + + AudioProcessor::Buffer &result = *output; + result.resize(nb_channels); + + int byte_offset = 0; + + while (true) { + av_frame_unref(out_frame_); + r = av_buffersink_get_frame(buffersink_ctx_, out_frame_); + if (r < 0) { + if (r == AVERROR(EAGAIN)) { + r = 0; + } else { + // Handle unexpected error + qCritical() << "Failed to pull from buffersink:" << r; + } + break; + } + + int nb_bytes = out_frame_->nb_samples * to_.bytes_per_sample_per_channel(); + if (to_.FormatIsPacked()) { + nb_bytes *= to_.channel_count(); + } + + for (int i=0; idata[i], nb_bytes); + } + byte_offset += nb_bytes; + } + av_frame_unref(out_frame_); + } + + return r; +} + +void AudioProcessor::Flush() +{ + int r = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + qCritical() << "Failed to flush:" << r; + } +} + +AVFilterContext *AudioProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo) +{ + // Set up tempo param, which is taken as a C string + char speed_param[20]; + snprintf(speed_param, 20, "%f", tempo); + + AVFilterContext* tempo_ctx = nullptr; + + if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0 + && avfilter_link(link, 0, tempo_ctx, 0) == 0) { + return tempo_ctx; + } + + return nullptr; +} + +} diff --git a/app/audio/tempoprocessor.h b/app/audio/audioprocessor.h similarity index 66% rename from app/audio/tempoprocessor.h rename to app/audio/audioprocessor.h index c56eafc5d..534cca9b1 100644 --- a/app/audio/tempoprocessor.h +++ b/app/audio/audioprocessor.h @@ -18,14 +18,8 @@ ***/ -#ifndef TEMPOPROCESSOR_H -#define TEMPOPROCESSOR_H - -#ifdef __MINGW32__ -#ifndef __USE_MINGW_ANSI_STDIO -#define __USE_MINGW_ANSI_STDIO -#endif -#endif +#ifndef AUDIOPROCESSOR_H +#define AUDIOPROCESSOR_H #include @@ -37,29 +31,26 @@ extern "C" { namespace olive { -class TempoProcessor +class AudioProcessor { public: - TempoProcessor(); + AudioProcessor(); - ~TempoProcessor(); + ~AudioProcessor(); - DISABLE_COPY_MOVE(TempoProcessor) + DISABLE_COPY_MOVE(AudioProcessor) - bool IsOpen() const; - - const double& GetSpeed() const; - - bool Open(const AudioParams& params, const double &speed); - - void Push(const QByteArray &packed); - - void Flush(); - - QByteArray Pull(); + bool Open(const AudioParams &from, const AudioParams &to, double tempo = 1.0); void Close(); + bool IsOpen() const { return filter_graph_; } + + using Buffer = QVector; + int Convert(float **in, int nb_in_samples, AudioProcessor::Buffer *output); + + void Flush(); + private: static AVFilterContext* CreateTempoFilter(AVFilterGraph *graph, AVFilterContext *link, const double& tempo); @@ -69,17 +60,18 @@ private: AVFilterContext* buffersink_ctx_; - AudioParams params_; + AudioParams from_; + AVSampleFormat from_fmt_; - int64_t timestamp_; + AudioParams to_; + AVSampleFormat to_fmt_; - double speed_; + AVFrame *in_frame_; - bool open_; + AVFrame *out_frame_; - bool flushed_; }; } -#endif // TEMPOPROCESSOR_H +#endif // AUDIOPROCESSOR_H diff --git a/app/audio/packedprocessor.cpp b/app/audio/packedprocessor.cpp deleted file mode 100644 index 91fd25a5c..000000000 --- a/app/audio/packedprocessor.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "packedprocessor.h" - -#include "common/ffmpegutils.h" - -namespace olive { - -PackedProcessor::PackedProcessor() : - swr_ctx_(nullptr) -{ -} - -PackedProcessor::~PackedProcessor() -{ - Close(); -} - -bool PackedProcessor::Open(const AudioParams ¶ms) -{ - if (IsOpen()) { - return true; - } - - swr_ctx_ = swr_alloc_set_opts(nullptr, - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(AudioParams::GetPackedEquivalent(params.format())), - params.sample_rate(), - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(params.format()), - params.sample_rate(), - 0, - nullptr); - - if (!swr_ctx_) { - qCritical() << "Failed to allocate resample context"; - return false; - } - - if (swr_init(swr_ctx_) < 0) { - qCritical() << "Failed to init resample context"; - swr_free(&swr_ctx_); - return false; - } - - return true; -} - -QByteArray PackedProcessor::Convert(SampleBufferPtr planar) -{ - if (!IsOpen()) { - qCritical() << "Tried to convert while closed"; - return QByteArray(); - } - - int nb_samples = planar->sample_count(); - if (nb_samples == 0) { - return QByteArray(); - } - - QByteArray output(planar->audio_params().samples_to_bytes(nb_samples), Qt::Uninitialized); - uint8_t *output_data = reinterpret_cast(output.data()); - - int ret = swr_convert(swr_ctx_, &output_data, nb_samples, - const_cast(reinterpret_cast(planar->to_raw_ptrs())), - nb_samples); - if (ret < 0) { - char buf[200]; - av_strerror(ret, buf, 200); - qDebug() << "Packed processor failed with error:" << buf << ret; - } - - return output; -} - -void PackedProcessor::Close() -{ - if (swr_ctx_) { - swr_free(&swr_ctx_); - } -} - -} diff --git a/app/audio/packedprocessor.h b/app/audio/packedprocessor.h deleted file mode 100644 index 8b1bb59c5..000000000 --- a/app/audio/packedprocessor.h +++ /dev/null @@ -1,60 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef PACKEDPROCESSOR_H -#define PACKEDPROCESSOR_H - -extern "C" { -#include -} - -#include "codec/samplebuffer.h" -#include "render/audioparams.h" - -namespace olive { - -class PackedProcessor -{ -public: - PackedProcessor(); - - ~PackedProcessor(); - - DISABLE_COPY_MOVE(PackedProcessor) - - bool Open(const AudioParams ¶ms); - - QByteArray Convert(SampleBufferPtr planar); - - void Close(); - - bool IsOpen() const - { - return swr_ctx_; - } - -private: - SwrContext *swr_ctx_; - -}; - -} - -#endif // PACKEDPROCESSOR_H diff --git a/app/audio/planarprocessor.cpp b/app/audio/planarprocessor.cpp deleted file mode 100644 index 0d5222b76..000000000 --- a/app/audio/planarprocessor.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "planarprocessor.h" - -#include "common/ffmpegutils.h" - -namespace olive { - -PlanarProcessor::PlanarProcessor() : - swr_ctx_(nullptr) -{ -} - -PlanarProcessor::~PlanarProcessor() -{ - Close(); -} - -bool PlanarProcessor::Open(const AudioParams ¶ms) -{ - if (IsOpen()) { - return true; - } - - swr_ctx_ = swr_alloc_set_opts(nullptr, - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(AudioParams::GetPlanarEquivalent(params.format())), - params.sample_rate(), - params.channel_layout(), - FFmpegUtils::GetFFmpegSampleFormat(params.format()), - params.sample_rate(), - 0, - nullptr); - - if (!swr_ctx_) { - qCritical() << "Failed to allocate resample context"; - return false; - } - - if (swr_init(swr_ctx_) < 0) { - qCritical() << "Failed to init resample context"; - swr_free(&swr_ctx_); - return false; - } - - params_ = params; - - return true; -} - -SampleBufferPtr PlanarProcessor::Convert(const QByteArray &packed) -{ - if (!IsOpen()) { - qCritical() << "Tried to convert while closed"; - return nullptr; - } - - if (packed.isEmpty()) { - return nullptr; - } - - int nb_samples_per_channel = params_.bytes_to_samples(packed.size()); - - SampleBufferPtr output = SampleBuffer::CreateAllocated(params_, nb_samples_per_channel); - - const uint8_t *input = reinterpret_cast(packed.constData()); - int ret = swr_convert(swr_ctx_, - reinterpret_cast(output->to_raw_ptrs()), nb_samples_per_channel, - &input, nb_samples_per_channel); - if (ret < 0) { - char buf[200]; - av_strerror(ret, buf, 200); - qDebug() << "Planar processor failed with error:" << buf << ret; - } - - return output; -} - -void PlanarProcessor::Close() -{ - if (swr_ctx_) { - swr_free(&swr_ctx_); - } -} - -} diff --git a/app/audio/planarprocessor.h b/app/audio/planarprocessor.h deleted file mode 100644 index ad5167a95..000000000 --- a/app/audio/planarprocessor.h +++ /dev/null @@ -1,62 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef PLANARPROCESSOR_H -#define PLANARPROCESSOR_H - -extern "C" { -#include -} - -#include "codec/samplebuffer.h" -#include "render/audioparams.h" - -namespace olive { - -class PlanarProcessor -{ -public: - PlanarProcessor(); - - ~PlanarProcessor(); - - DISABLE_COPY_MOVE(PlanarProcessor) - - bool Open(const AudioParams ¶ms); - - SampleBufferPtr Convert(const QByteArray &packed); - - void Close(); - - bool IsOpen() const - { - return swr_ctx_; - } - -private: - SwrContext *swr_ctx_; - - AudioParams params_; - -}; - -} - -#endif // PLANARPROCESSOR_H diff --git a/app/audio/tempoprocessor.cpp b/app/audio/tempoprocessor.cpp deleted file mode 100644 index f22946227..000000000 --- a/app/audio/tempoprocessor.cpp +++ /dev/null @@ -1,269 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2021 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "tempoprocessor.h" - -extern "C" { -#include -#include -#include -} - -#include - -#include "common/ffmpegutils.h" - -namespace olive { - -TempoProcessor::TempoProcessor() : - filter_graph_(nullptr), - buffersrc_ctx_(nullptr), - buffersink_ctx_(nullptr), - open_(false) -{ -} - -TempoProcessor::~TempoProcessor() -{ - Close(); -} - -bool TempoProcessor::IsOpen() const -{ - return open_; -} - -const double &TempoProcessor::GetSpeed() const -{ - return speed_; -} - -bool TempoProcessor::Open(const AudioParams ¶ms, const double& speed) -{ - if (open_) { - return true; - } - - params_ = params; - speed_ = speed; - - // Create AVFilterGraph instance - filter_graph_ = avfilter_graph_alloc(); - if (!filter_graph_) { - qCritical() << "Failed to create AVFilterGraph"; - Close(); - return false; - } - - // Set up audio buffer args - char filter_args[200]; - snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64, - 1, - params_.sample_rate(), - params_.sample_rate(), - FFmpegUtils::GetFFmpegSampleFormat(params_.format()), - params.channel_layout()); - - // Create buffer and buffersink - if (avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_) < 0) { - qCritical() << "Failed to create audio buffer source"; - Close(); - return false; - } - - if (avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_) < 0) { - qCritical() << "Failed to create audio buffer sink"; - Close(); - return false; - } - - // Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside - // those boundaries, we need to daisychain more than one together. - double base = (speed_ > 1.0) ? 2.0 : 0.5; - double speed_log = log(speed_) / log(base); - - // This is the number of how many 0.5 or 2.0 tempos we need to daisychain - int whole = qFloor(speed_log); - - // Set speed_log to the remainder - speed_log -= whole; - - AVFilterContext* previous_filter = buffersrc_ctx_; - - for (int i=0;i<=whole;i++) { - double filter_tempo = (i == whole) ? qPow(base, speed_log) : base; - - if (qFuzzyCompare(filter_tempo, 1.0)) { - // This filter would do nothing - continue; - } - - previous_filter = CreateTempoFilter(filter_graph_, - previous_filter, - filter_tempo); - - if (!previous_filter) { - qCritical() << "Failed to create audio tempo filter"; - Close(); - return false; - } - } - - // Link the last filter to the buffersink - if (avfilter_link(previous_filter, 0, buffersink_ctx_, 0) != 0) { - qCritical() << "Failed to link final filter and buffer sink"; - Close(); - return false; - } - - // Config graph - if (avfilter_graph_config(filter_graph_, nullptr) < 0) { - qCritical() << "Failed to configure filter graph"; - Close(); - return false; - } - - timestamp_ = 0; - - open_ = true; - - flushed_ = false; - - return true; -} - -void TempoProcessor::Push(const QByteArray &packed) -{ - if (!IsOpen()) { - qWarning() << "Tried to push to closed TempoProcessor"; - return; - } - - if (flushed_) { - qWarning() << "Tried to push to flushed TempoProcessor"; - return; - } - - AVFrame* src_frame = av_frame_alloc(); - - if (!src_frame) { - qCritical() << "Failed to allocate source frame"; - return; - } - - // Allocate a buffer for the number of samples we got - src_frame->sample_rate = params_.sample_rate(); - src_frame->format = FFmpegUtils::GetFFmpegSampleFormat(params_.format()); - src_frame->channel_layout = params_.channel_layout(); - src_frame->nb_samples = params_.bytes_to_samples(packed.size()); - src_frame->pts = timestamp_; - timestamp_ += src_frame->nb_samples; - - if (av_frame_get_buffer(src_frame, 0) < 0) { - qCritical() << "Failed to allocate buffer for source frame"; - av_frame_free(&src_frame); - return; - } - - // Copy buffer from data array to frame - memcpy(src_frame->data[0], packed, packed.size()); - - int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, src_frame, AV_BUFFERSRC_FLAG_KEEP_REF); - - if (ret < 0) { - qCritical() << "Failed to feed buffer source" << ret; - } - - if (src_frame) { - av_frame_free(&src_frame); - } -} - -void TempoProcessor::Flush() -{ - if (!flushed_) { - int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF); - if (ret < 0) { - qCritical() << "Failed to feed buffer source" << ret; - } - flushed_ = true; - } -} - -QByteArray TempoProcessor::Pull() -{ - QByteArray b; - AVFrame *processed_frame = av_frame_alloc(); - - // Try to pull samples from the buffersink - int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame); - - if (ret < 0) { - // We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the - // error might be fatal... - if (ret != AVERROR(EAGAIN)) { - qCritical() << "Failed to pull from buffersink" << ret; - } - - av_frame_free(&processed_frame); - return b; - } - - b.resize(params_.samples_to_bytes(processed_frame->nb_samples)); - - // Copy the bytes - memcpy(b.data(), processed_frame->data[0], b.size()); - - // If the index has reached the limit of this processed frame, we can dispose of the frame now - av_frame_free(&processed_frame); - - return b; -} - -void TempoProcessor::Close() -{ - open_ = false; - - if (filter_graph_) { - avfilter_graph_free(&filter_graph_); - filter_graph_ = nullptr; - } - - buffersrc_ctx_ = nullptr; - buffersink_ctx_ = nullptr; -} - -AVFilterContext *TempoProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo) -{ - // Set up tempo param, which is taken as a C string - char speed_param[20]; - snprintf(speed_param, 20, "%f", tempo); - - AVFilterContext* tempo_ctx = nullptr; - - if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0 - && avfilter_link(link, 0, tempo_ctx, 0) == 0) { - return tempo_ctx; - } - - return nullptr; -} - -} diff --git a/app/render/audioparams.h b/app/render/audioparams.h index 9213fd872..cb4c591b7 100644 --- a/app/render/audioparams.h +++ b/app/render/audioparams.h @@ -186,6 +186,26 @@ public: duration_ = duration; } + static bool FormatIsPacked(Format f) + { + return f >= kPackedStart && f < kPackedEnd; + } + + bool FormatIsPacked() const + { + return FormatIsPacked(format_); + } + + static bool FormatIsPlanar(Format f) + { + return f >= kPlanarStart && f < kPlanarEnd; + } + + bool FormatIsPlanar() const + { + return FormatIsPlanar(format_); + } + qint64 time_to_bytes(const double& time) const; qint64 time_to_bytes(const rational& time) const; qint64 time_to_bytes_per_channel(const double& time) const; diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index dc4556f8d..7e3876f8d 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -25,9 +25,7 @@ #include #include -#include "audio/packedprocessor.h" -#include "audio/planarprocessor.h" -#include "audio/tempoprocessor.h" +#include "audio/audioprocessor.h" #include "node/block/clip/clip.h" #include "node/block/transition/transition.h" #include "node/project/project.h" @@ -300,14 +298,10 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim samples_from_this_block->silence(); } else if (!qFuzzyCompare(speed_value, 1.0)) { if (clip_cast->maintain_audio_pitch()) { - PackedProcessor packer; - packer.Open(samples_from_this_block->audio_params()); + AudioProcessor processor; - QByteArray packed = packer.Convert(samples_from_this_block); - - if (!packed.isEmpty()) { - TempoProcessor tp; - tp.Open(samples_from_this_block->audio_params(), speed_value); + if (processor.Open(samples_from_this_block->audio_params(), samples_from_this_block->audio_params(), speed_value)) { + AudioProcessor::Buffer out; // FIXME: This is not the best way to do this, the TempoProcessor works best // when it's given a continuous stream of audio, which is challenging @@ -315,15 +309,31 @@ NodeValueTable RenderProcessor::GenerateBlockTable(const Track *track, const Tim // well on export (assuming audio is all generated at once on export), but // users may hear clicks and pops in the audio during preview due to this // approach. - tp.Push(packed); - tp.Flush(); - packed = tp.Pull(); - tp.Close(); + int r = processor.Convert(samples_from_this_block->to_raw_ptrs(), samples_from_this_block->sample_count(), nullptr); - if (!packed.isEmpty()) { - PlanarProcessor planar; - planar.Open(samples_from_this_block->audio_params()); - samples_from_this_block = planar.Convert(packed); + if (r < 0) { + qCritical() << "Failed to change tempo of audio:" << r; + } else { + processor.Flush(); + + processor.Convert(nullptr, 0, &out); + + if (!out.empty()) { + int nb_samples = out.front().size() * samples_from_this_block->audio_params().bytes_per_sample_per_channel(); + + if (nb_samples) { + SampleBufferPtr new_samples = SampleBuffer::Create(); + new_samples->set_audio_params(samples_from_this_block->audio_params()); + new_samples->set_sample_count(nb_samples); + new_samples->allocate(); + + for (int i=0; idata(i), out[i].data(), out[i].size()); + } + + samples_from_this_block = new_samples; + } + } } } } else { diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index d404fb5ed..012e7650e 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -210,8 +210,7 @@ void ViewerWidget::ConnectNodeEvent(ViewerOutput *n) last_length_ = 0; LengthChangedSlot(n->GetLength()); - AudioParams ap = n->GetAudioParams(); - packed_processor_.Open(ap); + UpdateAudioProcessor(); ColorManager* color_manager = n->project()->color_manager(); @@ -245,7 +244,7 @@ void ViewerWidget::DisconnectNodeEvent(ViewerOutput *n) disconnect(n->video_frame_cache(), &FrameHashCache::Shifted, this, &ViewerWidget::ViewerShiftedRange); disconnect(n, &ViewerOutput::TextureInputChanged, this, &ViewerWidget::UpdateStack); - packed_processor_.Close(); + CloseAudioProcessor(); SetDisplayImage(QVariant()); @@ -456,6 +455,21 @@ void ViewerWidget::DisarmRecording() record_armed_ = false; } +void ViewerWidget::UpdateAudioProcessor() +{ + audio_processor_.Close(); + + AudioParams ap = GetConnectedNode()->GetAudioParams(); + AudioParams packed = ap; + packed.set_format(AudioParams::GetPackedEquivalent(ap.format())); + audio_processor_.Open(ap, packed, (playback_speed_ == 0) ? 1 : std::abs(playback_speed_)); +} + +void ViewerWidget::CloseAudioProcessor() +{ + audio_processor_.Close(); +} + void ViewerWidget::QueueNextAudioBuffer() { rational queue_end = audio_playback_queue_time_ + (kAudioPlaybackInterval * playback_speed_); @@ -493,23 +507,23 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() } // Convert to packed data for audio output - QByteArray pack = packed_processor_.Convert(samples); - - // If the tempo must be adjusted, adjust now - if (tempo_processor_.IsOpen()) { - tempo_processor_.Push(pack); - pack = tempo_processor_.Pull(); - } + AudioProcessor::Buffer buf; + int r = audio_processor_.Convert(samples->to_raw_ptrs(), samples->sample_count(), &buf); // TempoProcessor may have emptied the array - if (!pack.isEmpty()) { - if (prequeuing_audio_) { - // Add to prequeued audio buffer - prequeued_audio_.append(pack); - } else { - // Push directly to audio manager - AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack); + if (r >= 0) { + if (!buf.empty()) { + const QByteArray &pack = buf.at(0); + if (prequeuing_audio_) { + // Add to prequeued audio buffer + prequeued_audio_.append(pack); + } else { + // Push directly to audio manager + AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack); + } } + } else { + qCritical() << "Failed to process audio for playback:" << r; } } } @@ -540,10 +554,19 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing() samples->transform_volume_for_sample(samples->sample_count() - i - 1, amt); }*/ - QByteArray packed = packed_processor_.Convert(samples); - AudioManager::instance()->ClearBufferedOutput(); - AudioManager::instance()->PushToOutput(samples->audio_params(), packed); - AudioMonitor::PushBytesOnAll(packed); + AudioProcessor::Buffer buf; + int r = audio_processor_.Convert(samples->to_raw_ptrs(), samples->sample_count(), &buf); + + if (r >= 0) { + if (!buf.empty()) { + const QByteArray &packed = buf.at(0); + AudioManager::instance()->ClearBufferedOutput(); + AudioManager::instance()->PushToOutput(samples->audio_params(), packed); + AudioMonitor::PushBytesOnAll(packed); + } + } else { + qCritical() << "Failed to process audio for scrubbing:" << r; + } } } } @@ -675,9 +698,7 @@ void ViewerWidget::PlayInternal(int speed, bool in_to_out_only) AudioManager::instance()->SetOutputNotifyInterval(ap.time_to_bytes(kAudioPlaybackInterval)); connect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer); - if (std::abs(playback_speed_) > 1) { - tempo_processor_.Open(GetConnectedNode()->GetAudioParams(), std::abs(playback_speed_)); - } + UpdateAudioProcessor(); static const int prequeue_count = 2; prequeuing_audio_ = prequeue_count; // Queue two buffers ahead of time @@ -721,9 +742,7 @@ void ViewerWidget::PauseInternal() disconnect(AudioManager::instance(), &AudioManager::OutputNotify, this, &ViewerWidget::QueueNextAudioBuffer); qDeleteAll(audio_playback_queue_); audio_playback_queue_.clear(); - if (tempo_processor_.IsOpen()) { - tempo_processor_.Close(); - } + UpdateAudioProcessor(); foreach (ViewerWidget* viewer, instances_) { viewer->auto_cacher_.SetAudioPaused(false); @@ -1434,11 +1453,7 @@ void ViewerWidget::UpdateRendererVideoParameters() void ViewerWidget::UpdateRendererAudioParameters() { - packed_processor_.Close(); - - AudioParams ap = GetConnectedNode()->GetAudioParams(); - - packed_processor_.Open(ap); + UpdateAudioProcessor(); } void ViewerWidget::SetZoomFromMenu(QAction *action) diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 4bf043af1..be515ff4d 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -28,8 +28,7 @@ #include #include -#include "audio/packedprocessor.h" -#include "audio/tempoprocessor.h" +#include "audio/audioprocessor.h" #include "audiowaveformview.h" #include "common/rational.h" #include "node/output/viewer/viewer.h" @@ -212,6 +211,9 @@ private: void DisarmRecording(); + void UpdateAudioProcessor(); + void CloseAudioProcessor(); + QStackedWidget* stack_; ViewerSizer* sizer_; @@ -255,8 +257,7 @@ private: std::list audio_playback_queue_; rational audio_playback_queue_time_; - PackedProcessor packed_processor_; - TempoProcessor tempo_processor_; + AudioProcessor audio_processor_; QByteArray prequeued_audio_; static const rational kAudioPlaybackInterval; From d727ab535319c57d23b3ca15dfe50c00fdc920c0 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 5 May 2022 15:30:34 -0700 Subject: [PATCH 7/9] viewer: fixed audio reverse playback regression --- app/widget/viewer/viewer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 012e7650e..5218e6698 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -476,7 +476,8 @@ void ViewerWidget::QueueNextAudioBuffer() // Clamp queue end by zero and the audio length queue_end = clamp(queue_end, rational(0), GetConnectedNode()->GetAudioLength()); - if (queue_end <= audio_playback_queue_time_) { + if ((playback_speed_ > 0 && queue_end <= audio_playback_queue_time_) + || (playback_speed_ < 0 && queue_end >= audio_playback_queue_time_)) { // This will queue nothing, so stop the loop here if (prequeuing_audio_) { DecrementPrequeuedAudio(); From 5bed4396ea9d11d677b09a29f2eb83dc8dc09f45 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 5 May 2022 16:32:17 -0700 Subject: [PATCH 8/9] use audioprocessor to match device's output --- app/audio/audiomanager.cpp | 15 +++++-- app/audio/audiomanager.h | 5 ++- app/audio/audioprocessor.h | 3 ++ app/config/config.cpp | 4 ++ .../preferences/tabs/preferencesaudiotab.cpp | 40 +++++++++++++++++++ .../preferences/tabs/preferencesaudiotab.h | 4 ++ .../standardcombos/sampleformatcombobox.h | 18 +++++++++ app/widget/viewer/viewer.cpp | 28 +++++++++---- app/widget/viewer/viewer.h | 3 +- 9 files changed, 107 insertions(+), 13 deletions(-) diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index fe791b3d5..3817f285a 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -80,10 +80,11 @@ int InputCallback(const void *input, void *output, unsigned long frameCount, con return paContinue; } -void AudioManager::PushToOutput(const AudioParams ¶ms, const QByteArray &samples) +bool AudioManager::PushToOutput(const AudioParams ¶ms, const QByteArray &samples, QString *error) { if (output_device_ == paNoDevice) { - return; + if (error) *error = tr("No output device is set"); + return false; } if (output_params_ != params || output_stream_ == nullptr) { @@ -93,7 +94,13 @@ void AudioManager::PushToOutput(const AudioParams ¶ms, const QByteArray &sam PaStreamParameters p = GetPortAudioParams(params, output_device_); - Pa_OpenStream(&output_stream_, nullptr, &p, output_params_.sample_rate(), paFramesPerBufferUnspecified, paNoFlag, OutputCallback, output_buffer_); + PaError r = Pa_OpenStream(&output_stream_, nullptr, &p, output_params_.sample_rate(), paFramesPerBufferUnspecified, paNoFlag, OutputCallback, output_buffer_); + if (r != paNoError) { + // Unhandled error + //qCritical() << "Failed to open output stream:" << Pa_GetErrorText(r); + if (error) *error = Pa_GetErrorText(r); + return false; + } output_buffer_->set_bytes_per_frame(output_params_.samples_to_bytes(1)); } @@ -103,6 +110,8 @@ void AudioManager::PushToOutput(const AudioParams ¶ms, const QByteArray &sam if (!Pa_IsStreamActive(output_stream_)) { Pa_StartStream(output_stream_); } + + return true; } void AudioManager::ClearBufferedOutput() diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index 20d1be75b..2916d29d2 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -27,6 +27,7 @@ #include #include "audiovisualwaveform.h" +#include "audio/audioprocessor.h" #include "common/define.h" #include "codec/ffmpeg/ffmpegencoder.h" #include "render/audioparams.h" @@ -52,7 +53,7 @@ public: void SetOutputNotifyInterval(int n); - void PushToOutput(const AudioParams ¶ms, const QByteArray& samples); + bool PushToOutput(const AudioParams ¶ms, const QByteArray& samples, QString *error = nullptr); void ClearBufferedOutput(); @@ -86,6 +87,8 @@ public: signals: void OutputNotify(); + void OutputParamsChanged(); + private: AudioManager(); diff --git a/app/audio/audioprocessor.h b/app/audio/audioprocessor.h index 534cca9b1..38826c707 100644 --- a/app/audio/audioprocessor.h +++ b/app/audio/audioprocessor.h @@ -51,6 +51,9 @@ public: void Flush(); + const AudioParams &from() const { return from_; } + const AudioParams &to() const { return to_; } + private: static AVFilterContext* CreateTempoFilter(AVFilterGraph *graph, AVFilterContext *link, const double& tempo); diff --git a/app/config/config.cpp b/app/config/config.cpp index d114b0a10..0aaaa6253 100644 --- a/app/config/config.cpp +++ b/app/config/config.cpp @@ -121,6 +121,10 @@ void Config::SetDefaults() SetEntryInternal(QStringLiteral("AudioOutput"), NodeValue::kText, QString()); SetEntryInternal(QStringLiteral("AudioInput"), NodeValue::kText, QString()); + SetEntryInternal(QStringLiteral("AudioOutputSampleRate"), NodeValue::kInt, 48000); + SetEntryInternal(QStringLiteral("AudioOutputChannelLayout"), NodeValue::kInt, AV_CH_LAYOUT_STEREO); + SetEntryInternal(QStringLiteral("AudioOutputSampleFormat"), NodeValue::kInt, AudioParams::kFormatSigned16Packed); + SetEntryInternal(QStringLiteral("AudioRecordingFormat"), NodeValue::kInt, ExportFormat::kFormatWAV); SetEntryInternal(QStringLiteral("AudioRecordingCodec"), NodeValue::kInt, ExportCodec::kCodecPCM); SetEntryInternal(QStringLiteral("AudioRecordingSampleRate"), NodeValue::kInt, 48000); diff --git a/app/dialog/preferences/tabs/preferencesaudiotab.cpp b/app/dialog/preferences/tabs/preferencesaudiotab.cpp index c8760ae8a..7d2edc892 100644 --- a/app/dialog/preferences/tabs/preferencesaudiotab.cpp +++ b/app/dialog/preferences/tabs/preferencesaudiotab.cpp @@ -69,6 +69,40 @@ PreferencesAudioTab::PreferencesAudioTab() audio_output_devices_ = new QComboBox(); output_layout->addWidget(audio_output_devices_, row, 1); + + row++; + + { + int output_row = 0; + + QGroupBox *output_param_group = new QGroupBox(tr("Advanced")); + output_layout->addWidget(output_param_group, row, 0, 1, 2); + + QGridLayout *output_param_layout = new QGridLayout(output_param_group); + + output_param_layout->addWidget(new QLabel(tr("Sample Rate:")), output_row, 0); + + output_rate_combo_ = new SampleRateComboBox(); + output_rate_combo_->SetSampleRate(OLIVE_CONFIG("AudioOutputSampleRate").toInt()); + output_param_layout->addWidget(output_rate_combo_, output_row, 1); + + output_row++; + + output_param_layout->addWidget(new QLabel(tr("Channel Layout:")), output_row, 0); + + output_ch_layout_combo_ = new ChannelLayoutComboBox(); + output_ch_layout_combo_->SetChannelLayout(OLIVE_CONFIG("AudioOutputChannelLayout").toULongLong()); + output_param_layout->addWidget(output_ch_layout_combo_, output_row, 1); + + output_row++; + + output_param_layout->addWidget(new QLabel(tr("Sample Format:")), output_row, 0); + + output_fmt_combo_ = new SampleFormatComboBox(); + output_fmt_combo_->SetPackedFormats(); + output_fmt_combo_->SetSampleFormat(static_cast(OLIVE_CONFIG("AudioOutputSampleFormat").toInt())); + output_param_layout->addWidget(output_fmt_combo_, output_row, 1); + } } row = 0; @@ -146,12 +180,18 @@ void PreferencesAudioTab::Accept(MultiUndoCommand *command) AudioManager::instance()->SetOutputDevice(output_device); AudioManager::instance()->SetInputDevice(input_device); + OLIVE_CONFIG("AudioOutputSampleRate") = output_rate_combo_->GetSampleRate(); + OLIVE_CONFIG("AudioOutputChannelLayout") = QVariant::fromValue(output_ch_layout_combo_->GetChannelLayout()); + OLIVE_CONFIG("AudioOutputSampleFormat") = output_fmt_combo_->GetSampleFormat(); + OLIVE_CONFIG("AudioRecordingFormat") = record_format_combo_->GetFormat(); OLIVE_CONFIG("AudioRecordingCodec") = record_options_->GetCodec(); OLIVE_CONFIG("AudioRecordingSampleRate") = record_options_->sample_rate_combobox()->GetSampleRate(); OLIVE_CONFIG("AudioRecordingChannelLayout") = QVariant::fromValue(record_options_->channel_layout_combobox()->GetChannelLayout()); OLIVE_CONFIG("AudioRecordingBitRate") = QVariant::fromValue(record_options_->bit_rate_slider()->GetValue()); OLIVE_CONFIG("AudioRecordingSampleFormat") = record_options_->sample_format_combobox()->GetSampleFormat(); + + emit AudioManager::instance()->OutputParamsChanged(); } void PreferencesAudioTab::RefreshBackends() diff --git a/app/dialog/preferences/tabs/preferencesaudiotab.h b/app/dialog/preferences/tabs/preferencesaudiotab.h index 8fccd4276..550b6fa9e 100644 --- a/app/dialog/preferences/tabs/preferencesaudiotab.h +++ b/app/dialog/preferences/tabs/preferencesaudiotab.h @@ -61,6 +61,10 @@ private: */ QPushButton* refresh_devices_btn_; + SampleRateComboBox *output_rate_combo_; + ChannelLayoutComboBox *output_ch_layout_combo_; + SampleFormatComboBox *output_fmt_combo_; + ExportFormatComboBox *record_format_combo_; ExportAudioTab *record_options_; diff --git a/app/widget/standardcombos/sampleformatcombobox.h b/app/widget/standardcombos/sampleformatcombobox.h index a48e9016a..d52a20709 100644 --- a/app/widget/standardcombos/sampleformatcombobox.h +++ b/app/widget/standardcombos/sampleformatcombobox.h @@ -57,6 +57,24 @@ public: } } + void SetPackedFormats() + { + AudioParams::Format tmp = AudioParams::kFormatInvalid; + + if (attempt_to_restore_format_) { + tmp = GetSampleFormat(); + } + + clear(); + for (int i=AudioParams::kPackedStart; i(i)); + } + + if (attempt_to_restore_format_) { + SetSampleFormat(tmp); + } + } + AudioParams::Format GetSampleFormat() const { return static_cast(this->currentData().toInt()); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 5218e6698..3cac18a5e 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -141,6 +141,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) : connect(Core::instance(), &Core::ColorPickerEnabled, this, &ViewerWidget::SetSignalCursorColorEnabled); connect(this, &ViewerWidget::CursorColor, Core::instance(), &Core::ColorPickerColorEmitted); + connect(AudioManager::instance(), &AudioManager::OutputParamsChanged, this, &ViewerWidget::UpdateAudioProcessor); } ViewerWidget::~ViewerWidget() @@ -457,12 +458,16 @@ void ViewerWidget::DisarmRecording() void ViewerWidget::UpdateAudioProcessor() { - audio_processor_.Close(); + if (GetConnectedNode()) { + audio_processor_.Close(); - AudioParams ap = GetConnectedNode()->GetAudioParams(); - AudioParams packed = ap; - packed.set_format(AudioParams::GetPackedEquivalent(ap.format())); - audio_processor_.Open(ap, packed, (playback_speed_ == 0) ? 1 : std::abs(playback_speed_)); + AudioParams ap = GetConnectedNode()->GetAudioParams(); + AudioParams packed(OLIVE_CONFIG("AudioOutputSampleRate").toInt(), + OLIVE_CONFIG("AudioOutputChannelLayout").toULongLong(), + static_cast(OLIVE_CONFIG("AudioOutputSampleFormat").toInt())); + + audio_processor_.Open(ap, packed, (playback_speed_ == 0) ? 1 : std::abs(playback_speed_)); + } } void ViewerWidget::CloseAudioProcessor() @@ -520,7 +525,7 @@ void ViewerWidget::ReceivedAudioBufferForPlayback() prequeued_audio_.append(pack); } else { // Push directly to audio manager - AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), pack); + AudioManager::instance()->PushToOutput(audio_processor_.to(), pack); } } } else { @@ -560,9 +565,12 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing() if (r >= 0) { if (!buf.empty()) { + QString error; const QByteArray &packed = buf.at(0); AudioManager::instance()->ClearBufferedOutput(); - AudioManager::instance()->PushToOutput(samples->audio_params(), packed); + if (!AudioManager::instance()->PushToOutput(audio_processor_.to(), packed, &error)) { + Core::instance()->ShowStatusBarMessage(tr("Audio scrubbing failed: %1").arg(error)); + } AudioMonitor::PushBytesOnAll(packed); } } else { @@ -872,7 +880,11 @@ void ViewerWidget::FinishPlayPreprocess() // Start audio waveform playback if (!prequeued_audio_.isEmpty()) { - AudioManager::instance()->PushToOutput(GetConnectedNode()->GetAudioParams(), prequeued_audio_); + QString error; + if (!AudioManager::instance()->PushToOutput(audio_processor_.to(), prequeued_audio_, &error)) { + QMessageBox::critical(this, tr("Audio Error"), tr("Failed to start audio: %1\n\n" + "Please check your audio preferences and try again.").arg(error)); + } prequeued_audio_.clear(); AudioMonitor::StartWaveformOnAll(&GetConnectedNode()->audio_playback_cache()->visual(), diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index be515ff4d..ba7f7be56 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -211,7 +211,6 @@ private: void DisarmRecording(); - void UpdateAudioProcessor(); void CloseAudioProcessor(); QStackedWidget* stack_; @@ -319,6 +318,8 @@ private slots: void ForceRequeueFromCurrentTime(); + void UpdateAudioProcessor(); + }; } From 5920af9901194400ade450b93cf972b85252f3a8 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 5 May 2022 16:49:15 -0700 Subject: [PATCH 9/9] fixed issue with audio failing to resolve Regression, particularly evident in footage viewer --- app/render/renderprocessor.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index 7e3876f8d..ce1db8b41 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -191,7 +191,11 @@ void RenderProcessor::Run() table = GenerateTable(texture_output, viewer->GetValueHintForInput(ViewerOutput::kSamplesInput),time); } - QVariant sample_variant = table.Get(NodeValue::kSamples); + NodeValue sample_val = table.GetWithMeta(NodeValue::kSamples); + + ResolveJobs(sample_val, time); + + QVariant sample_variant = sample_val.data(); SampleBufferPtr samples = sample_variant.value(); if (samples && ticket_->property("enablewaveforms").toBool()) { AudioVisualWaveform vis;