diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index cee334a34..b785c3c78 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -26,5 +26,7 @@ set(OLIVE_SOURCES audio/outputmanager.cpp audio/sampleformat.h audio/sampleformat.cpp + audio/tempoprocessor.h + audio/tempoprocessor.cpp PARENT_SCOPE ) diff --git a/app/audio/audiomanager.cpp b/app/audio/audiomanager.cpp index 22ee8d906..57ee51fd0 100644 --- a/app/audio/audiomanager.cpp +++ b/app/audio/audiomanager.cpp @@ -80,9 +80,9 @@ void AudioManager::PushToOutput(const QByteArray &samples) output_manager_.Push(samples); } -void AudioManager::StartOutput(QIODevice *device) +void AudioManager::StartOutput(QIODevice *device, int playback_speed) { - output_manager_.PullFromDevice(device); + output_manager_.PullFromDevice(device, playback_speed); } void AudioManager::StopOutput() @@ -147,6 +147,8 @@ void AudioManager::SetOutputParams(const AudioRenderingParams ¶ms) if (output_params_ != params) { output_params_ = params; + output_manager_.SetParameters(params); + // Refresh output device SetOutputDevice(output_device_info_); } @@ -168,6 +170,24 @@ const QList &AudioManager::ListOutputDevices() return output_devices_; } +void AudioManager::ReverseBuffer(char *buffer, int buffer_size, int sample_size) +{ + int half_buffer_sz = buffer_size / 2; + char* temp_buffer = new char[sample_size]; + + for (int src_index=0;src_index(sample_size)); + memcpy(src_ptr, dst_ptr, static_cast(sample_size)); + memcpy(dst_ptr, temp_buffer, static_cast(sample_size)); + } + + delete [] temp_buffer; +} + AudioManager::AudioManager() : input_(nullptr), input_file_(nullptr), diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index eb9f11227..ee3de09eb 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -79,7 +79,7 @@ public: * * This takes ownership of the QIODevice and will delete it when StopOutput() is called */ - void StartOutput(QIODevice* device); + void StartOutput(QIODevice* device, int playback_speed); /** * @brief Stop audio output immediately @@ -95,6 +95,8 @@ public: const QList& ListInputDevices(); const QList& ListOutputDevices(); + static void ReverseBuffer(char* buffer, int size, int resolution); + signals: void DeviceListReady(); diff --git a/app/audio/outputdeviceproxy.cpp b/app/audio/outputdeviceproxy.cpp index 63dbeeb90..e01f04be5 100644 --- a/app/audio/outputdeviceproxy.cpp +++ b/app/audio/outputdeviceproxy.cpp @@ -1,5 +1,6 @@ #include "outputdeviceproxy.h" +#include "audiomanager.h" #include "bufferaverage.h" AudioOutputDeviceProxy::AudioOutputDeviceProxy() : @@ -8,9 +9,24 @@ AudioOutputDeviceProxy::AudioOutputDeviceProxy() : { } -void AudioOutputDeviceProxy::SetDevice(QIODevice *device) +void AudioOutputDeviceProxy::SetParameters(const AudioRenderingParams ¶ms) +{ + params_ = params; +} + +void AudioOutputDeviceProxy::SetDevice(QIODevice *device, int playback_speed) { device_ = device; + + if (!device_->isOpen()) { + device_->open(QIODevice::ReadOnly); + } + + playback_speed_ = playback_speed; + + if (qAbs(playback_speed_) != 1) { + tempo_processor_.Open(params_, qAbs(playback_speed_)); + } } void AudioOutputDeviceProxy::SetSendAverages(bool send) @@ -23,14 +39,39 @@ void AudioOutputDeviceProxy::close() QIODevice::close(); device_->close(); + + if (tempo_processor_.IsOpen()) { + tempo_processor_.Close(); + } } qint64 AudioOutputDeviceProxy::readData(char *data, qint64 maxlen) { if (device_) { - qint64 read_count = device_->read(data, maxlen); - if (send_averages_ && read_count > 0) { + qint64 read_count; + + + + if (tempo_processor_.IsOpen()) { + + while ((read_count = tempo_processor_.Pull(data, static_cast(maxlen))) == 0) { + int dev_read = static_cast(ReverseAwareRead(data, maxlen)); + + if (!dev_read) { + break; + } + + tempo_processor_.Push(data, dev_read); + } + + } else { + // If we aren't doing any tempo processing, simply passthrough the read signal + read_count = ReverseAwareRead(data, maxlen); + } + + // If we read any + if (read_count > 0 && send_averages_) { emit ProcessedAverages(AudioBufferAverage::ProcessAverages(data, static_cast(read_count))); } @@ -44,3 +85,32 @@ qint64 AudioOutputDeviceProxy::writeData(const char *data, qint64 maxSize) { return -1; } + +qint64 AudioOutputDeviceProxy::ReverseAwareRead(char *data, qint64 maxlen) +{ + qint64 new_pos; + + if (playback_speed_ < 0) { + // If we're reversing, we'll seek back by maxlen bytes before we read + new_pos = device_->pos() - maxlen; + + if (new_pos < 0) { + maxlen = device_->pos(); + + new_pos = 0; + } + + device_->seek(new_pos); + } + + qint64 read_count = device_->read(data, maxlen); + + if (playback_speed_ < 0) { + device_->seek(new_pos); + + // Reverse the samples here + AudioManager::ReverseBuffer(data, static_cast(read_count), params_.samples_to_bytes(1)); + } + + return read_count; +} diff --git a/app/audio/outputdeviceproxy.h b/app/audio/outputdeviceproxy.h index d2dc3f821..48332282e 100644 --- a/app/audio/outputdeviceproxy.h +++ b/app/audio/outputdeviceproxy.h @@ -3,13 +3,17 @@ #include +#include "tempoprocessor.h" + class AudioOutputDeviceProxy : public QIODevice { Q_OBJECT public: AudioOutputDeviceProxy(); - void SetDevice(QIODevice* device); + void SetParameters(const AudioRenderingParams& params); + + void SetDevice(QIODevice* device, int playback_speed); void SetSendAverages(bool send); @@ -24,10 +28,18 @@ protected: virtual qint64 writeData(const char *data, qint64 maxSize) override; private: + qint64 ReverseAwareRead(char* data, qint64 maxlen); + QIODevice* device_; + TempoProcessor tempo_processor_; + bool send_averages_; + AudioRenderingParams params_; + + int playback_speed_; + }; #endif // AUDIOOUTPUTDEVICEPROXY_H diff --git a/app/audio/outputmanager.cpp b/app/audio/outputmanager.cpp index 9bcccd1bc..de27d92ac 100644 --- a/app/audio/outputmanager.cpp +++ b/app/audio/outputmanager.cpp @@ -70,7 +70,12 @@ void AudioOutputManager::ResetToPushMode() } } -void AudioOutputManager::PullFromDevice(QIODevice *device) +void AudioOutputManager::SetParameters(const AudioRenderingParams ¶ms) +{ + device_proxy_.SetParameters(params); +} + +void AudioOutputManager::PullFromDevice(QIODevice *device, int playback_speed) { if (!output_ || !device) { return; @@ -82,7 +87,7 @@ void AudioOutputManager::PullFromDevice(QIODevice *device) pushed_samples_.clear(); // Pull from the device - device_proxy_.SetDevice(device); + device_proxy_.SetDevice(device, playback_speed); device_proxy_.open(QIODevice::ReadOnly); output_->start(&device_proxy_); } diff --git a/app/audio/outputmanager.h b/app/audio/outputmanager.h index a8cb0ca65..58b9cb3a0 100644 --- a/app/audio/outputmanager.h +++ b/app/audio/outputmanager.h @@ -51,10 +51,12 @@ public: * This will clear any pushed samples or QIODevices currently being read and will start reading from this next time * the audio output requests data. */ - void PullFromDevice(QIODevice* device); + void PullFromDevice(QIODevice* device, int playback_speed); void ResetToPushMode(); + void SetParameters(const AudioRenderingParams& params); + signals: /** * @brief Signal emitted when samples are sent to the output device diff --git a/app/audio/tempoprocessor.cpp b/app/audio/tempoprocessor.cpp new file mode 100644 index 000000000..bbcd08b7d --- /dev/null +++ b/app/audio/tempoprocessor.cpp @@ -0,0 +1,252 @@ +#include "tempoprocessor.h" + +extern "C" { +#include +#include +#include +} + +#include + +#include "codec/ffmpeg/ffmpegcommon.h" + +TempoProcessor::TempoProcessor() : + filter_graph_(nullptr), + buffersrc_ctx_(nullptr), + buffersink_ctx_(nullptr), + open_(false) +{ + +} + +bool TempoProcessor::IsOpen() const +{ + return open_; +} + +const double &TempoProcessor::GetSpeed() const +{ + return speed_; +} + +bool TempoProcessor::Open(const AudioRenderingParams ¶ms, const double& speed) +{ + if (open_) { + return true; + } + + params_ = params; + speed_ = speed; + + // Create AVFilterGraph instance + filter_graph_ = avfilter_graph_alloc(); + if (!filter_graph_) { + qCritical() << "Failed to create AVFilterGraph"; + Close(); + return false; + } + + // Set up audio buffer args + char filter_args[200]; + snprintf(filter_args, 200, "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%" PRIx64, + 1, + params_.sample_rate(), + params_.sample_rate(), + FFmpegCommon::GetFFmpegSampleFormat(params_.format()), + params.channel_layout()); + + // Create buffer and buffersink + if (avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("abuffer"), "in", filter_args, nullptr, filter_graph_) < 0) { + qCritical() << "Failed to create audio buffer source"; + Close(); + return false; + } + + if (avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("abuffersink"), "out", nullptr, nullptr, filter_graph_) < 0) { + qCritical() << "Failed to create audio buffer sink"; + Close(); + return false; + } + + // Create audio tempo filters: FFmpeg's atempo can only be set between 0.5 and 2.0. If the requested speed is outside + // those boundaries, we need to daisychain more than one together. + double base = (speed_ > 1.0) ? 2.0 : 0.5; + double speed_log = log(speed_) / log(base); + + // This is the number of how many 0.5 or 2.0 tempos we need to daisychain + int whole = qFloor(speed_log); + + // Set speed_log to the remainder + speed_log -= whole; + + AVFilterContext* previous_filter = buffersrc_ctx_; + + for (int i=0;i<=whole;i++) { + double filter_tempo = (i == whole) ? qPow(base, speed_log) : base; + + if (qFuzzyCompare(filter_tempo, 1.0)) { + // This filter would do nothing + continue; + } + + previous_filter = CreateTempoFilter(filter_graph_, + previous_filter, + filter_tempo); + + if (!previous_filter) { + qCritical() << "Failed to create audio tempo filter"; + Close(); + return false; + } + } + + // Link the last filter to the buffersink + if (avfilter_link(previous_filter, 0, buffersink_ctx_, 0) != 0) { + qCritical() << "Failed to link final filter and buffer sink"; + Close(); + return false; + } + + // Config graph + if (avfilter_graph_config(filter_graph_, nullptr) < 0) { + qCritical() << "Failed to configure filter graph"; + Close(); + return false; + } + + timestamp_ = 0; + + open_ = true; + + flushed_ = false; + + return true; +} + +void TempoProcessor::Push(const char *data, int length) +{ + if (flushed_) { + if (length > 0) { + qCritical() << "Tried to push" << length << "bytes after TempoProcessor was closed"; + } + return; + } + + AVFrame* src_frame; + + if (length == 0) { + // No audio data, flush the last out of the filter graph + src_frame = nullptr; + flushed_ = true; + } else { + src_frame = av_frame_alloc(); + + if (!src_frame) { + qCritical() << "Failed to allocate source frame"; + return; + } + + // Allocate a buffer for the number of samples we got + src_frame->sample_rate = params_.sample_rate(); + src_frame->format = FFmpegCommon::GetFFmpegSampleFormat(params_.format()); + src_frame->channel_layout = params_.channel_layout(); + src_frame->nb_samples = params_.bytes_to_samples(length); + src_frame->pts = timestamp_; + timestamp_ += src_frame->nb_samples; + + if (av_frame_get_buffer(src_frame, 0) < 0) { + qCritical() << "Failed to allocate buffer for source frame"; + av_frame_free(&src_frame); + return; + } + + // Copy buffer from data array to frame + memcpy(src_frame->data[0], data, length); + } + + int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, src_frame, AV_BUFFERSRC_FLAG_KEEP_REF); + + if (ret < 0) { + qCritical() << "Failed to feed buffer source" << ret; + } + + if (src_frame) { + av_frame_free(&src_frame); + } +} + +int TempoProcessor::Pull(char *data, int max_length) +{ + if (!processed_frame_) { + processed_frame_ = av_frame_alloc(); + + // Try to pull samples from the buffersink + int ret = av_buffersink_get_frame(buffersink_ctx_, processed_frame_); + + if (ret < 0) { + // We couldn't pull for some reason, if the error was EAGAIN, we just need to send more samples. Otherwise the + // error might be fatal... + if (ret != AVERROR(EAGAIN)) { + qCritical() << "Failed to pull from buffersink" << ret; + } + + av_frame_free(&processed_frame_); + + return 0; + } + + processed_frame_byte_index_ = 0; + processed_frame_max_bytes_ = params_.samples_to_bytes(processed_frame_->nb_samples); + } + + // Determine how many bytes we should copy into the data array + int copy_length = qMin(max_length, processed_frame_max_bytes_ - processed_frame_byte_index_); + + // Copy the bytes + memcpy(data, processed_frame_->data[0] + processed_frame_byte_index_, copy_length); + + // Add the copied amount to the current index + processed_frame_byte_index_ += copy_length; + + // If the index has reached the limit of this processed frame, we can dispose of the frame now + if (processed_frame_byte_index_ == processed_frame_max_bytes_) { + av_frame_free(&processed_frame_); + processed_frame_ = nullptr; + } + + return copy_length; +} + +void TempoProcessor::Close() +{ + open_ = false; + + if (filter_graph_) { + avfilter_graph_free(&filter_graph_); + filter_graph_ = nullptr; + } + + if (processed_frame_) { + av_frame_free(&processed_frame_); + processed_frame_ = nullptr; + } + + buffersrc_ctx_ = nullptr; + buffersink_ctx_ = nullptr; +} + +AVFilterContext *TempoProcessor::CreateTempoFilter(AVFilterGraph* graph, AVFilterContext* link, const double &tempo) +{ + // Set up tempo param, which is taken as a C string + char speed_param[20]; + snprintf(speed_param, 20, "%f", tempo); + + AVFilterContext* tempo_ctx = nullptr; + + if (avfilter_graph_create_filter(&tempo_ctx, avfilter_get_by_name("atempo"), "atempo", speed_param, nullptr, graph) >= 0 + && avfilter_link(link, 0, tempo_ctx, 0) == 0) { + return tempo_ctx; + } + + return nullptr; +} diff --git a/app/audio/tempoprocessor.h b/app/audio/tempoprocessor.h new file mode 100644 index 000000000..c06eeb2b0 --- /dev/null +++ b/app/audio/tempoprocessor.h @@ -0,0 +1,53 @@ +#ifndef TEMPOPROCESSOR_H +#define TEMPOPROCESSOR_H + +#include + +extern "C" { +#include +} + +#include "render/audioparams.h" + +class TempoProcessor +{ +public: + TempoProcessor(); + + bool IsOpen() const; + + const double& GetSpeed() const; + + bool Open(const AudioRenderingParams& params, const double &speed); + + void Push(const char *data, int length); + + int Pull(char* data, int max_length); + + void Close(); + +private: + static AVFilterContext* CreateTempoFilter(AVFilterGraph *graph, AVFilterContext *link, const double& tempo); + + AVFilterGraph* filter_graph_; + + AVFilterContext* buffersrc_ctx_; + + AVFilterContext* buffersink_ctx_; + + AVFrame* processed_frame_; + int processed_frame_byte_index_; + int processed_frame_max_bytes_; + + AudioRenderingParams params_; + + int64_t timestamp_; + + double speed_; + + bool open_; + + bool flushed_; +}; + +#endif // TEMPOPROCESSOR_H diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index 3589be63f..5f7a999a6 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -73,21 +73,8 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti } if (b->is_reversed()) { - int sample_size = audio_params_.samples_to_bytes(1); - int half_buffer_sz = samples_from_this_block.size() / 2; - char* temp_buffer = new char[sample_size]; - - for (int src_index=0;src_index(sample_size)); - memcpy(src_ptr, dst_ptr, static_cast(sample_size)); - memcpy(dst_ptr, temp_buffer, static_cast(sample_size)); - } - - delete [] temp_buffer; + // Reverse the audio buffer + AudioManager::ReverseBuffer(samples_from_this_block.data(), samples_from_this_block.size(), audio_params_.samples_to_bytes(1)); } copied_size = samples_from_this_block.size(); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index cdb86ca25..8955f3447 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -257,16 +257,17 @@ void ViewerWidget::PlayInternal(int speed) return; } + playback_speed_ = speed; + QIODevice* audio_src = audio_renderer_->GetAudioPullDevice(); if (audio_src != nullptr && audio_src->open(QIODevice::ReadOnly)) { audio_src->seek(audio_renderer_->params().time_to_bytes(GetTime())); AudioManager::instance()->SetOutputParams(audio_renderer_->params()); - AudioManager::instance()->StartOutput(audio_src); + AudioManager::instance()->StartOutput(audio_src, playback_speed_); } start_msec_ = QDateTime::currentMSecsSinceEpoch(); start_timestamp_ = ruler_->GetTime(); - playback_speed_ = speed; playback_timer_.start();