From 7107afaf99a0f2faf81008a5803a5e5be49e3a8a Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Tue, 13 Apr 2021 23:41:17 +1000 Subject: [PATCH] audiovisualwaveform: mipmap waveforms --- app/audio/audiovisualwaveform.cpp | 338 +++++++++++++----------- app/audio/audiovisualwaveform.h | 56 ++-- app/widget/viewer/audiowaveformview.cpp | 4 +- 3 files changed, 209 insertions(+), 189 deletions(-) diff --git a/app/audio/audiovisualwaveform.cpp b/app/audio/audiovisualwaveform.cpp index 4063d413a..b21fcc361 100644 --- a/app/audio/audiovisualwaveform.cpp +++ b/app/audio/audiovisualwaveform.cpp @@ -23,14 +23,69 @@ #include #include "config/config.h" +#include "common/functiontimer.h" namespace olive { -const int AudioVisualWaveform::kSumSampleRate = 200; - -void AudioVisualWaveform::AddSum(const float *samples, int nb_samples, int nb_channels) +AudioVisualWaveform::AudioVisualWaveform() : + channels_(0) { - data_.append(SumSamples(samples, nb_samples, nb_channels)); + // Must be a power of 2 + static const rational kMinimumSampleRate = rational(1, 8); + static const rational kMaximumSampleRate = 8192; + + for (rational i=kMinimumSampleRate; i<=kMaximumSampleRate; i*=2) { + mipmapped_data_.insert({i, Sample()}); + } +} + +void AudioVisualWaveform::OverwriteSamplesFromBuffer(SampleBufferPtr samples, int sample_rate, const rational &start, double target_rate, Sample& data, int &start_index, int &samples_length) +{ + start_index = time_to_samples(start, target_rate); + samples_length = time_to_samples(static_cast(samples->sample_count()) / static_cast(sample_rate), target_rate); + + int end_index = start_index + samples_length; + if (data.size() < end_index) { + data.resize(end_index); + } + + int chunk_size = sample_rate / target_rate; + + for (int i=0; isample_count() - src_index)); + + memcpy(&data.data()[i + start_index], + summary.constData(), + summary.size() * sizeof(SamplePerChannel)); + } +} + +void AudioVisualWaveform::OverwriteSamplesFromMipmap(const AudioVisualWaveform::Sample &input, double input_sample_rate, int &input_start, int &input_length, const rational &start, double output_rate, AudioVisualWaveform::Sample &output_data) +{ + int start_index = time_to_samples(start, output_rate); + int samples_length = time_to_samples(static_cast(input_length / channels_) / input_sample_rate, output_rate); + + int end_index = start_index + samples_length; + if (output_data.size() < end_index) { + output_data.resize(end_index); + } + + int chunk_size = input_sample_rate / output_rate; + + for (int i=0; i(samples->sample_count()) / static_cast(sample_rate)); + // Old less optimized code. Keeping this around as a reference, but the below code is at least + // 10x faster so this shouldn't be used in production. + // + // int input_start, input_length; + // for (auto it=mipmapped_data_.begin(); it!=mipmapped_data_.end(); it++) { + // OverwriteSamplesFromBuffer(samples, sample_rate, start, it->first.toDouble(), it->second, input_start, input_length); + // } - int end_index = start_index + samples_length; - if (data_.size() < end_index) { - data_.resize(end_index); - } + // Process the largest mipmap directly for the samples + auto current_mipmap = mipmapped_data_.rbegin(); + int input_start, input_length; + OverwriteSamplesFromBuffer(samples, sample_rate, start, current_mipmap->first.toDouble(), current_mipmap->second, input_start, input_length); - int chunk_size = sample_rate / kSumSampleRate; + while (true) { + // For each smaller mipmap, we just process from the mipmap before it, making each one + // exponentially faster to create + auto previous_mipmap = current_mipmap; + current_mipmap++; + if (current_mipmap == mipmapped_data_.rend()) { + break; + } - for (int i=0; i summary = SumSamples(samples, - src_index, - qMin(chunk_size, samples->sample_count() - src_index)); - - memcpy(&data_.data()[i + start_index], - summary.constData(), - summary.size() * sizeof(SamplePerChannel)); + OverwriteSamplesFromMipmap(previous_mipmap->second, previous_mipmap->first.toDouble(), + input_start, input_length, start, current_mipmap->first.toDouble(), + current_mipmap->second); } } void AudioVisualWaveform::OverwriteSums(const AudioVisualWaveform &sums, const rational &dest, const rational& offset, const rational& length) { - if (sums.data_.isEmpty()) { - return; + for (auto it=mipmapped_data_.begin(); it!=mipmapped_data_.end(); it++) { + rational rate = it->first; + + Sample& our_arr = it->second; + const Sample& their_arr = sums.mipmapped_data_.at(rate); + + double rate_dbl = rate.toDouble(); + + int start_index = time_to_samples(dest, rate_dbl); + int sample_start = time_to_samples(offset, rate_dbl); + + int copy_len = their_arr.size() - sample_start; + if (!length.isNull()) { + copy_len = qMin(copy_len, time_to_samples(length, rate_dbl)); + } + + int end_index = start_index + copy_len; + + if (our_arr.size() < end_index) { + our_arr.resize(end_index); + } + + memcpy(reinterpret_cast(our_arr.data()) + start_index * sizeof(SamplePerChannel), + reinterpret_cast(their_arr.constData()) + time_to_samples(offset, rate_dbl) * sizeof(SamplePerChannel), + copy_len * sizeof(SamplePerChannel)); } - - int start_index = time_to_samples(dest); - int sample_start = time_to_samples(offset); - - int copy_len = sums.data_.size() - sample_start; - if (!length.isNull()) { - copy_len = qMin(copy_len, time_to_samples(length)); - } - - int end_index = start_index + copy_len; - - if (data_.size() < end_index) { - data_.resize(end_index); - } - - memcpy(reinterpret_cast(data_.data()) + start_index * sizeof(SamplePerChannel), - reinterpret_cast(sums.data_.constData()) + time_to_samples(offset) * sizeof(SamplePerChannel), - copy_len * sizeof(SamplePerChannel)); -} - -AudioVisualWaveform AudioVisualWaveform::Mid(const rational &time) const -{ - int sample_index = time_to_samples(time); - - // Create a copy of this waveform chop the early section off - AudioVisualWaveform copy = *this; - copy.data_ = data_.mid(sample_index); - - return copy; -} - -void AudioVisualWaveform::Append(const AudioVisualWaveform &waveform) -{ - data_.append(waveform.data_); -} - -void AudioVisualWaveform::TrimIn(const rational &time) -{ - data_ = data_.mid(time_to_samples(time)); -} - -void AudioVisualWaveform::TrimOut(const rational &time) -{ - data_.resize(data_.size() - time_to_samples(time)); -} - -void AudioVisualWaveform::PrependSilence(const rational &time) -{ - int added_samples = time_to_samples(time); - - // Resize buffer for extra space - data_.resize(data_.size() + added_samples); - - // Shift all data forward - for (int i=data_.size()-1; i>=added_samples; i--) { - data_[i] = data_[i - added_samples]; - } - - // Fill remainder with silence - memset(reinterpret_cast(data_.data()), 0, added_samples * sizeof(SamplePerChannel)); -} - -void AudioVisualWaveform::AppendSilence(const rational &time) -{ - int added_samples = time_to_samples(time); - - // Resize buffer for extra space - int old_size = data_.size(); - data_.resize(old_size + added_samples); - - // Fill remainder with silence - memset(reinterpret_cast(&data_[old_size]), 0, (data_.size() - old_size) * sizeof(SamplePerChannel)); } void AudioVisualWaveform::Shift(const rational &from, const rational &to) { - int from_index = time_to_samples(from); - int to_index = time_to_samples(to); + for (auto it=mipmapped_data_.begin(); it!=mipmapped_data_.end(); it++) { + rational rate = it->first; + double rate_dbl = rate.toDouble(); + Sample& data = it->second; - if (from_index == to_index) { - return; - } + int from_index = time_to_samples(from, rate_dbl); + int to_index = time_to_samples(to, rate_dbl); - if (from_index > data_.size()) { - return; - } - - if (from_index > to_index) { - // Shifting backwards <- - int copy_sz = data_.size() - from_index; - - for (int i=0; i - int old_sz = data_.size(); - - int distance = (to_index - from_index); - - data_.resize(data_.size() + distance); - - int copy_sz = old_sz - from_index; - - for (int i=0; i data.size()) { + return; } - memset(reinterpret_cast(&data_[from_index]), 0, distance * sizeof(SamplePerChannel)); + if (from_index > to_index) { + // Shifting backwards <- + int copy_sz = data.size() - from_index; + + for (int i=0; i + int old_sz = data.size(); + + int distance = (to_index - from_index); + + data.resize(data.size() + distance); + + int copy_sz = old_sz - from_index; + + for (int i=0; i(&data[from_index]), 0, distance * sizeof(SamplePerChannel)); + } } } -QVector AudioVisualWaveform::SumSamples(const float *samples, int nb_samples, int nb_channels) +AudioVisualWaveform::Sample AudioVisualWaveform::SumSamples(const float *samples, int nb_samples, int nb_channels) { - return SumSamplesInternal(samples, nb_samples, nb_channels); + return SumSamplesInternal(samples, nb_samples, nb_channels); } -QVector AudioVisualWaveform::SumSamples(const qfloat16 *samples, int nb_samples, int nb_channels) +AudioVisualWaveform::Sample AudioVisualWaveform::SumSamples(SampleBufferPtr samples, int start_index, int length) { - return SumSamplesInternal(samples, nb_samples, nb_channels); -} - -QVector AudioVisualWaveform::SumSamples(SampleBufferPtr samples, int start_index, int length) -{ - QVector summed_samples(samples->audio_params().channel_count()); + AudioVisualWaveform::Sample summed_samples(samples->audio_params().channel_count()); int end_index = start_index + length; @@ -207,11 +219,11 @@ QVector AudioVisualWaveform::SumSamples(S return summed_samples; } -QVector AudioVisualWaveform::ReSumSamples(const SamplePerChannel* samples, +AudioVisualWaveform::Sample AudioVisualWaveform::ReSumSamples(const SamplePerChannel* samples, int nb_samples, int nb_channels) { - QVector summed_samples(nb_channels); + AudioVisualWaveform::Sample summed_samples(nb_channels); for (int i=0;i AudioVisualWaveform::ReSumSamples return summed_samples; } -void AudioVisualWaveform::DrawSample(QPainter *painter, const QVector& sample, int x, int y, int height) +void AudioVisualWaveform::DrawSample(QPainter *painter, const Sample& sample, int x, int y, int height, bool rectified) { int channel_height = height / sample.size(); int channel_half_height = channel_height / 2; for (int i=0;i(1.0f)); - qfloat16 min = qMax(sample.at(i).min, static_cast(-1.0)); + float max = qMin(sample.at(i).max, 1.0f); + float min = qMax(sample.at(i).min, -1.0f); - if (Config::Current()[QStringLiteral("RectifiedWaveforms")].toBool()) { + if (rectified) { int channel_bottom = y + channel_height * (i + 1); int diff = qRound((max - min) * channel_half_height); @@ -261,16 +273,34 @@ void AudioVisualWaveform::DrawSample(QPainter *painter, const QVector= samples.nb_samples()) { + // Find largest mipmap for this scale (or the largest if we don't find one sufficient) + auto using_mipmap = samples.mipmapped_data_.cend(); + using_mipmap--; + for (auto it=samples.mipmapped_data_.cbegin(); it!=samples.mipmapped_data_.cend(); it++) { + if (it->first.toDouble() >= scale) { + using_mipmap = it; + break; + } + } + + rational rate = using_mipmap->first; + double rate_dbl = rate.toDouble(); + const Sample& arr = using_mipmap->second; + + int start_sample_index = samples.time_to_samples(start_time, rate_dbl); + + if (start_sample_index >= arr.size()) { return; } int next_sample_index = start_sample_index; int sample_index; - QVector summary; + Sample summary; int summary_index = -1; const QRect& viewport = painter->viewport(); @@ -279,41 +309,43 @@ void AudioVisualWaveform::DrawWaveform(QPainter *painter, const QRect& rect, con int start = qMax(rect.x(), -top_left.x()); int end = qMin(rect.right(), -top_left.x() + viewport.width()); + bool rectified = Config::Current()[QStringLiteral("RectifiedWaveforms")].toBool(); + for (int i=start;i(kSumSampleRate) * static_cast(i - rect.x() + 1) / scale) * samples.channel_count()); + next_sample_index = qMin(arr.size(), + start_sample_index + qFloor(rate_dbl * static_cast(i - rect.x() + 1) / scale) * samples.channel_count()); if (summary_index != sample_index) { - summary = AudioVisualWaveform::ReSumSamples(&samples.data_.at(sample_index), + summary = AudioVisualWaveform::ReSumSamples(&arr.at(sample_index), qMax(samples.channel_count(), next_sample_index - sample_index), samples.channel_count()); summary_index = sample_index; } - DrawSample(painter, summary, i, rect.y(), rect.height()); + DrawSample(painter, summary, i, rect.y(), rect.height(), rectified); } } -int AudioVisualWaveform::time_to_samples(const rational &time) const +int AudioVisualWaveform::time_to_samples(const rational &time, double sample_rate) const { - return time_to_samples(time.toDouble()); + return time_to_samples(time.toDouble(), sample_rate); } -int AudioVisualWaveform::time_to_samples(const double &time) const +int AudioVisualWaveform::time_to_samples(const double &time, double sample_rate) const { - return qFloor(time * kSumSampleRate) * channels_; + return qFloor(time * sample_rate) * channels_; } template -QVector AudioVisualWaveform::SumSamplesInternal(const T *samples, int nb_samples, int nb_channels) +AudioVisualWaveform::Sample AudioVisualWaveform::SumSamplesInternal(const T *samples, int nb_samples, int nb_channels) { - QVector summed_samples(nb_channels); + AudioVisualWaveform::Sample summed_samples(nb_channels); for (int i=0;i(summed_samples[i%nb_channels], samples[i]); diff --git a/app/audio/audiovisualwaveform.h b/app/audio/audiovisualwaveform.h index 13e68b664..447e4fd84 100644 --- a/app/audio/audiovisualwaveform.h +++ b/app/audio/audiovisualwaveform.h @@ -21,7 +21,6 @@ #ifndef SUMSAMPLES_H #define SUMSAMPLES_H -#include #include #include @@ -37,11 +36,11 @@ namespace olive { */ class AudioVisualWaveform { public: - AudioVisualWaveform() = default; + AudioVisualWaveform(); struct SamplePerChannel { - qfloat16 min; - qfloat16 max; + float min; + float max; }; using Sample = QVector; @@ -56,18 +55,11 @@ public: channels_ = channels; } - int nb_samples() const - { - return data_.size(); - } - - const SamplePerChannel* const_data() const - { - return data_.constData(); - } - - void AddSum(const float* samples, int nb_samples, int nb_channels); - + /** + * @brief Writes samples into the visual waveform buffer + * + * Starting at `start`, writes samples over anything in the buffer, expanding it if necessary. + */ void OverwriteSamples(SampleBufferPtr samples, int sample_rate, const rational& start = rational()); /** @@ -91,40 +83,34 @@ public: */ void OverwriteSums(const AudioVisualWaveform& sums, const rational& dest, const rational& offset = rational(), const rational &length = rational()); - AudioVisualWaveform Mid(const rational& time) const; - void Append(const AudioVisualWaveform& waveform); - void TrimIn(const rational& time); - void TrimOut(const rational& time); - void PrependSilence(const rational& time); - void AppendSilence(const rational& time); void Shift(const rational& from, const rational& to); - // FIXME: Move to dynamic - static const int kSumSampleRate; + static Sample SumSamples(const float* samples, int nb_samples, int nb_channels); + static Sample SumSamples(SampleBufferPtr samples, int start_index, int length); - static QVector SumSamples(const float* samples, int nb_samples, int nb_channels); - static QVector SumSamples(const qfloat16* samples, int nb_samples, int nb_channels); - static QVector SumSamples(SampleBufferPtr samples, int start_index, int length); + static Sample ReSumSamples(const SamplePerChannel *samples, int nb_samples, int nb_channels); - static QVector ReSumSamples(const SamplePerChannel *samples, int nb_samples, int nb_channels); - - static void DrawSample(QPainter* painter, const QVector &sample, int x, int y, int height); + static void DrawSample(QPainter* painter, const Sample &sample, int x, int y, int height, bool rectified); static void DrawWaveform(QPainter* painter, const QRect &rect, const double &scale, const AudioVisualWaveform& samples, const rational &start_time); private: template - static QVector SumSamplesInternal(const T* samples, int nb_samples, int nb_channels); + static Sample SumSamplesInternal(const T* samples, int nb_samples, int nb_channels); template static void ExpandMinMax(SamplePerChannel &sum, T value); - int time_to_samples(const rational& time) const; - int time_to_samples(const double& time) const; + void OverwriteSamplesFromBuffer(SampleBufferPtr samples, int sample_rate, const rational& start, double target_rate, Sample &data, int &start_index, int &samples_length); - int channels_ = 0; + void OverwriteSamplesFromMipmap(const Sample& input, double input_sample_rate, int &input_start, int &input_length, const rational& start, double output_rate, Sample &output_data); - QVector data_; + int time_to_samples(const rational& time, double sample_rate) const; + int time_to_samples(const double& time, double sample_rate) const; + + int channels_; + + std::map mipmapped_data_; }; diff --git a/app/widget/viewer/audiowaveformview.cpp b/app/widget/viewer/audiowaveformview.cpp index 462d6ddf8..3dbea3af2 100644 --- a/app/widget/viewer/audiowaveformview.cpp +++ b/app/widget/viewer/audiowaveformview.cpp @@ -127,6 +127,8 @@ QPixmap AudioWaveformView::DrawWaveform(QIODevice* fs, CachedWaveformInfo info, QPixmap pixmap(slice_end - slice_start, info.size.height()); pixmap.fill(Qt::transparent); + bool rectified = Config::Current()[QStringLiteral("RectifiedWaveforms")].toBool(); + if (fs->open(QFile::ReadOnly)) { QPainter wave_painter(&pixmap); @@ -154,7 +156,7 @@ QPixmap AudioWaveformView::DrawWaveform(QIODevice* fs, CachedWaveformInfo info, info.params.channel_count()); for (int i=0;i