From c75a0e84981410e328045be7f83557e8931d387f Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Fri, 6 Mar 2020 16:12:29 +1100 Subject: [PATCH] timeline/renderer: implemented basic waveform display support --- app/audio/CMakeLists.txt | 2 + app/audio/sumsamples.cpp | 64 +++++++++++++++++++ app/audio/sumsamples.h | 30 +++++++++ app/render/backend/audio/audiobackend.cpp | 5 +- app/render/backend/audio/audioworker.cpp | 4 +- app/render/backend/audio/audioworker.h | 2 +- app/render/backend/audiorenderbackend.cpp | 14 ++++ app/render/backend/audiorenderbackend.h | 6 ++ app/render/backend/audiorenderworker.cpp | 49 +++++++++++++- app/render/backend/audiorenderworker.h | 4 +- app/render/backend/renderbackend.h | 7 +- .../timelinewidget/view/timelineviewbase.cpp | 2 - .../view/timelineviewblockitem.cpp | 55 ++++++++++++++++ 13 files changed, 230 insertions(+), 14 deletions(-) create mode 100644 app/audio/sumsamples.cpp create mode 100644 app/audio/sumsamples.h diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index b785c3c78..c23744131 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -26,6 +26,8 @@ set(OLIVE_SOURCES audio/outputmanager.cpp audio/sampleformat.h audio/sampleformat.cpp + audio/sumsamples.h + audio/sumsamples.cpp audio/tempoprocessor.h audio/tempoprocessor.cpp PARENT_SCOPE diff --git a/app/audio/sumsamples.cpp b/app/audio/sumsamples.cpp new file mode 100644 index 000000000..7f90641d8 --- /dev/null +++ b/app/audio/sumsamples.cpp @@ -0,0 +1,64 @@ +#include "sumsamples.h" + +#include + +const int SampleSummer::kSumSampleRate = 200; + +QVector SampleSummer::SumSamples(const float *samples, int nb_samples, int nb_channels) +{ + return SumSamplesInternal(samples, nb_samples, nb_channels); +} + +QVector SampleSummer::SumSamples(const qfloat16 *samples, int nb_samples, int nb_channels) +{ + return SumSamplesInternal(samples, nb_samples, nb_channels); +} + +QVector SampleSummer::ReSumSamples(const SampleSummer::Sum *samples, int nb_samples, int nb_channels) +{ + QVector summed_samples(nb_channels); + + for (int i=0;i summed_samples[channel].max) { + summed_samples[channel].max = sample.max; + } + } + + return summed_samples; +} + +SampleSummer::Sum::Sum() +{ + min = 0; + max = 0; +} + +template +QVector SampleSummer::SumSamplesInternal(const T *samples, int nb_samples, int nb_channels) +{ + QVector summed_samples(nb_channels); + + for (int i=0;i summed_samples[channel].max) { + summed_samples[channel].max = sample; + } + } + + return summed_samples; +} diff --git a/app/audio/sumsamples.h b/app/audio/sumsamples.h new file mode 100644 index 000000000..c74f9b30c --- /dev/null +++ b/app/audio/sumsamples.h @@ -0,0 +1,30 @@ +#ifndef SUMSAMPLES_H +#define SUMSAMPLES_H + +#include +#include + +class SampleSummer { +public: + struct Sum { + Sum(); + + qfloat16 min; + qfloat16 max; + }; + + // FIXME: Move to config + static const int kSumSampleRate; + + static QVector SumSamples(const float* samples, int nb_samples, int nb_channels); + static QVector SumSamples(const qfloat16* samples, int nb_samples, int nb_channels); + + static QVector ReSumSamples(const SampleSummer::Sum* samples, int nb_samples, int nb_channels); + +private: + template + static QVector SumSamplesInternal(const T* samples, int nb_samples, int nb_channels); + +}; + +#endif // SUMSAMPLES_H diff --git a/app/render/backend/audio/audiobackend.cpp b/app/render/backend/audio/audiobackend.cpp index 59d70245a..b6ccd42fa 100644 --- a/app/render/backend/audio/audiobackend.cpp +++ b/app/render/backend/audio/audiobackend.cpp @@ -24,7 +24,7 @@ bool AudioBackend::InitInternal() // Initiate one thread per CPU core for (int i=0;iSetParameters(params()); processors_.append(processor); } @@ -39,12 +39,13 @@ void AudioBackend::CloseInternal() bool AudioBackend::CompileInternal() { // This backend doesn't compile anything yet - return true; + return AudioRenderBackend::CompileInternal(); } void AudioBackend::DecompileInternal() { // This backend doesn't compile anything yet + AudioRenderBackend::DecompileInternal(); } void AudioBackend::ConnectWorkerToThis(RenderWorker *worker) diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index 9740b74f4..5b6de78d7 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -1,7 +1,7 @@ #include "audioworker.h" -AudioWorker::AudioWorker(DecoderCache* decoder_cache, QObject *parent) : - AudioRenderWorker(decoder_cache, parent) +AudioWorker::AudioWorker(DecoderCache* decoder_cache, QHash *copy_map, QObject *parent) : + AudioRenderWorker(decoder_cache, copy_map, parent) { } diff --git a/app/render/backend/audio/audioworker.h b/app/render/backend/audio/audioworker.h index 00c063b76..1db52a23f 100644 --- a/app/render/backend/audio/audioworker.h +++ b/app/render/backend/audio/audioworker.h @@ -6,7 +6,7 @@ class AudioWorker : public AudioRenderWorker { public: - AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); + AudioWorker(DecoderCache* decoder_cache, QHash* copy_map, QObject* parent = nullptr); protected: virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override; diff --git a/app/render/backend/audiorenderbackend.cpp b/app/render/backend/audiorenderbackend.cpp index 79f7e0d8f..5baf68c17 100644 --- a/app/render/backend/audiorenderbackend.cpp +++ b/app/render/backend/audiorenderbackend.cpp @@ -41,6 +41,20 @@ void AudioRenderBackend::DisconnectViewer(ViewerOutput *node) disconnect(node, &ViewerOutput::AudioGraphChanged, this, &AudioRenderBackend::QueueRecompile); } +bool AudioRenderBackend::CompileInternal() +{ + for (int i=0;i copy_map_; + private: struct ConformWaitInfo { StreamPtr stream; diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index 5e8a37954..ea28bec0c 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -1,9 +1,15 @@ #include "audiorenderworker.h" -#include "audio/audiomanager.h" +#include +#include -AudioRenderWorker::AudioRenderWorker(DecoderCache* decoder_cache, QObject *parent) : - RenderWorker(decoder_cache, parent) +#include "audio/audiomanager.h" +#include "audio/sumsamples.h" +#include "config/config.h" + +AudioRenderWorker::AudioRenderWorker(DecoderCache* decoder_cache, QHash *copy_map, QObject *parent) : + RenderWorker(decoder_cache, parent), + copy_map_(copy_map) { } @@ -82,6 +88,43 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti memcpy(block_range_buffer.data()+destination_offset, samples_from_this_block.data(), actual_copy_size); + + // Save waveform to file + QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString()); + QDir waveform_loc = local_appdata_dir.filePath("waveform"); + waveform_loc.mkpath("."); + QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast(copy_map_->value(b))))); + wave_file.open(QFile::ReadWrite); + + // FIXME: Assumes 32-bit float + float* flt_samples = reinterpret_cast(samples_from_this_block.data()); + int nb_sample = samples_from_this_block.size() / sizeof(float); + + // FIXME: Hardcoded sample rate + // We use S16 here as a size-compatible substitute for qfloat16/half + AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S16); + int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()) * waveform_params.channel_count(); + + qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()) * 2; + qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()) * 2; + qint64 end_offset = start_offset + length_offset; + + if (wave_file.size() < end_offset) { + wave_file.resize(end_offset); + } + + wave_file.seek(start_offset); + + for (int i=0;i summary = SampleSummer::SumSamples(&flt_samples[i], + qMin(chunk_size, nb_sample - i), + audio_params().channel_count()); + + wave_file.write(reinterpret_cast(summary.constData()), + summary.size() * sizeof(SampleSummer::Sum)); + } + + wave_file.close(); } } diff --git a/app/render/backend/audiorenderworker.h b/app/render/backend/audiorenderworker.h index 8bde8ace4..b53ccd04e 100644 --- a/app/render/backend/audiorenderworker.h +++ b/app/render/backend/audiorenderworker.h @@ -7,7 +7,7 @@ class AudioRenderWorker : public RenderWorker { Q_OBJECT public: - AudioRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); + AudioRenderWorker(DecoderCache* decoder_cache, QHash* copy_map, QObject* parent = nullptr); void SetParameters(const AudioRenderingParams& audio_params); @@ -26,6 +26,8 @@ protected: private: AudioRenderingParams audio_params_; + QHash* copy_map_; + }; #endif // AUDIORENDERWORKER_H diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index e48edbe81..519f0de01 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -108,6 +108,10 @@ protected: QHash render_job_info_; + QList source_node_list_; + + NodeGraph copied_graph_; + protected slots: void QueueRecompile(); @@ -139,9 +143,6 @@ private: QString cache_id_; - QList source_node_list_; - NodeGraph copied_graph_; - bool recompile_queued_; bool input_update_queued_; diff --git a/app/widget/timelinewidget/view/timelineviewbase.cpp b/app/widget/timelinewidget/view/timelineviewbase.cpp index 31101557d..dae4264d2 100644 --- a/app/widget/timelinewidget/view/timelineviewbase.cpp +++ b/app/widget/timelinewidget/view/timelineviewbase.cpp @@ -31,8 +31,6 @@ TimelineViewBase::TimelineViewBase(QWidget *parent) : SetDefaultDragMode(NoDrag); - setViewportUpdateMode(FullViewportUpdate); - connect(&scene_, SIGNAL(changed(const QList&)), this, SLOT(UpdateSceneRect())); connect(Core::instance(), &Core::ToolChanged, this, &TimelineViewBase::ApplicationToolChanged); diff --git a/app/widget/timelinewidget/view/timelineviewblockitem.cpp b/app/widget/timelinewidget/view/timelineviewblockitem.cpp index f8023af4a..294dd3af6 100644 --- a/app/widget/timelinewidget/view/timelineviewblockitem.cpp +++ b/app/widget/timelinewidget/view/timelineviewblockitem.cpp @@ -22,12 +22,16 @@ #include #include +#include +#include #include #include #include #include +#include "audio/sumsamples.h" #include "common/qtutils.h" +#include "config/config.h" #include "node/block/transition/transition.h" TimelineViewBlockItem::TimelineViewBlockItem(QGraphicsItem* parent) : @@ -97,6 +101,57 @@ void TimelineViewBlockItem::paint(QPainter *painter, const QStyleOptionGraphicsI painter->fillRect(rect(), QColor(0, 0, 0, 64)); } + // Draw waveform if one is available + QFile wave_file(QDir(QDir(Config::Current()["DiskCachePath"].toString()).filePath("waveform")).filePath(QString::number(reinterpret_cast(block_)))); + if (wave_file.open(QFile::ReadOnly)){ + painter->setPen(QColor(64, 64, 64)); + + QByteArray w = wave_file.readAll(); + + // FIXME: Hardcoded channel count + int channels = 2; + + const SampleSummer::Sum* samples = reinterpret_cast(w.constData()); + int nb_samples = w.size() / sizeof(SampleSummer::Sum); + + int sample_index, next_sample_index = 0; + + QVector summary; + int summary_index = -1; + + int channel_height = rect().height() / channels; + int channel_half_height = channel_height / 2; + + for (int i=0;i(SampleSummer::kSumSampleRate) * static_cast(i+1) / this->GetScale()) * channels); + + if (summary_index != sample_index) { + summary = SampleSummer::ReSumSamples(&samples[sample_index], + qMax(channels, next_sample_index - sample_index), + channels); + summary_index = sample_index; + } + + for (int j=0;jdrawLine(i, + channel_mid + qRound(summary.at(j).min * channel_half_height), + i, + channel_mid + qRound(summary.at(j).max * channel_half_height)); + } + } + + wave_file.close(); + } + painter->setPen(Qt::white); painter->drawLine(rect().topLeft(), QPointF(rect().right(), rect().top())); painter->drawLine(rect().topLeft(), QPointF(rect().left(), rect().bottom() - 1));