From 6f0d72c05da9656479c8b6b65522125b9d5eb228 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Tue, 31 Mar 2020 18:01:20 +1100 Subject: [PATCH] renderer: switched audio rendering over to planar buffers While planar audio cannot be used for playback, it's much easier for processing. This should lead to cleaner and easier to write code in the future. --- app/audio/sumsamples.cpp | 39 +++- app/audio/sumsamples.h | 6 + app/codec/CMakeLists.txt | 2 + app/codec/decoder.cpp | 2 +- app/codec/decoder.h | 3 +- app/codec/ffmpeg/ffmpegdecoder.cpp | 28 +-- app/codec/ffmpeg/ffmpegdecoder.h | 2 +- app/codec/ffmpeg/ffmpegframecache.cpp | 4 +- app/codec/frame.cpp | 61 ++--- app/codec/frame.h | 41 +--- app/codec/oiio/oiiodecoder.cpp | 6 +- app/codec/samplebuffer.cpp | 211 ++++++++++++++++++ app/codec/samplebuffer.h | 67 ++++++ app/core.cpp | 1 + .../tabs/preferencesqualitytab.cpp | 1 + app/node/audio/pan/pan.cpp | 20 +- app/node/audio/pan/pan.h | 2 +- app/node/audio/volume/volume.cpp | 6 +- app/node/audio/volume/volume.h | 2 +- app/node/node.cpp | 7 +- app/node/node.h | 3 +- app/node/param.h | 2 +- app/render/backend/audio/audiobackend.cpp | 2 +- app/render/backend/audio/audioworker.cpp | 17 +- app/render/backend/audiorenderworker.cpp | 103 ++++----- app/render/backend/videorenderworker.cpp | 4 +- app/render/pixelformat.cpp | 4 +- app/render/videoparams.cpp | 6 + app/render/videoparams.h | 1 + app/widget/viewer/viewerglwidget.cpp | 4 +- 30 files changed, 436 insertions(+), 221 deletions(-) create mode 100644 app/codec/samplebuffer.cpp create mode 100644 app/codec/samplebuffer.h diff --git a/app/audio/sumsamples.cpp b/app/audio/sumsamples.cpp index 7f90641d8..1ed6cb498 100644 --- a/app/audio/sumsamples.cpp +++ b/app/audio/sumsamples.cpp @@ -14,6 +14,21 @@ QVector SampleSummer::SumSamples(const qfloat16 *samples, int return SumSamplesInternal(samples, nb_samples, nb_channels); } +QVector SampleSummer::SumSamples(SampleBufferPtr samples, int start_index, int length) +{ + QVector summed_samples(samples->audio_params().channel_count()); + + int end_index = start_index + length; + + for (int i=start_index;iaudio_params().channel_count();channel++) { + ClampMinMax(summed_samples[channel], samples->data()[channel][i]); + } + } + + return summed_samples; +} + QVector SampleSummer::ReSumSamples(const SampleSummer::Sum *samples, int nb_samples, int nb_channels) { QVector summed_samples(nb_channels); @@ -47,18 +62,20 @@ QVector SampleSummer::SumSamplesInternal(const T *samples, in QVector summed_samples(nb_channels); for (int i=0;i summed_samples[channel].max) { - summed_samples[channel].max = sample; - } + ClampMinMax(summed_samples[i%nb_channels], samples[i]); } return summed_samples; } + +template +void SampleSummer::ClampMinMax(SampleSummer::Sum &sum, T value) +{ + if (value < sum.min) { + sum.min = value; + } + + if (value > sum.max) { + sum.max = value; + } +} diff --git a/app/audio/sumsamples.h b/app/audio/sumsamples.h index c74f9b30c..38bb29546 100644 --- a/app/audio/sumsamples.h +++ b/app/audio/sumsamples.h @@ -4,6 +4,8 @@ #include #include +#include "codec/samplebuffer.h" + class SampleSummer { public: struct Sum { @@ -18,6 +20,7 @@ public: static QVector SumSamples(const float* samples, int nb_samples, int nb_channels); static QVector SumSamples(const qfloat16* samples, int nb_samples, int nb_channels); + static QVector SumSamples(SampleBufferPtr samples, int start_index, int length); static QVector ReSumSamples(const SampleSummer::Sum* samples, int nb_samples, int nb_channels); @@ -25,6 +28,9 @@ private: template static QVector SumSamplesInternal(const T* samples, int nb_samples, int nb_channels); + template + static void ClampMinMax(Sum &sum, T value); + }; #endif // SUMSAMPLES_H diff --git a/app/codec/CMakeLists.txt b/app/codec/CMakeLists.txt index 6a1158a72..791500cdd 100644 --- a/app/codec/CMakeLists.txt +++ b/app/codec/CMakeLists.txt @@ -25,6 +25,8 @@ set(OLIVE_SOURCES codec/encoder.cpp codec/frame.h codec/frame.cpp + codec/samplebuffer.h + codec/samplebuffer.cpp codec/waveinput.h codec/waveinput.cpp codec/waveoutput.h diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index 224818912..1ebdf9d35 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -66,7 +66,7 @@ FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const int &/*divid return nullptr; } -FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/) +SampleBufferPtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/) { return nullptr; } diff --git a/app/codec/decoder.h b/app/codec/decoder.h index af3507e33..1517ebaa3 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -26,6 +26,7 @@ #include #include "codec/frame.h" +#include "codec/samplebuffer.h" #include "codec/waveoutput.h" #include "common/constructors.h" #include "common/rational.h" @@ -168,7 +169,7 @@ public: * A FramePtr of valid data at this timecode of the requested length or nullptr if there was nothing to retrieve at * the provided timecode or the media could not be opened. */ - virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params); + virtual SampleBufferPtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params); virtual bool SupportsVideo(); virtual bool SupportsAudio(); diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index dc5795f36..ab9c290ae 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -400,9 +400,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid // We found the frame, we'll return a copy if (return_frame) { FramePtr copy = Frame::Create(); - copy->set_width(return_frame->width()); - copy->set_height(return_frame->height()); - copy->set_format(return_frame->format()); + copy->set_video_params(return_frame->video_params()); copy->set_timestamp(return_frame->timestamp()); copy->set_sample_aspect_ratio(return_frame->sample_aspect_ratio()); copy->allocate(); @@ -415,7 +413,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid return nullptr; } -FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams ¶ms) +SampleBufferPtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams ¶ms) { QMutexLocker locker(&mutex_); @@ -428,29 +426,19 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational & return nullptr; } - //Conform(params, cancelled); - WaveInput input(GetConformedFilename(params)); if (input.open()) { const AudioRenderingParams& input_params = input.params(); - FramePtr audio_frame = Frame::Create(); - audio_frame->set_audio_params(input_params); - audio_frame->set_sample_count(input_params.time_to_samples(length)); - audio_frame->allocate(); - - qint64 actual_read = input.read(input_params.time_to_bytes(timecode), - audio_frame->data(), - audio_frame->allocated_size()); - - if (actual_read < audio_frame->allocated_size()) { - memset(audio_frame->data() + actual_read, 0, audio_frame->allocated_size() - actual_read); - } - + // Read bytes from wav + QByteArray packed_data = input.read(input_params.time_to_bytes(timecode), input_params.time_to_bytes(length)); input.close(); - return audio_frame; + // Create sample buffer + SampleBufferPtr sample_buffer = SampleBuffer::CreateFromPackedData(input_params, packed_data); + + return sample_buffer; } return nullptr; diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 4ceb50184..8a95d7544 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -55,7 +55,7 @@ public: virtual bool Open() override; virtual RetrieveState GetRetrieveState(const rational &time) override; virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override; - virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override; + virtual SampleBufferPtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override; virtual void Close() override; virtual QString id() override; diff --git a/app/codec/ffmpeg/ffmpegframecache.cpp b/app/codec/ffmpeg/ffmpegframecache.cpp index 97e43e4f1..e56d61fad 100644 --- a/app/codec/ffmpeg/ffmpegframecache.cpp +++ b/app/codec/ffmpeg/ffmpegframecache.cpp @@ -85,9 +85,7 @@ Frame* FFmpegFrameCache::Get(const VideoRenderingParams ¶ms) // Otherwise we'll need to create one Frame* f = new Frame(); - f->set_width(params.width()); - f->set_height(params.height()); - f->set_format(params.format()); + f->set_video_params(params); f->allocate(); return f; diff --git a/app/codec/frame.cpp b/app/codec/frame.cpp index 07fc08012..34bb4d742 100644 --- a/app/codec/frame.cpp +++ b/app/codec/frame.cpp @@ -24,10 +24,6 @@ #include Frame::Frame() : - width_(0), - height_(0), - format_(PixelFormat::PIX_FMT_INVALID), - sample_count_(0), timestamp_(0), sample_aspect_ratio_(1) { @@ -38,24 +34,29 @@ FramePtr Frame::Create() return std::make_shared(); } -const int &Frame::width() const +const VideoRenderingParams &Frame::video_params() const { - return width_; + return params_; } -void Frame::set_width(const int &width) +void Frame::set_video_params(const VideoRenderingParams ¶ms) { - width_ = width; + params_ = params; +} + +const int &Frame::width() const +{ + return params_.width(); } const int &Frame::height() const { - return height_; + return params_.height(); } -void Frame::set_height(const int &height) +const PixelFormat::Format &Frame::format() const { - height_ = height; + return params_.format(); } const rational &Frame::sample_aspect_ratio() const @@ -68,16 +69,6 @@ void Frame::set_sample_aspect_ratio(const rational &aspect_ratio) sample_aspect_ratio_ = aspect_ratio; } -const AudioRenderingParams &Frame::audio_params() const -{ - return audio_params_; -} - -void Frame::set_audio_params(const AudioRenderingParams ¶ms) -{ - audio_params_ = params; -} - const rational &Frame::timestamp() const { return timestamp_; @@ -98,31 +89,11 @@ void Frame::set_native_timestamp(const int64_t ×tamp) native_timestamp_ = timestamp; } -const PixelFormat::Format &Frame::format() const -{ - return format_; -} - -void Frame::set_format(const PixelFormat::Format &format) -{ - format_ = format; -} - QByteArray Frame::ToByteArray() const { return data_; } -const int &Frame::sample_count() const -{ - return sample_count_; -} - -void Frame::set_sample_count(const int &audio_sample_count) -{ - sample_count_ = audio_sample_count; -} - char *Frame::data() { return data_.data(); @@ -136,11 +107,11 @@ const char *Frame::const_data() const void Frame::allocate() { // Assume this frame is intended to be a video frame - if (width_ > 0 && height_ > 0) { - data_.resize(PixelFormat::GetBufferSize(static_cast(format_), width_, height_)); - } else if (sample_count_ > 0) { - data_.resize(audio_params_.samples_to_bytes(sample_count_)); + if (!params_.is_valid()) { + qWarning() << "Tried to allocate a frame with invalid parameters"; } + + data_.resize(PixelFormat::GetBufferSize(params_.format(), params_.width(), params_.height())); } bool Frame::is_allocated() const diff --git a/app/codec/frame.h b/app/codec/frame.h index 1311a80dd..fe95ab05f 100644 --- a/app/codec/frame.h +++ b/app/codec/frame.h @@ -25,8 +25,8 @@ #include #include "common/rational.h" -#include "render/audioparams.h" #include "render/pixelformat.h" +#include "render/videoparams.h" class Frame; using FramePtr = std::shared_ptr; @@ -41,27 +41,16 @@ public: static FramePtr Create(); - /** - * @brief Get frame's width in pixels - */ - const int& width() const; - void set_width(const int& width); + const VideoRenderingParams& video_params() const; + void set_video_params(const VideoRenderingParams& params); - /** - * @brief Get frame's height in pixels - */ + const int& width() const; const int& height() const; - void set_height(const int& height); + const PixelFormat::Format& format() const; const rational& sample_aspect_ratio() const; void set_sample_aspect_ratio(const rational& sample_aspect_ratio); - const AudioRenderingParams& audio_params() const; - void set_audio_params(const AudioRenderingParams& params); - - const int &sample_count() const; - void set_sample_count(const int &sample_count); - /** * @brief Get frame's timestamp. * @@ -73,16 +62,6 @@ public: const int64_t& native_timestamp(); void set_native_timestamp(const int64_t& timestamp); - /** - * @brief Get frame's format - * - * @return - * - * Currently this will either be an olive::PixelFormat (video) or an olive::SampleFormat (audio). - */ - const PixelFormat::Format& format() const; - void set_format(const PixelFormat::Format& format); - /** * @brief Returns a copy of the data in this frame as a QByteArray * @@ -127,15 +106,7 @@ public: int allocated_size() const; private: - int width_; - - int height_; - - PixelFormat::Format format_; - - AudioRenderingParams audio_params_; - - int sample_count_; + VideoRenderingParams params_; QByteArray data_; diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index d92b371f3..1712829b8 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -183,9 +183,9 @@ FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const int& divider FramePtr frame = Frame::Create(); - frame->set_width(buffer_->spec().width / divider); - frame->set_height(buffer_->spec().height / divider); - frame->set_format(pix_fmt_); + frame->set_video_params(VideoRenderingParams(buffer_->spec().width / divider, + buffer_->spec().height / divider, + pix_fmt_)); frame->allocate(); if (divider == 1) { diff --git a/app/codec/samplebuffer.cpp b/app/codec/samplebuffer.cpp new file mode 100644 index 000000000..dd089c876 --- /dev/null +++ b/app/codec/samplebuffer.cpp @@ -0,0 +1,211 @@ +#include "samplebuffer.h" + +SampleBuffer::SampleBuffer() : + sample_count_per_channel_(0), + data_(nullptr) +{ +} + +SampleBuffer::~SampleBuffer() +{ + destroy(); +} + +SampleBufferPtr SampleBuffer::Create() +{ + return std::make_shared(); +} + +SampleBufferPtr SampleBuffer::CreateAllocated(const AudioRenderingParams &audio_params, int samples_per_channel) +{ + SampleBufferPtr buffer = Create(); + + buffer->set_audio_params(audio_params); + buffer->set_sample_count_per_channel(samples_per_channel); + buffer->allocate(); + + return buffer; +} + +SampleBufferPtr SampleBuffer::CreateFromPackedData(const AudioRenderingParams &audio_params, const QByteArray &bytes) +{ + if (!audio_params.is_valid()) { + qWarning() << "Tried to create from packed data with invalid parameters"; + return nullptr; + } + + int samples_per_channel = audio_params.bytes_to_samples(bytes.size()); + SampleBufferPtr buffer = CreateAllocated(audio_params, samples_per_channel); + + int total_samples = samples_per_channel * audio_params.channel_count(); + + const float* packed_data = reinterpret_cast(bytes.constData()); + + for (int i=0;idata_[channel][index] = packed_data[i]; + } + + return buffer; +} + +const AudioRenderingParams &SampleBuffer::audio_params() const +{ + return audio_params_; +} + +void SampleBuffer::set_audio_params(const AudioRenderingParams ¶ms) +{ + if (data_) { + qWarning() << "Tried to set parameters on allocated sample buffer"; + return; + } + + audio_params_ = params; +} + +const int &SampleBuffer::sample_count_per_channel() const +{ + return sample_count_per_channel_; +} + +void SampleBuffer::set_sample_count_per_channel(const int &sample_count) +{ + if (data_) { + qWarning() << "Tried to set sample count on allocated sample buffer"; + return; + } + + sample_count_per_channel_ = sample_count; +} + +float **SampleBuffer::data() +{ + return data_; +} + +float *SampleBuffer::channel_data(int channel) +{ + return data_[channel]; +} + +bool SampleBuffer::is_allocated() const +{ + return data_; +} + +void SampleBuffer::allocate() +{ + if (!audio_params_.is_valid()) { + qWarning() << "Tried to allocate sample buffer with invalid audio parameters"; + return; + } + + if (!sample_count_per_channel_) { + qWarning() << "Tried to allocate sample buffer with zero sample count"; + return; + } + + if (data_) { + qWarning() << "Tried to allocate already allocated sample buffer"; + return; + } + + allocate_sample_buffer(&data_, audio_params_.channel_count(), sample_count_per_channel_); +} + +void SampleBuffer::destroy() +{ + destroy_sample_buffer(&data_, audio_params_.channel_count()); +} + +void SampleBuffer::reverse() +{ + if (!is_allocated()) { + qWarning() << "Tried to reverse an unallocated sample buffer"; + return; + } + + int half_nb_sample = sample_count_per_channel_ / 2; + + for (int i=0;i(sample_count_per_channel_) * speed); + + float** input_data = data_; + float** output_data; + + allocate_sample_buffer(&output_data, audio_params_.channel_count(), max_adjusted_nb_samples); + + for (int i=0;i(i) * speed); + + for (int j=0;j(packed_data.data()); + + int output_index = 0; + + for (int j=0;j + +#include "common/constructors.h" +#include "render/audioparams.h" + +class SampleBuffer; +using SampleBufferPtr = std::shared_ptr; + +/** + * @brief A buffer of audio samples + * + * Audio samples in this structure are always stored in PLANAR (separated by channel). This is done to simplify audio + * rendering code. This replaces the old system of using QByteArrays (containing packed audio) and while SampleBuffer + * replaces many of those in the rendering/processing side of things, QByteArrays are currently still in use for + * playback, including reading to and from the cache. + */ +class SampleBuffer +{ +public: + SampleBuffer(); + + virtual ~SampleBuffer(); + + static SampleBufferPtr Create(); + static SampleBufferPtr CreateAllocated(const AudioRenderingParams& audio_params, int samples_per_channel); + static SampleBufferPtr CreateFromPackedData(const AudioRenderingParams& audio_params, const QByteArray& bytes); + + DISABLE_COPY_MOVE(SampleBuffer) + + const AudioRenderingParams& audio_params() const; + void set_audio_params(const AudioRenderingParams& params); + + const int &sample_count_per_channel() const; + void set_sample_count_per_channel(const int &sample_count_per_channel); + + float** data(); + float* channel_data(int channel); + float* sample_data(int index); + + bool is_allocated() const; + void allocate(); + void destroy(); + + void reverse(); + void speed(double speed); + + QByteArray toPackedData() const; + +private: + static void allocate_sample_buffer(float*** data, int nb_channels, int nb_samples); + + static void destroy_sample_buffer(float*** data, int nb_channels); + + AudioRenderingParams audio_params_; + + int sample_count_per_channel_; + + float** data_; + +}; + +Q_DECLARE_METATYPE(SampleBufferPtr) + +#endif // SAMPLEBUFFER_H diff --git a/app/core.cpp b/app/core.cpp index f7cfca5c5..51ee9d95a 100644 --- a/app/core.cpp +++ b/app/core.cpp @@ -374,6 +374,7 @@ void Core::DeclareTypesForQt() qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); + qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); diff --git a/app/dialog/preferences/tabs/preferencesqualitytab.cpp b/app/dialog/preferences/tabs/preferencesqualitytab.cpp index 357444335..1ebb4d3a4 100644 --- a/app/dialog/preferences/tabs/preferencesqualitytab.cpp +++ b/app/dialog/preferences/tabs/preferencesqualitytab.cpp @@ -4,6 +4,7 @@ #include #include +#include "audio/sampleformat.h" #include "render/colormanager.h" #include "render/pixelformat.h" diff --git a/app/node/audio/pan/pan.cpp b/app/node/audio/pan/pan.cpp index 267817616..8dbd3a5b8 100644 --- a/app/node/audio/pan/pan.cpp +++ b/app/node/audio/pan/pan.cpp @@ -47,7 +47,7 @@ NodeInput *PanNode::ProcessesSamplesFrom() const return samples_input_; } -void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams ¶ms, const float *input, float *output, int index) const +void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams ¶ms, const SampleBufferPtr input, SampleBufferPtr output, int index) const { if (params.channel_count() != 2) { // This node currently only works for stereo audio @@ -56,18 +56,14 @@ void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderi float pan_val = (*values)[panning_input_].Get(NodeParam::kFloat).toFloat(); - output[index] = input[index]; + for (int i=0;idata()[i][index] = input->data()[i][index]; + } - if (index%2 == 0) { - // Sample is left channel - if (pan_val > 0) { - output[index] *= (1.0F - pan_val); - } - } else { - // Sample is right channel - if (pan_val < 0) { - output[index] *= (1.0F - qAbs(pan_val)); - } + if (pan_val > 0) { + output->data()[0][index] *= (1.0F - pan_val); + } else if (pan_val < 0) { + output->data()[1][index] *= (1.0F - qAbs(pan_val)); } } diff --git a/app/node/audio/pan/pan.h b/app/node/audio/pan/pan.h index 2aaf1c656..cf7c62460 100644 --- a/app/node/audio/pan/pan.h +++ b/app/node/audio/pan/pan.h @@ -17,7 +17,7 @@ public: virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; virtual NodeInput* ProcessesSamplesFrom() const override; - virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override; + virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; virtual void Retranslate() override; diff --git a/app/node/audio/volume/volume.cpp b/app/node/audio/volume/volume.cpp index 2e785e11d..6a941ea6e 100644 --- a/app/node/audio/volume/volume.cpp +++ b/app/node/audio/volume/volume.cpp @@ -46,11 +46,13 @@ NodeInput *VolumeNode::ProcessesSamplesFrom() const return samples_input_; } -void VolumeNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const float* input, float* output, int index) const +void VolumeNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const { float volume_val = (*values)[volume_input_].Get(NodeParam::kFloat).toFloat(); - output[index] = input[index] * volume_val; + for (int i=0;idata()[i][index] = input->data()[i][index] * volume_val; + } } void VolumeNode::Retranslate() diff --git a/app/node/audio/volume/volume.h b/app/node/audio/volume/volume.h index e0f2546c1..f417b7449 100644 --- a/app/node/audio/volume/volume.h +++ b/app/node/audio/volume/volume.h @@ -17,7 +17,7 @@ public: virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; virtual NodeInput* ProcessesSamplesFrom() const override; - virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override; + virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; virtual void Retranslate() override; diff --git a/app/node/node.cpp b/app/node/node.cpp index a730e55b7..89d33b2e1 100644 --- a/app/node/node.cpp +++ b/app/node/node.cpp @@ -482,13 +482,8 @@ NodeInput* Node::ProcessesSamplesFrom() const return nullptr; } -void Node::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams ¶ms, const float *input, float *output, int index) const +void Node::ProcessSamples(const NodeValueDatabase*, const AudioRenderingParams&, const SampleBufferPtr, SampleBufferPtr, int) const { - Q_UNUSED(values) - Q_UNUSED(params) - Q_UNUSED(input) - Q_UNUSED(output) - Q_UNUSED(index) } NodeParam *Node::GetParameterWithID(const QString &id) const diff --git a/app/node/node.h b/app/node/node.h index c226cf661..66ab19bb7 100644 --- a/app/node/node.h +++ b/app/node/node.h @@ -26,6 +26,7 @@ #include #include +#include "codec/samplebuffer.h" #include "common/rational.h" #include "common/xmlutils.h" #include "node/dependency.h" @@ -190,7 +191,7 @@ public: /** * @brief If ProcessesSamples() is true, this is the function that will process them. */ - virtual void ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const float* input, float* output, int index) const; + virtual void ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const; /** * @brief Returns the parameter with the specified ID (or nullptr if it doesn't exist) diff --git a/app/node/param.h b/app/node/param.h index 73d488efb..28bfb6766 100644 --- a/app/node/param.h +++ b/app/node/param.h @@ -137,7 +137,7 @@ public: /** * Audio samples type * - * Resolves to `QVector4D`. + * Resolves to `SampleBufferPtr`. */ kSamples = 0x400, diff --git a/app/render/backend/audio/audiobackend.cpp b/app/render/backend/audio/audiobackend.cpp index 3c9bd5003..85135e931 100644 --- a/app/render/backend/audio/audiobackend.cpp +++ b/app/render/backend/audio/audiobackend.cpp @@ -62,7 +62,7 @@ void AudioBackend::ThreadCompletedCache(NodeDependency dep, NodeValueTable data, if (job_time == render_job_info_.value(dep.range())) { render_job_info_.remove(dep.range()); - QByteArray cached_samples = data.Get(NodeParam::kSamples).toByteArray(); + QByteArray cached_samples = data.Get(NodeParam::kSamples).value()->toPackedData(); int offset = params().time_to_bytes(dep.in()); int length = params().time_to_bytes(dep.range().length()); diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index e32637e34..fffcb1a00 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -12,11 +12,10 @@ void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR } if (decoder->HasConformedVersion(audio_params())) { - FramePtr frame = nullptr; - frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params()); + SampleBufferPtr frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params()); if (frame) { - table->Push(NodeParam::kSamples, frame->ToByteArray()); + table->Push(NodeParam::kSamples, QVariant::fromValue(frame)); } } else { emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params()); @@ -41,10 +40,10 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c return; } - QByteArray input_buffer = samples_var.toByteArray(); - QByteArray output_buffer(input_buffer.size(), 0); + SampleBufferPtr input_buffer = samples_var.value(); + SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(input_buffer->audio_params(), input_buffer->sample_count_per_channel()); - int sample_count = input_buffer.size() / audio_params().bytes_per_sample_per_channel(); + int sample_count = input_buffer->sample_count_per_channel(); // FIXME: Hardcoded float sample format for (int i=0;iProcessSamples(&input_params, audio_params(), - reinterpret_cast(input_buffer.constData()), - reinterpret_cast(output_buffer.data()), + input_buffer, + output_buffer, i); } - output_params->Push(NodeParam::kSamples, output_buffer); + output_params->Push(NodeParam::kSamples, QVariant::fromValue(output_buffer)); } diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index 997fa521c..d251f6054 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -35,7 +35,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti QList active_blocks = track->BlocksAtTimeRange(range); // All these blocks will need to output to a buffer so we create one here - QByteArray block_range_buffer(audio_params_.time_to_bytes(range.length()), 0); + SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params_, audio_params_.time_to_samples(range.length())); NodeValueTable merged_table; @@ -46,69 +46,56 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti // Destination buffer NodeValueTable table = ProcessNode(NodeDependency(b, range_for_block)); - QByteArray samples_from_this_block = table.Take(NodeParam::kSamples).toByteArray(); + QVariant sample_val = table.Take(NodeParam::kSamples); - if (!samples_from_this_block.isEmpty()) { - // Stretch samples here - rational abs_speed = qAbs(b->speed()); + if (sample_val.isNull()) { + continue; + } - if (abs_speed != 1) { - QByteArray speed_adjusted_samples; + SampleBufferPtr samples_from_this_block = sample_val.value(); - double clip_speed = abs_speed.toDouble(); + // Stretch samples here + rational abs_speed = qAbs(b->speed()); - int sample_count = audio_params_.bytes_to_samples(samples_from_this_block.size()); + if (abs_speed != 1) { + samples_from_this_block->speed(abs_speed.toDouble()); + } - for (double i=0;iis_reversed()) { + // Reverse the audio buffer + samples_from_this_block->reverse(); + } - QByteArray sample_at_this_index = samples_from_this_block.mid(byte_index, audio_params_.samples_to_bytes(1)); - speed_adjusted_samples.append(sample_at_this_index); - } + int destination_offset = audio_params_.time_to_samples(range_for_block.in() - range.in()) * sizeof(float); + int max_dest_sz = audio_params_.time_to_samples(range_for_block.length()) * sizeof(float); + int input_sz = samples_from_this_block->sample_count_per_channel() * sizeof(float); + int actual_copy_size = qMin(max_dest_sz, input_sz); - samples_from_this_block = speed_adjusted_samples; + for (int i=0;idata()[i] + destination_offset, + samples_from_this_block->data()[i], + actual_copy_size); + + if (input_sz < max_dest_sz) { + memset(block_range_buffer->data()[i] + destination_offset + input_sz, 0, max_dest_sz - input_sz); } + } - if (b->is_reversed()) { - // Reverse the audio buffer - AudioManager::ReverseBuffer(samples_from_this_block.data(), samples_from_this_block.size(), audio_params_.samples_to_bytes(1)); - } + { + // Save waveform to file + Block* src_block = static_cast(copy_map_->value(b)); + QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString()); + QDir waveform_loc = local_appdata_dir.filePath("waveform"); + waveform_loc.mkpath("."); + QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast(src_block)))); - int destination_offset = audio_params_.time_to_bytes(range_for_block.in() - range.in()); - int actual_copy_size = qMin(audio_params_.time_to_bytes(range_for_block.length()), samples_from_this_block.size()); + if (wave_file.open(QFile::ReadWrite)) { + // We use S32 as a size-compatible substitute for SampleSummer::Sum which is 4 bytes in size + AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S32); + int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()); - if (destination_offset < 0 - || destination_offset + actual_copy_size > block_range_buffer.size()) { - qCritical() << "Tried to copy audio beyond the size of the destination buffer"; - qCritical() << " Destination size:" << block_range_buffer.size(); - qCritical() << " Destination offset:" << destination_offset; - qCritical() << " Copy size:" << actual_copy_size; - abort(); - } else { - memcpy(block_range_buffer.data()+destination_offset, - samples_from_this_block.data(), - actual_copy_size); - - // Save waveform to file - Block* src_block = static_cast(copy_map_->value(b)); - QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString()); - QDir waveform_loc = local_appdata_dir.filePath("waveform"); - waveform_loc.mkpath("."); - QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast(src_block)))); - wave_file.open(QFile::ReadWrite); - - // FIXME: Assumes 32-bit float - float* flt_samples = reinterpret_cast(samples_from_this_block.data()); - int nb_sample = samples_from_this_block.size() / sizeof(float); - - // FIXME: Hardcoded sample rate - // We use S16 here as a size-compatible substitute for qfloat16/half - AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S16); - int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()) * waveform_params.channel_count(); - - qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()) * 2; - qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()) * 2; + qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()); + qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()); qint64 end_offset = start_offset + length_offset; if (wave_file.size() < end_offset) { @@ -117,10 +104,10 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti wave_file.seek(start_offset); - for (int i=0;i summary = SampleSummer::SumSamples(&flt_samples[i], - qMin(chunk_size, nb_sample - i), - audio_params().channel_count()); + for (int i=0;isample_count_per_channel();i+=chunk_size) { + QVector summary = SampleSummer::SumSamples(samples_from_this_block, + i, + qMin(chunk_size, samples_from_this_block->sample_count_per_channel() - i)); wave_file.write(reinterpret_cast(summary.constData()), summary.size() * sizeof(SampleSummer::Sum)); @@ -137,7 +124,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti NodeValueTable::Merge({merged_table, table}); } - merged_table.Push(NodeParam::kSamples, block_range_buffer); + merged_table.Push(NodeParam::kSamples, QVariant::fromValue(block_range_buffer)); return merged_table; } diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index a2de88f3b..ea3c29993 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -317,9 +317,7 @@ void VideoRenderWorker::Download(const rational& time, QVariant texture, QString } else { FramePtr frame = Frame::Create(); - frame->set_width(video_params().width()); - frame->set_height(video_params().height()); - frame->set_format(video_params().format()); + frame->set_video_params(video_params()); frame->allocate(); TextureToBuffer(texture, frame->data()); diff --git a/app/render/pixelformat.cpp b/app/render/pixelformat.cpp index 9fa451c3b..21221a48b 100644 --- a/app/render/pixelformat.cpp +++ b/app/render/pixelformat.cpp @@ -237,10 +237,8 @@ FramePtr PixelFormat::ConvertPixelFormat(FramePtr frame, const PixelFormat::Form FramePtr converted = Frame::Create(); // Copy parameters - converted->set_width(frame->width()); - converted->set_height(frame->height()); + converted->set_video_params(frame->video_params()); converted->set_timestamp(frame->timestamp()); - converted->set_format(dest_format); converted->allocate(); OIIO::ImageBuf src(OIIO::ImageSpec(frame->width(), frame->height(), ChannelCount(frame->format()), GetOIIOTypeDesc(frame->format())), frame->data()); diff --git a/app/render/videoparams.cpp b/app/render/videoparams.cpp index b81fd6059..a71e80327 100644 --- a/app/render/videoparams.cpp +++ b/app/render/videoparams.cpp @@ -34,6 +34,12 @@ VideoRenderingParams::VideoRenderingParams() : { } +VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const PixelFormat::Format &format) : + VideoParams(width, height, rational()), + format_(format) +{ +} + VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const rational &time_base, const PixelFormat::Format &format, const RenderMode::Mode& mode, const int ÷r) : VideoParams(width, height, time_base), format_(format), diff --git a/app/render/videoparams.h b/app/render/videoparams.h index 68b6efb4e..2a593c783 100644 --- a/app/render/videoparams.h +++ b/app/render/videoparams.h @@ -25,6 +25,7 @@ private: class VideoRenderingParams : public VideoParams { public: VideoRenderingParams(); + VideoRenderingParams(const int& width, const int& height, const PixelFormat::Format& format); VideoRenderingParams(const int& width, const int& height, const rational& time_base, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1); VideoRenderingParams(const VideoParams& params, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1); diff --git a/app/widget/viewer/viewerglwidget.cpp b/app/widget/viewer/viewerglwidget.cpp index d89125d55..8dc05e027 100644 --- a/app/widget/viewer/viewerglwidget.cpp +++ b/app/widget/viewer/viewerglwidget.cpp @@ -101,9 +101,7 @@ void ViewerGLWidget::SetImage(const QString &fn) load_buffer_.destroy(); texture_.Destroy(); - load_buffer_.set_width(input->spec().width); - load_buffer_.set_height(input->spec().height); - load_buffer_.set_format(image_format); + load_buffer_.set_video_params(VideoRenderingParams(input->spec().width, input->spec().height, image_format)); load_buffer_.allocate(); texture_.Create(context(), input->spec().width, input->spec().height, image_format);