diff --git a/app/audio/sumsamples.cpp b/app/audio/sumsamples.cpp index 7f90641d8..1ed6cb498 100644 --- a/app/audio/sumsamples.cpp +++ b/app/audio/sumsamples.cpp @@ -14,6 +14,21 @@ QVector SampleSummer::SumSamples(const qfloat16 *samples, int return SumSamplesInternal(samples, nb_samples, nb_channels); } +QVector SampleSummer::SumSamples(SampleBufferPtr samples, int start_index, int length) +{ + QVector summed_samples(samples->audio_params().channel_count()); + + int end_index = start_index + length; + + for (int i=start_index;iaudio_params().channel_count();channel++) { + ClampMinMax(summed_samples[channel], samples->data()[channel][i]); + } + } + + return summed_samples; +} + QVector SampleSummer::ReSumSamples(const SampleSummer::Sum *samples, int nb_samples, int nb_channels) { QVector summed_samples(nb_channels); @@ -47,18 +62,20 @@ QVector SampleSummer::SumSamplesInternal(const T *samples, in QVector summed_samples(nb_channels); for (int i=0;i summed_samples[channel].max) { - summed_samples[channel].max = sample; - } + ClampMinMax(summed_samples[i%nb_channels], samples[i]); } return summed_samples; } + +template +void SampleSummer::ClampMinMax(SampleSummer::Sum &sum, T value) +{ + if (value < sum.min) { + sum.min = value; + } + + if (value > sum.max) { + sum.max = value; + } +} diff --git a/app/audio/sumsamples.h b/app/audio/sumsamples.h index c74f9b30c..38bb29546 100644 --- a/app/audio/sumsamples.h +++ b/app/audio/sumsamples.h @@ -4,6 +4,8 @@ #include #include +#include "codec/samplebuffer.h" + class SampleSummer { public: struct Sum { @@ -18,6 +20,7 @@ public: static QVector SumSamples(const float* samples, int nb_samples, int nb_channels); static QVector SumSamples(const qfloat16* samples, int nb_samples, int nb_channels); + static QVector SumSamples(SampleBufferPtr samples, int start_index, int length); static QVector ReSumSamples(const SampleSummer::Sum* samples, int nb_samples, int nb_channels); @@ -25,6 +28,9 @@ private: template static QVector SumSamplesInternal(const T* samples, int nb_samples, int nb_channels); + template + static void ClampMinMax(Sum &sum, T value); + }; #endif // SUMSAMPLES_H diff --git a/app/codec/CMakeLists.txt b/app/codec/CMakeLists.txt index 6a1158a72..791500cdd 100644 --- a/app/codec/CMakeLists.txt +++ b/app/codec/CMakeLists.txt @@ -25,6 +25,8 @@ set(OLIVE_SOURCES codec/encoder.cpp codec/frame.h codec/frame.cpp + codec/samplebuffer.h + codec/samplebuffer.cpp codec/waveinput.h codec/waveinput.cpp codec/waveoutput.h diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index 224818912..1ebdf9d35 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -66,7 +66,7 @@ FramePtr Decoder::RetrieveVideo(const rational &/*timecode*/, const int &/*divid return nullptr; } -FramePtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/) +SampleBufferPtr Decoder::RetrieveAudio(const rational &/*timecode*/, const rational &/*length*/, const AudioRenderingParams &/*params*/) { return nullptr; } diff --git a/app/codec/decoder.h b/app/codec/decoder.h index af3507e33..1517ebaa3 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -26,6 +26,7 @@ #include #include "codec/frame.h" +#include "codec/samplebuffer.h" #include "codec/waveoutput.h" #include "common/constructors.h" #include "common/rational.h" @@ -168,7 +169,7 @@ public: * A FramePtr of valid data at this timecode of the requested length or nullptr if there was nothing to retrieve at * the provided timecode or the media could not be opened. */ - virtual FramePtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params); + virtual SampleBufferPtr RetrieveAudio(const rational& timecode, const rational& length, const AudioRenderingParams& params); virtual bool SupportsVideo(); virtual bool SupportsAudio(); diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index dc5795f36..ab9c290ae 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -400,9 +400,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid // We found the frame, we'll return a copy if (return_frame) { FramePtr copy = Frame::Create(); - copy->set_width(return_frame->width()); - copy->set_height(return_frame->height()); - copy->set_format(return_frame->format()); + copy->set_video_params(return_frame->video_params()); copy->set_timestamp(return_frame->timestamp()); copy->set_sample_aspect_ratio(return_frame->sample_aspect_ratio()); copy->allocate(); @@ -415,7 +413,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid return nullptr; } -FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams ¶ms) +SampleBufferPtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams ¶ms) { QMutexLocker locker(&mutex_); @@ -428,29 +426,19 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational & return nullptr; } - //Conform(params, cancelled); - WaveInput input(GetConformedFilename(params)); if (input.open()) { const AudioRenderingParams& input_params = input.params(); - FramePtr audio_frame = Frame::Create(); - audio_frame->set_audio_params(input_params); - audio_frame->set_sample_count(input_params.time_to_samples(length)); - audio_frame->allocate(); - - qint64 actual_read = input.read(input_params.time_to_bytes(timecode), - audio_frame->data(), - audio_frame->allocated_size()); - - if (actual_read < audio_frame->allocated_size()) { - memset(audio_frame->data() + actual_read, 0, audio_frame->allocated_size() - actual_read); - } - + // Read bytes from wav + QByteArray packed_data = input.read(input_params.time_to_bytes(timecode), input_params.time_to_bytes(length)); input.close(); - return audio_frame; + // Create sample buffer + SampleBufferPtr sample_buffer = SampleBuffer::CreateFromPackedData(input_params, packed_data); + + return sample_buffer; } return nullptr; diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 4ceb50184..8a95d7544 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -55,7 +55,7 @@ public: virtual bool Open() override; virtual RetrieveState GetRetrieveState(const rational &time) override; virtual FramePtr RetrieveVideo(const rational &timecode, const int& divider) override; - virtual FramePtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override; + virtual SampleBufferPtr RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams& params) override; virtual void Close() override; virtual QString id() override; diff --git a/app/codec/ffmpeg/ffmpegframecache.cpp b/app/codec/ffmpeg/ffmpegframecache.cpp index 97e43e4f1..e56d61fad 100644 --- a/app/codec/ffmpeg/ffmpegframecache.cpp +++ b/app/codec/ffmpeg/ffmpegframecache.cpp @@ -85,9 +85,7 @@ Frame* FFmpegFrameCache::Get(const VideoRenderingParams ¶ms) // Otherwise we'll need to create one Frame* f = new Frame(); - f->set_width(params.width()); - f->set_height(params.height()); - f->set_format(params.format()); + f->set_video_params(params); f->allocate(); return f; diff --git a/app/codec/frame.cpp b/app/codec/frame.cpp index 07fc08012..34bb4d742 100644 --- a/app/codec/frame.cpp +++ b/app/codec/frame.cpp @@ -24,10 +24,6 @@ #include Frame::Frame() : - width_(0), - height_(0), - format_(PixelFormat::PIX_FMT_INVALID), - sample_count_(0), timestamp_(0), sample_aspect_ratio_(1) { @@ -38,24 +34,29 @@ FramePtr Frame::Create() return std::make_shared(); } -const int &Frame::width() const +const VideoRenderingParams &Frame::video_params() const { - return width_; + return params_; } -void Frame::set_width(const int &width) +void Frame::set_video_params(const VideoRenderingParams ¶ms) { - width_ = width; + params_ = params; +} + +const int &Frame::width() const +{ + return params_.width(); } const int &Frame::height() const { - return height_; + return params_.height(); } -void Frame::set_height(const int &height) +const PixelFormat::Format &Frame::format() const { - height_ = height; + return params_.format(); } const rational &Frame::sample_aspect_ratio() const @@ -68,16 +69,6 @@ void Frame::set_sample_aspect_ratio(const rational &aspect_ratio) sample_aspect_ratio_ = aspect_ratio; } -const AudioRenderingParams &Frame::audio_params() const -{ - return audio_params_; -} - -void Frame::set_audio_params(const AudioRenderingParams ¶ms) -{ - audio_params_ = params; -} - const rational &Frame::timestamp() const { return timestamp_; @@ -98,31 +89,11 @@ void Frame::set_native_timestamp(const int64_t ×tamp) native_timestamp_ = timestamp; } -const PixelFormat::Format &Frame::format() const -{ - return format_; -} - -void Frame::set_format(const PixelFormat::Format &format) -{ - format_ = format; -} - QByteArray Frame::ToByteArray() const { return data_; } -const int &Frame::sample_count() const -{ - return sample_count_; -} - -void Frame::set_sample_count(const int &audio_sample_count) -{ - sample_count_ = audio_sample_count; -} - char *Frame::data() { return data_.data(); @@ -136,11 +107,11 @@ const char *Frame::const_data() const void Frame::allocate() { // Assume this frame is intended to be a video frame - if (width_ > 0 && height_ > 0) { - data_.resize(PixelFormat::GetBufferSize(static_cast(format_), width_, height_)); - } else if (sample_count_ > 0) { - data_.resize(audio_params_.samples_to_bytes(sample_count_)); + if (!params_.is_valid()) { + qWarning() << "Tried to allocate a frame with invalid parameters"; } + + data_.resize(PixelFormat::GetBufferSize(params_.format(), params_.width(), params_.height())); } bool Frame::is_allocated() const diff --git a/app/codec/frame.h b/app/codec/frame.h index 1311a80dd..fe95ab05f 100644 --- a/app/codec/frame.h +++ b/app/codec/frame.h @@ -25,8 +25,8 @@ #include #include "common/rational.h" -#include "render/audioparams.h" #include "render/pixelformat.h" +#include "render/videoparams.h" class Frame; using FramePtr = std::shared_ptr; @@ -41,27 +41,16 @@ public: static FramePtr Create(); - /** - * @brief Get frame's width in pixels - */ - const int& width() const; - void set_width(const int& width); + const VideoRenderingParams& video_params() const; + void set_video_params(const VideoRenderingParams& params); - /** - * @brief Get frame's height in pixels - */ + const int& width() const; const int& height() const; - void set_height(const int& height); + const PixelFormat::Format& format() const; const rational& sample_aspect_ratio() const; void set_sample_aspect_ratio(const rational& sample_aspect_ratio); - const AudioRenderingParams& audio_params() const; - void set_audio_params(const AudioRenderingParams& params); - - const int &sample_count() const; - void set_sample_count(const int &sample_count); - /** * @brief Get frame's timestamp. * @@ -73,16 +62,6 @@ public: const int64_t& native_timestamp(); void set_native_timestamp(const int64_t& timestamp); - /** - * @brief Get frame's format - * - * @return - * - * Currently this will either be an olive::PixelFormat (video) or an olive::SampleFormat (audio). - */ - const PixelFormat::Format& format() const; - void set_format(const PixelFormat::Format& format); - /** * @brief Returns a copy of the data in this frame as a QByteArray * @@ -127,15 +106,7 @@ public: int allocated_size() const; private: - int width_; - - int height_; - - PixelFormat::Format format_; - - AudioRenderingParams audio_params_; - - int sample_count_; + VideoRenderingParams params_; QByteArray data_; diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index d92b371f3..1712829b8 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -183,9 +183,9 @@ FramePtr OIIODecoder::RetrieveVideo(const rational &timecode, const int& divider FramePtr frame = Frame::Create(); - frame->set_width(buffer_->spec().width / divider); - frame->set_height(buffer_->spec().height / divider); - frame->set_format(pix_fmt_); + frame->set_video_params(VideoRenderingParams(buffer_->spec().width / divider, + buffer_->spec().height / divider, + pix_fmt_)); frame->allocate(); if (divider == 1) { diff --git a/app/codec/samplebuffer.cpp b/app/codec/samplebuffer.cpp new file mode 100644 index 000000000..dd089c876 --- /dev/null +++ b/app/codec/samplebuffer.cpp @@ -0,0 +1,211 @@ +#include "samplebuffer.h" + +SampleBuffer::SampleBuffer() : + sample_count_per_channel_(0), + data_(nullptr) +{ +} + +SampleBuffer::~SampleBuffer() +{ + destroy(); +} + +SampleBufferPtr SampleBuffer::Create() +{ + return std::make_shared(); +} + +SampleBufferPtr SampleBuffer::CreateAllocated(const AudioRenderingParams &audio_params, int samples_per_channel) +{ + SampleBufferPtr buffer = Create(); + + buffer->set_audio_params(audio_params); + buffer->set_sample_count_per_channel(samples_per_channel); + buffer->allocate(); + + return buffer; +} + +SampleBufferPtr SampleBuffer::CreateFromPackedData(const AudioRenderingParams &audio_params, const QByteArray &bytes) +{ + if (!audio_params.is_valid()) { + qWarning() << "Tried to create from packed data with invalid parameters"; + return nullptr; + } + + int samples_per_channel = audio_params.bytes_to_samples(bytes.size()); + SampleBufferPtr buffer = CreateAllocated(audio_params, samples_per_channel); + + int total_samples = samples_per_channel * audio_params.channel_count(); + + const float* packed_data = reinterpret_cast(bytes.constData()); + + for (int i=0;idata_[channel][index] = packed_data[i]; + } + + return buffer; +} + +const AudioRenderingParams &SampleBuffer::audio_params() const +{ + return audio_params_; +} + +void SampleBuffer::set_audio_params(const AudioRenderingParams ¶ms) +{ + if (data_) { + qWarning() << "Tried to set parameters on allocated sample buffer"; + return; + } + + audio_params_ = params; +} + +const int &SampleBuffer::sample_count_per_channel() const +{ + return sample_count_per_channel_; +} + +void SampleBuffer::set_sample_count_per_channel(const int &sample_count) +{ + if (data_) { + qWarning() << "Tried to set sample count on allocated sample buffer"; + return; + } + + sample_count_per_channel_ = sample_count; +} + +float **SampleBuffer::data() +{ + return data_; +} + +float *SampleBuffer::channel_data(int channel) +{ + return data_[channel]; +} + +bool SampleBuffer::is_allocated() const +{ + return data_; +} + +void SampleBuffer::allocate() +{ + if (!audio_params_.is_valid()) { + qWarning() << "Tried to allocate sample buffer with invalid audio parameters"; + return; + } + + if (!sample_count_per_channel_) { + qWarning() << "Tried to allocate sample buffer with zero sample count"; + return; + } + + if (data_) { + qWarning() << "Tried to allocate already allocated sample buffer"; + return; + } + + allocate_sample_buffer(&data_, audio_params_.channel_count(), sample_count_per_channel_); +} + +void SampleBuffer::destroy() +{ + destroy_sample_buffer(&data_, audio_params_.channel_count()); +} + +void SampleBuffer::reverse() +{ + if (!is_allocated()) { + qWarning() << "Tried to reverse an unallocated sample buffer"; + return; + } + + int half_nb_sample = sample_count_per_channel_ / 2; + + for (int i=0;i(sample_count_per_channel_) * speed); + + float** input_data = data_; + float** output_data; + + allocate_sample_buffer(&output_data, audio_params_.channel_count(), max_adjusted_nb_samples); + + for (int i=0;i(i) * speed); + + for (int j=0;j(packed_data.data()); + + int output_index = 0; + + for (int j=0;j + +#include "common/constructors.h" +#include "render/audioparams.h" + +class SampleBuffer; +using SampleBufferPtr = std::shared_ptr; + +/** + * @brief A buffer of audio samples + * + * Audio samples in this structure are always stored in PLANAR (separated by channel). This is done to simplify audio + * rendering code. This replaces the old system of using QByteArrays (containing packed audio) and while SampleBuffer + * replaces many of those in the rendering/processing side of things, QByteArrays are currently still in use for + * playback, including reading to and from the cache. + */ +class SampleBuffer +{ +public: + SampleBuffer(); + + virtual ~SampleBuffer(); + + static SampleBufferPtr Create(); + static SampleBufferPtr CreateAllocated(const AudioRenderingParams& audio_params, int samples_per_channel); + static SampleBufferPtr CreateFromPackedData(const AudioRenderingParams& audio_params, const QByteArray& bytes); + + DISABLE_COPY_MOVE(SampleBuffer) + + const AudioRenderingParams& audio_params() const; + void set_audio_params(const AudioRenderingParams& params); + + const int &sample_count_per_channel() const; + void set_sample_count_per_channel(const int &sample_count_per_channel); + + float** data(); + float* channel_data(int channel); + float* sample_data(int index); + + bool is_allocated() const; + void allocate(); + void destroy(); + + void reverse(); + void speed(double speed); + + QByteArray toPackedData() const; + +private: + static void allocate_sample_buffer(float*** data, int nb_channels, int nb_samples); + + static void destroy_sample_buffer(float*** data, int nb_channels); + + AudioRenderingParams audio_params_; + + int sample_count_per_channel_; + + float** data_; + +}; + +Q_DECLARE_METATYPE(SampleBufferPtr) + +#endif // SAMPLEBUFFER_H diff --git a/app/core.cpp b/app/core.cpp index f7cfca5c5..51ee9d95a 100644 --- a/app/core.cpp +++ b/app/core.cpp @@ -374,6 +374,7 @@ void Core::DeclareTypesForQt() qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); + qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); diff --git a/app/dialog/preferences/tabs/preferencesqualitytab.cpp b/app/dialog/preferences/tabs/preferencesqualitytab.cpp index 357444335..1ebb4d3a4 100644 --- a/app/dialog/preferences/tabs/preferencesqualitytab.cpp +++ b/app/dialog/preferences/tabs/preferencesqualitytab.cpp @@ -4,6 +4,7 @@ #include #include +#include "audio/sampleformat.h" #include "render/colormanager.h" #include "render/pixelformat.h" diff --git a/app/node/audio/pan/pan.cpp b/app/node/audio/pan/pan.cpp index 267817616..8dbd3a5b8 100644 --- a/app/node/audio/pan/pan.cpp +++ b/app/node/audio/pan/pan.cpp @@ -47,7 +47,7 @@ NodeInput *PanNode::ProcessesSamplesFrom() const return samples_input_; } -void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams ¶ms, const float *input, float *output, int index) const +void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams ¶ms, const SampleBufferPtr input, SampleBufferPtr output, int index) const { if (params.channel_count() != 2) { // This node currently only works for stereo audio @@ -56,18 +56,14 @@ void PanNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderi float pan_val = (*values)[panning_input_].Get(NodeParam::kFloat).toFloat(); - output[index] = input[index]; + for (int i=0;idata()[i][index] = input->data()[i][index]; + } - if (index%2 == 0) { - // Sample is left channel - if (pan_val > 0) { - output[index] *= (1.0F - pan_val); - } - } else { - // Sample is right channel - if (pan_val < 0) { - output[index] *= (1.0F - qAbs(pan_val)); - } + if (pan_val > 0) { + output->data()[0][index] *= (1.0F - pan_val); + } else if (pan_val < 0) { + output->data()[1][index] *= (1.0F - qAbs(pan_val)); } } diff --git a/app/node/audio/pan/pan.h b/app/node/audio/pan/pan.h index 2aaf1c656..cf7c62460 100644 --- a/app/node/audio/pan/pan.h +++ b/app/node/audio/pan/pan.h @@ -17,7 +17,7 @@ public: virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; virtual NodeInput* ProcessesSamplesFrom() const override; - virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override; + virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; virtual void Retranslate() override; diff --git a/app/node/audio/volume/volume.cpp b/app/node/audio/volume/volume.cpp index 2e785e11d..6a941ea6e 100644 --- a/app/node/audio/volume/volume.cpp +++ b/app/node/audio/volume/volume.cpp @@ -46,11 +46,13 @@ NodeInput *VolumeNode::ProcessesSamplesFrom() const return samples_input_; } -void VolumeNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const float* input, float* output, int index) const +void VolumeNode::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const { float volume_val = (*values)[volume_input_].Get(NodeParam::kFloat).toFloat(); - output[index] = input[index] * volume_val; + for (int i=0;idata()[i][index] = input->data()[i][index] * volume_val; + } } void VolumeNode::Retranslate() diff --git a/app/node/audio/volume/volume.h b/app/node/audio/volume/volume.h index e0f2546c1..f417b7449 100644 --- a/app/node/audio/volume/volume.h +++ b/app/node/audio/volume/volume.h @@ -17,7 +17,7 @@ public: virtual Capabilities GetCapabilities(const NodeValueDatabase&) const override; virtual NodeInput* ProcessesSamplesFrom() const override; - virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const float* input, float* output, int index) const override; + virtual void ProcessSamples(const NodeValueDatabase* values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const override; virtual void Retranslate() override; diff --git a/app/node/node.cpp b/app/node/node.cpp index a730e55b7..89d33b2e1 100644 --- a/app/node/node.cpp +++ b/app/node/node.cpp @@ -482,13 +482,8 @@ NodeInput* Node::ProcessesSamplesFrom() const return nullptr; } -void Node::ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams ¶ms, const float *input, float *output, int index) const +void Node::ProcessSamples(const NodeValueDatabase*, const AudioRenderingParams&, const SampleBufferPtr, SampleBufferPtr, int) const { - Q_UNUSED(values) - Q_UNUSED(params) - Q_UNUSED(input) - Q_UNUSED(output) - Q_UNUSED(index) } NodeParam *Node::GetParameterWithID(const QString &id) const diff --git a/app/node/node.h b/app/node/node.h index c226cf661..66ab19bb7 100644 --- a/app/node/node.h +++ b/app/node/node.h @@ -26,6 +26,7 @@ #include #include +#include "codec/samplebuffer.h" #include "common/rational.h" #include "common/xmlutils.h" #include "node/dependency.h" @@ -190,7 +191,7 @@ public: /** * @brief If ProcessesSamples() is true, this is the function that will process them. */ - virtual void ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const float* input, float* output, int index) const; + virtual void ProcessSamples(const NodeValueDatabase *values, const AudioRenderingParams& params, const SampleBufferPtr input, SampleBufferPtr output, int index) const; /** * @brief Returns the parameter with the specified ID (or nullptr if it doesn't exist) diff --git a/app/node/param.h b/app/node/param.h index 73d488efb..28bfb6766 100644 --- a/app/node/param.h +++ b/app/node/param.h @@ -137,7 +137,7 @@ public: /** * Audio samples type * - * Resolves to `QVector4D`. + * Resolves to `SampleBufferPtr`. */ kSamples = 0x400, diff --git a/app/render/backend/audio/audiobackend.cpp b/app/render/backend/audio/audiobackend.cpp index 3c9bd5003..85135e931 100644 --- a/app/render/backend/audio/audiobackend.cpp +++ b/app/render/backend/audio/audiobackend.cpp @@ -62,7 +62,7 @@ void AudioBackend::ThreadCompletedCache(NodeDependency dep, NodeValueTable data, if (job_time == render_job_info_.value(dep.range())) { render_job_info_.remove(dep.range()); - QByteArray cached_samples = data.Get(NodeParam::kSamples).toByteArray(); + QByteArray cached_samples = data.Get(NodeParam::kSamples).value()->toPackedData(); int offset = params().time_to_bytes(dep.in()); int length = params().time_to_bytes(dep.range().length()); diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index e32637e34..fffcb1a00 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -12,11 +12,10 @@ void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR } if (decoder->HasConformedVersion(audio_params())) { - FramePtr frame = nullptr; - frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params()); + SampleBufferPtr frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params()); if (frame) { - table->Push(NodeParam::kSamples, frame->ToByteArray()); + table->Push(NodeParam::kSamples, QVariant::fromValue(frame)); } } else { emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params()); @@ -41,10 +40,10 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c return; } - QByteArray input_buffer = samples_var.toByteArray(); - QByteArray output_buffer(input_buffer.size(), 0); + SampleBufferPtr input_buffer = samples_var.value(); + SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(input_buffer->audio_params(), input_buffer->sample_count_per_channel()); - int sample_count = input_buffer.size() / audio_params().bytes_per_sample_per_channel(); + int sample_count = input_buffer->sample_count_per_channel(); // FIXME: Hardcoded float sample format for (int i=0;iProcessSamples(&input_params, audio_params(), - reinterpret_cast(input_buffer.constData()), - reinterpret_cast(output_buffer.data()), + input_buffer, + output_buffer, i); } - output_params->Push(NodeParam::kSamples, output_buffer); + output_params->Push(NodeParam::kSamples, QVariant::fromValue(output_buffer)); } diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index 997fa521c..d251f6054 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -35,7 +35,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti QList active_blocks = track->BlocksAtTimeRange(range); // All these blocks will need to output to a buffer so we create one here - QByteArray block_range_buffer(audio_params_.time_to_bytes(range.length()), 0); + SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params_, audio_params_.time_to_samples(range.length())); NodeValueTable merged_table; @@ -46,69 +46,56 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti // Destination buffer NodeValueTable table = ProcessNode(NodeDependency(b, range_for_block)); - QByteArray samples_from_this_block = table.Take(NodeParam::kSamples).toByteArray(); + QVariant sample_val = table.Take(NodeParam::kSamples); - if (!samples_from_this_block.isEmpty()) { - // Stretch samples here - rational abs_speed = qAbs(b->speed()); + if (sample_val.isNull()) { + continue; + } - if (abs_speed != 1) { - QByteArray speed_adjusted_samples; + SampleBufferPtr samples_from_this_block = sample_val.value(); - double clip_speed = abs_speed.toDouble(); + // Stretch samples here + rational abs_speed = qAbs(b->speed()); - int sample_count = audio_params_.bytes_to_samples(samples_from_this_block.size()); + if (abs_speed != 1) { + samples_from_this_block->speed(abs_speed.toDouble()); + } - for (double i=0;iis_reversed()) { + // Reverse the audio buffer + samples_from_this_block->reverse(); + } - QByteArray sample_at_this_index = samples_from_this_block.mid(byte_index, audio_params_.samples_to_bytes(1)); - speed_adjusted_samples.append(sample_at_this_index); - } + int destination_offset = audio_params_.time_to_samples(range_for_block.in() - range.in()) * sizeof(float); + int max_dest_sz = audio_params_.time_to_samples(range_for_block.length()) * sizeof(float); + int input_sz = samples_from_this_block->sample_count_per_channel() * sizeof(float); + int actual_copy_size = qMin(max_dest_sz, input_sz); - samples_from_this_block = speed_adjusted_samples; + for (int i=0;idata()[i] + destination_offset, + samples_from_this_block->data()[i], + actual_copy_size); + + if (input_sz < max_dest_sz) { + memset(block_range_buffer->data()[i] + destination_offset + input_sz, 0, max_dest_sz - input_sz); } + } - if (b->is_reversed()) { - // Reverse the audio buffer - AudioManager::ReverseBuffer(samples_from_this_block.data(), samples_from_this_block.size(), audio_params_.samples_to_bytes(1)); - } + { + // Save waveform to file + Block* src_block = static_cast(copy_map_->value(b)); + QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString()); + QDir waveform_loc = local_appdata_dir.filePath("waveform"); + waveform_loc.mkpath("."); + QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast(src_block)))); - int destination_offset = audio_params_.time_to_bytes(range_for_block.in() - range.in()); - int actual_copy_size = qMin(audio_params_.time_to_bytes(range_for_block.length()), samples_from_this_block.size()); + if (wave_file.open(QFile::ReadWrite)) { + // We use S32 as a size-compatible substitute for SampleSummer::Sum which is 4 bytes in size + AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S32); + int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()); - if (destination_offset < 0 - || destination_offset + actual_copy_size > block_range_buffer.size()) { - qCritical() << "Tried to copy audio beyond the size of the destination buffer"; - qCritical() << " Destination size:" << block_range_buffer.size(); - qCritical() << " Destination offset:" << destination_offset; - qCritical() << " Copy size:" << actual_copy_size; - abort(); - } else { - memcpy(block_range_buffer.data()+destination_offset, - samples_from_this_block.data(), - actual_copy_size); - - // Save waveform to file - Block* src_block = static_cast(copy_map_->value(b)); - QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString()); - QDir waveform_loc = local_appdata_dir.filePath("waveform"); - waveform_loc.mkpath("."); - QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast(src_block)))); - wave_file.open(QFile::ReadWrite); - - // FIXME: Assumes 32-bit float - float* flt_samples = reinterpret_cast(samples_from_this_block.data()); - int nb_sample = samples_from_this_block.size() / sizeof(float); - - // FIXME: Hardcoded sample rate - // We use S16 here as a size-compatible substitute for qfloat16/half - AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S16); - int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()) * waveform_params.channel_count(); - - qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()) * 2; - qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()) * 2; + qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()); + qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()); qint64 end_offset = start_offset + length_offset; if (wave_file.size() < end_offset) { @@ -117,10 +104,10 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti wave_file.seek(start_offset); - for (int i=0;i summary = SampleSummer::SumSamples(&flt_samples[i], - qMin(chunk_size, nb_sample - i), - audio_params().channel_count()); + for (int i=0;isample_count_per_channel();i+=chunk_size) { + QVector summary = SampleSummer::SumSamples(samples_from_this_block, + i, + qMin(chunk_size, samples_from_this_block->sample_count_per_channel() - i)); wave_file.write(reinterpret_cast(summary.constData()), summary.size() * sizeof(SampleSummer::Sum)); @@ -137,7 +124,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti NodeValueTable::Merge({merged_table, table}); } - merged_table.Push(NodeParam::kSamples, block_range_buffer); + merged_table.Push(NodeParam::kSamples, QVariant::fromValue(block_range_buffer)); return merged_table; } diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index a2de88f3b..ea3c29993 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -317,9 +317,7 @@ void VideoRenderWorker::Download(const rational& time, QVariant texture, QString } else { FramePtr frame = Frame::Create(); - frame->set_width(video_params().width()); - frame->set_height(video_params().height()); - frame->set_format(video_params().format()); + frame->set_video_params(video_params()); frame->allocate(); TextureToBuffer(texture, frame->data()); diff --git a/app/render/pixelformat.cpp b/app/render/pixelformat.cpp index 9fa451c3b..21221a48b 100644 --- a/app/render/pixelformat.cpp +++ b/app/render/pixelformat.cpp @@ -237,10 +237,8 @@ FramePtr PixelFormat::ConvertPixelFormat(FramePtr frame, const PixelFormat::Form FramePtr converted = Frame::Create(); // Copy parameters - converted->set_width(frame->width()); - converted->set_height(frame->height()); + converted->set_video_params(frame->video_params()); converted->set_timestamp(frame->timestamp()); - converted->set_format(dest_format); converted->allocate(); OIIO::ImageBuf src(OIIO::ImageSpec(frame->width(), frame->height(), ChannelCount(frame->format()), GetOIIOTypeDesc(frame->format())), frame->data()); diff --git a/app/render/videoparams.cpp b/app/render/videoparams.cpp index b81fd6059..a71e80327 100644 --- a/app/render/videoparams.cpp +++ b/app/render/videoparams.cpp @@ -34,6 +34,12 @@ VideoRenderingParams::VideoRenderingParams() : { } +VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const PixelFormat::Format &format) : + VideoParams(width, height, rational()), + format_(format) +{ +} + VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const rational &time_base, const PixelFormat::Format &format, const RenderMode::Mode& mode, const int ÷r) : VideoParams(width, height, time_base), format_(format), diff --git a/app/render/videoparams.h b/app/render/videoparams.h index 68b6efb4e..2a593c783 100644 --- a/app/render/videoparams.h +++ b/app/render/videoparams.h @@ -25,6 +25,7 @@ private: class VideoRenderingParams : public VideoParams { public: VideoRenderingParams(); + VideoRenderingParams(const int& width, const int& height, const PixelFormat::Format& format); VideoRenderingParams(const int& width, const int& height, const rational& time_base, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1); VideoRenderingParams(const VideoParams& params, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1); diff --git a/app/widget/viewer/viewerglwidget.cpp b/app/widget/viewer/viewerglwidget.cpp index d89125d55..8dc05e027 100644 --- a/app/widget/viewer/viewerglwidget.cpp +++ b/app/widget/viewer/viewerglwidget.cpp @@ -101,9 +101,7 @@ void ViewerGLWidget::SetImage(const QString &fn) load_buffer_.destroy(); texture_.Destroy(); - load_buffer_.set_width(input->spec().width); - load_buffer_.set_height(input->spec().height); - load_buffer_.set_format(image_format); + load_buffer_.set_video_params(VideoRenderingParams(input->spec().width, input->spec().height, image_format)); load_buffer_.allocate(); texture_.Create(context(), input->spec().width, input->spec().height, image_format);