From a9660201f18db8d1766485ad420e41118f373856 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Fri, 15 Nov 2019 13:57:49 +0900 Subject: [PATCH] miscellaneous other changes to make audio rendering work Many changes were made throughout the codebase to support audio, these are most of the small changes necessary. The audio support still is not perfect. I still need to write in resampling support. After that it should work correctly with all audio types. --- app/audio/CMakeLists.txt | 1 + app/audio/audiohybriddevice.cpp | 6 +- app/audio/sampleformat.cpp | 1 + app/audio/sampleformat.h | 7 - app/decoder/wave.cpp | 143 ------------------ app/decoder/wave.h | 39 ----- app/node/block/block.cpp | 2 + app/node/input/media/audio/audio.cpp | 25 ++- app/node/input/media/audio/audio.h | 2 + app/node/input/media/video/video.h | 1 - app/node/output.h | 2 +- app/project/item/sequence/sequence.cpp | 1 + .../playbackcontrols/playbackcontrols.cpp | 22 ++- .../playbackcontrols/playbackcontrols.h | 4 + app/widget/timelinewidget/tool/import.cpp | 47 +++--- 15 files changed, 70 insertions(+), 233 deletions(-) delete mode 100644 app/decoder/wave.cpp delete mode 100644 app/decoder/wave.h diff --git a/app/audio/CMakeLists.txt b/app/audio/CMakeLists.txt index 4feb7d947..419f30cdb 100644 --- a/app/audio/CMakeLists.txt +++ b/app/audio/CMakeLists.txt @@ -21,5 +21,6 @@ set(OLIVE_SOURCES audio/audiomanager.h audio/audiomanager.cpp audio/sampleformat.h + audio/sampleformat.cpp PARENT_SCOPE ) diff --git a/app/audio/audiohybriddevice.cpp b/app/audio/audiohybriddevice.cpp index c4a942801..e9925b3f7 100644 --- a/app/audio/audiohybriddevice.cpp +++ b/app/audio/audiohybriddevice.cpp @@ -48,7 +48,11 @@ void AudioHybridDevice::Stop() { // Whatever is happening, stop it pushed_samples_.clear(); - device_ = nullptr; + + if (device_ != nullptr) { + device_->close(); + device_ = nullptr; + } } void AudioHybridDevice::ConnectDevice(QIODevice *device) diff --git a/app/audio/sampleformat.cpp b/app/audio/sampleformat.cpp index e69de29bb..08fff1777 100644 --- a/app/audio/sampleformat.cpp +++ b/app/audio/sampleformat.cpp @@ -0,0 +1 @@ +#include "sampleformat.h" diff --git a/app/audio/sampleformat.h b/app/audio/sampleformat.h index f19b8e7e8..c87ca4d39 100644 --- a/app/audio/sampleformat.h +++ b/app/audio/sampleformat.h @@ -21,11 +21,6 @@ #ifndef SAMPLEFORMAT_H #define SAMPLEFORMAT_H -namespace olive { - -/** - * @brief Olive's internal supported sample formats - */ enum SampleFormat { SAMPLE_FMT_INVALID = -1, @@ -39,6 +34,4 @@ enum SampleFormat { SAMPLE_FMT_COUNT }; -} - #endif // SAMPLEFORMAT_H diff --git a/app/decoder/wave.cpp b/app/decoder/wave.cpp deleted file mode 100644 index 3310e1a26..000000000 --- a/app/decoder/wave.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "wave.h" - -const int16_t kWAVIntegerFormat = 1; -const int16_t kWAVFloatFormat = 3; - -WaveOutput::WaveOutput(const QString &f, - const AudioRenderingParams& params) : - file_(f), - params_(params) -{ -} - -WaveOutput::~WaveOutput() -{ - close(); -} - -bool WaveOutput::open() -{ - data_length_ = 0; - - if (file_.open(QFile::WriteOnly)) { - // RIFF header - file_.write("RIFF"); - - // Total file size minus RIFF and this integer (minus 8 bytes, filled in later) - write_int(&file_, 0); - - // File type header - file_.write("WAVE"); - - // Begin format descriptor chunk - file_.write("fmt "); - - // Format chunk size - write_int(&file_, 16); - - // Type of format - switch (params_.format()) { - case olive::SAMPLE_FMT_U8: - case olive::SAMPLE_FMT_S16: - case olive::SAMPLE_FMT_S32: - case olive::SAMPLE_FMT_S64: - write_int(&file_, kWAVIntegerFormat); - break; - case olive::SAMPLE_FMT_FLT: - case olive::SAMPLE_FMT_DBL: - write_int(&file_, kWAVFloatFormat); - break; - case olive::SAMPLE_FMT_INVALID: - case olive::SAMPLE_FMT_COUNT: - qWarning() << "Invalid sample format for WAVE audio"; - file_.close(); - return false; - } - - // Number of channels - write_int(&file_, static_cast(params_.channel_count())); - - // Sample rate - write_int(&file_, params_.sample_rate()); - - // Bytes per second - write_int(&file_, params_.samples_to_bytes(params_.sample_rate())); - - // Bytes per sample - write_int(&file_, static_cast(params_.samples_to_bytes(1))); - - // Bits per sample per channel - write_int(&file_, static_cast(params_.bits_per_sample())); - - // Data chunk header - file_.write("data"); - - // Size of data chunk (filled in later) - write_int(&file_, 0); - - return true; - } - - return false; -} - -void WaveOutput::write(const QByteArray &bytes) -{ - if (file_.isOpen()) { - file_.write(bytes); - - data_length_ += bytes.size(); - } -} - -void WaveOutput::write(const char *bytes, int length) -{ - if (file_.isOpen()) { - file_.write(bytes, length); - - data_length_ += length; - } -} - -void WaveOutput::close() -{ - if (file_.isOpen()) { - - // Write file sizes - file_.seek(4); - write_int(&file_, data_length_ + 36); - - file_.seek(40); - write_int(&file_, data_length_); - - file_.close(); - } -} - -void WaveOutput::switch_endianness(QByteArray& array) -{ - int half_sz = array.size()/2; - - for (int i=0;i -void WaveOutput::write_int(QFile *file, T integer) -{ - QByteArray bytes; - bytes.resize(sizeof(T)); - memcpy(bytes.data(), &integer, static_cast(bytes.size())); - - // WAV expects little-endian, so if the integer is big endian we need to switch - if (QSysInfo::ByteOrder == QSysInfo::BigEndian) { - switch_endianness(bytes); - } - - file->write(bytes); -} diff --git a/app/decoder/wave.h b/app/decoder/wave.h deleted file mode 100644 index 3f0299846..000000000 --- a/app/decoder/wave.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef WAVEAUDIO_H -#define WAVEAUDIO_H - -#include -#include - -#include "audio/sampleformat.h" -#include "render/audioparams.h" - -class WaveOutput -{ -public: - WaveOutput(const QString& f, - const AudioRenderingParams& params); - - ~WaveOutput(); - - bool open(); - - void write(const QByteArray& bytes); - void write(const char* bytes, int length); - - void close(); - -private: - template - void write_int(QFile* file, T integer); - - void switch_endianness(QByteArray &array); - - QFile file_; - - AudioRenderingParams params_; - - int data_length_; - -}; - -#endif // WAVEAUDIO_H diff --git a/app/node/block/block.cpp b/app/node/block/block.cpp index 023359d0c..47b2fc46f 100644 --- a/app/node/block/block.cpp +++ b/app/node/block/block.cpp @@ -110,6 +110,7 @@ QVariant Block::Value(NodeOutput *output) void Block::EdgeAddedSlot(NodeEdgePtr edge) { if (edge->input() == previous_input()) { + // FIXME: No protection for if this connection is not a node static_cast(edge->output()->parent())->next_ = this; // The blocks surrounding this one have changed, we need to Refresh() @@ -123,6 +124,7 @@ void Block::EdgeAddedSlot(NodeEdgePtr edge) void Block::EdgeRemovedSlot(NodeEdgePtr edge) { if (edge->input() == previous_input()) { + // FIXME: No protection for if this connection is not a node static_cast(edge->output()->parent())->next_ = nullptr; // The blocks surrounding this one have changed, we need to Refresh() diff --git a/app/node/input/media/audio/audio.cpp b/app/node/input/media/audio/audio.cpp index f3fde6228..6c9f301fe 100644 --- a/app/node/input/media/audio/audio.cpp +++ b/app/node/input/media/audio/audio.cpp @@ -26,22 +26,17 @@ QString AudioInput::Description() return tr("Import an audio footage stream."); } -QVariant AudioInput::Value(NodeOutput *) +NodeOutput *AudioInput::samples_output() { - /*if (output == samples_output_) { - // Make sure decoder is set up - if (!SetupDecoder()) { - return 0; - } + return samples_output_; +} - // Retrieve audio samples from decoder - frame_ = decoder_->Retrieve(in, out - in); +QVariant AudioInput::Value(NodeOutput *output) +{ + if (output == samples_output_) { + // Simple passthrough from footage input + return footage_input_->value(); + } - QByteArray samples; - samples.resize(frame_->audio_params().samples_to_bytes(frame_->sample_count())); - memcpy(samples.data(), frame_->data(), static_cast(samples.size())); - return samples; - }*/ - - return 0; + return MediaInput::Value(output); } diff --git a/app/node/input/media/audio/audio.h b/app/node/input/media/audio/audio.h index 5ddc957a2..453be8bc2 100644 --- a/app/node/input/media/audio/audio.h +++ b/app/node/input/media/audio/audio.h @@ -13,6 +13,8 @@ public: virtual QString Category() override; virtual QString Description() override; + NodeOutput* samples_output(); + protected: virtual QVariant Value(NodeOutput* output) override; diff --git a/app/node/input/media/video/video.h b/app/node/input/media/video/video.h index 02da4f324..d96094692 100644 --- a/app/node/input/media/video/video.h +++ b/app/node/input/media/video/video.h @@ -27,7 +27,6 @@ public: //virtual void Hash(QCryptographicHash *hash, NodeOutput* from, const rational &time) override; protected: - //virtual QVariant Value(NodeOutput* output, const rational& in, const rational& out) override; private: NodeInput* matrix_input_; diff --git a/app/node/output.h b/app/node/output.h index 7d6ffa3ee..04a5aaadb 100644 --- a/app/node/output.h +++ b/app/node/output.h @@ -48,7 +48,7 @@ public: void drop_cached_values(); private: - QMap cached_values_; + QHash cached_values_; }; diff --git a/app/project/item/sequence/sequence.cpp b/app/project/item/sequence/sequence.cpp index 3735192ba..1f2f6f607 100644 --- a/app/project/item/sequence/sequence.cpp +++ b/app/project/item/sequence/sequence.cpp @@ -85,6 +85,7 @@ void Sequence::add_default_nodes() // Update the timebase on these nodes set_video_params(video_params_); + set_audio_params(audio_params_); } Item::Type Sequence::type() const diff --git a/app/widget/playbackcontrols/playbackcontrols.cpp b/app/widget/playbackcontrols/playbackcontrols.cpp index b5e8af2d4..051bbe44d 100644 --- a/app/widget/playbackcontrols/playbackcontrols.cpp +++ b/app/widget/playbackcontrols/playbackcontrols.cpp @@ -132,13 +132,12 @@ void PlaybackControls::SetTimebase(const rational &r) void PlaybackControls::SetTime(const int64_t &r) { - if (time_base_.isNull()) { - return; - } + SetTimeLabelInternal(cur_tc_lbl_, r); +} - cur_tc_lbl_->setText(olive::timestamp_to_timecode(r, - time_base_, - olive::CurrentTimecodeDisplay())); +void PlaybackControls::SetEndTime(const int64_t &r) +{ + SetTimeLabelInternal(end_tc_lbl_, r); } void PlaybackControls::ShowPauseButton() @@ -170,3 +169,14 @@ void PlaybackControls::UpdateIcons() next_frame_btn_->setIcon(olive::icon::NextFrame); go_to_end_btn_->setIcon(olive::icon::GoToEnd); } + +void PlaybackControls::SetTimeLabelInternal(QLabel* label, const int64_t& time) +{ + if (time_base_.isNull()) { + return; + } + + label->setText(olive::timestamp_to_timecode(time, + time_base_, + olive::CurrentTimecodeDisplay())); +} diff --git a/app/widget/playbackcontrols/playbackcontrols.h b/app/widget/playbackcontrols/playbackcontrols.h index 8e36a8af8..5464b4847 100644 --- a/app/widget/playbackcontrols/playbackcontrols.h +++ b/app/widget/playbackcontrols/playbackcontrols.h @@ -49,6 +49,8 @@ public: public slots: void SetTime(const int64_t &r); + void SetEndTime(const int64_t &r); + void ShowPauseButton(); void ShowPlayButton(); @@ -90,6 +92,8 @@ protected: private: void UpdateIcons(); + void SetTimeLabelInternal(QLabel *label, const int64_t &time); + QWidget* lower_left_container_; QWidget* lower_right_container_; diff --git a/app/widget/timelinewidget/tool/import.cpp b/app/widget/timelinewidget/tool/import.cpp index 5c5656e22..c6902885f 100644 --- a/app/widget/timelinewidget/tool/import.cpp +++ b/app/widget/timelinewidget/tool/import.cpp @@ -28,6 +28,7 @@ #include "core.h" #include "node/distort/transform/transform.h" #include "node/color/opacity/opacity.h" +#include "node/input/media/audio/audio.h" #include "node/input/media/video/video.h" TrackType TrackTypeFromStreamType(Stream::Type stream_type) @@ -205,10 +206,6 @@ void TimelineWidget::ImportTool::DragLeave(QDragLeaveEvent* event) void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event) { if (parent()->HasGhosts()) { - // We use QObject as the parent for the nodes we create. If there is no TimelineOutput node, this object going out - // of scope will delete the nodes. If there is, they'll become parents of the NodeGraph instead - QObject node_memory_manager; - QUndoCommand* command = new QUndoCommand(); QVector block_items(parent()->ghost_items_.size()); @@ -216,27 +213,37 @@ void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event) for (int i=0;ighost_items_.size();i++) { TimelineViewGhostItem* ghost = parent()->ghost_items_.at(i); - ClipBlock* clip = new ClipBlock(); - VideoInput* media = new VideoInput(); - //TransformDistort* transform = new TransformDistort(); - //OpacityNode* opacity = new OpacityNode(); - - // Set parents to node_memory_manager in case no TimelineOutput receives this signal - clip->setParent(&node_memory_manager); - media->setParent(&node_memory_manager); - //transform->setParent(&node_memory_manager); - //opacity->setParent(&node_memory_manager); - StreamPtr footage_stream = ghost->data(TimelineViewGhostItem::kAttachedFootage).value(); - media->SetFootage(footage_stream); + ClipBlock* clip = new ClipBlock(); clip->set_length(ghost->Length()); clip->set_block_name(footage_stream->footage()->name()); - //NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input()); - //NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input()); - //NodeParam::ConnectEdge(transform->matrix_output(), media->matrix_input()); - NodeParam::ConnectEdge(media->texture_output(), clip->texture_input()); + switch (footage_stream->type()) { + case Stream::kVideo: + { + VideoInput* video_input = new VideoInput(); + video_input->SetFootage(footage_stream); + NodeParam::ConnectEdge(video_input->texture_output(), clip->texture_input()); + + TransformDistort* transform = new TransformDistort(); + NodeParam::ConnectEdge(transform->matrix_output(), video_input->matrix_input()); + + //OpacityNode* opacity = new OpacityNode(); + //NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input()); + //NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input()); + break; + } + case Stream::kAudio: + { + AudioInput* audio_input = new AudioInput(); + audio_input->SetFootage(footage_stream); + NodeParam::ConnectEdge(audio_input->samples_output(), clip->texture_input()); + break; + } + default: + break; + } if (event->GetModifiers() & Qt::ControlModifier) { //emit parent()->RequestInsertBlockAtTime(clip, ghost->GetAdjustedIn());