From b74ea1a8baadad79811599b242cffd54284a39d0 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 18 May 2020 12:52:25 +1000 Subject: [PATCH] renderer: reimported audio rendering functions --- app/node/output/viewer/viewer.cpp | 4 +- app/render/audioplaybackcache.cpp | 53 ++++++++++ app/render/audioplaybackcache.h | 4 +- app/render/backend/renderbackend.cpp | 146 +++++++++++++++++++++++++-- app/render/backend/renderbackend.h | 15 ++- app/render/backend/renderworker.cpp | 37 ++++++- app/render/backend/renderworker.h | 4 +- 7 files changed, 243 insertions(+), 20 deletions(-) diff --git a/app/node/output/viewer/viewer.cpp b/app/node/output/viewer/viewer.cpp index a13650fee..afbce0005 100644 --- a/app/node/output/viewer/viewer.cpp +++ b/app/node/output/viewer/viewer.cpp @@ -86,9 +86,9 @@ QString ViewerOutput::Description() const void ViewerOutput::InvalidateCache(const TimeRange &range, NodeInput *from, NodeInput *source) { - if (from == texture_input_ || from == samples_input_) { - emit GraphChangedFrom(source); + emit GraphChangedFrom(source); + if (from == texture_input_ || from == samples_input_) { TimeRange invalidated_range(qMax(rational(), range.in()), qMin(GetLength(), range.out())); diff --git a/app/render/audioplaybackcache.cpp b/app/render/audioplaybackcache.cpp index b684d11e3..4be298478 100644 --- a/app/render/audioplaybackcache.cpp +++ b/app/render/audioplaybackcache.cpp @@ -57,6 +57,59 @@ void AudioPlaybackCache::SetParameters(const AudioRenderingParams ¶ms) emit ParametersChanged(); } +void AudioPlaybackCache::WritePCM(const TimeRange &range, SampleBufferPtr samples) +{ + QFile f(filename_); + qDebug() << "Writing PCM to" << filename_; + if (f.open(QFile::ReadWrite)) { + qint64 start_offset = params_.time_to_bytes(range.in()); + qint64 max_len = params_.time_to_bytes(range.out()); + qint64 write_len = max_len - start_offset; + + QByteArray a = samples->toPackedData(); + + if (f.size() < max_len) { + f.resize(max_len); + } + + f.seek(start_offset); + f.write(a); + + if (write_len > a.size()) { + // Fill remaining space with silence + QByteArray s(write_len - a.size(), 0x00); + f.write(s); + } + + f.close(); + } else { + qWarning() << "Failed to write PCM data to" << filename_; + } +} + +void AudioPlaybackCache::WriteSilence(const TimeRange &range) +{ + QFile f(filename_); + if (f.open(QFile::ReadWrite)) { + qint64 start_offset = params_.time_to_bytes(range.in()); + qint64 max_len = params_.time_to_bytes(range.out()); + qint64 write_len = max_len - start_offset; + + if (f.size() < max_len) { + f.resize(max_len); + } + + f.seek(start_offset); + + QByteArray a(write_len, 0x00); + f.write(a); + + f.close(); + } else { + qWarning() << "Failed to write PCM data to" << filename_; + } +} + const QString &AudioPlaybackCache::GetCacheFilename() const { return filename_; diff --git a/app/render/audioplaybackcache.h b/app/render/audioplaybackcache.h index bffffb0aa..71c633af9 100644 --- a/app/render/audioplaybackcache.h +++ b/app/render/audioplaybackcache.h @@ -39,7 +39,9 @@ public: void SetParameters(const AudioRenderingParams& params); - void WritePCM(SampleBufferPtr samples); + void WritePCM(const TimeRange &range, SampleBufferPtr samples); + + void WriteSilence(const TimeRange &range); const QString& GetCacheFilename() const; diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index 21d44b95f..e2dcc7675 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -25,6 +25,8 @@ #include "config/config.h" #include "core.h" +#include "task/conform/conform.h" +#include "task/taskmanager.h" #include "window/mainwindow/mainwindow.h" OLIVE_NAMESPACE_ENTER @@ -37,7 +39,8 @@ RenderBackend::RenderBackend(QObject *parent) : divider_(1), render_mode_(RenderMode::kOnline), pix_fmt_(PixelFormat::PIX_FMT_RGBA32F), - sample_fmt_(SampleFormat::SAMPLE_FMT_FLT) + sample_fmt_(SampleFormat::SAMPLE_FMT_FLT), + ic_from_conform_(false) { // FIXME: Don't create in CLI mode cancel_dialog_ = new RenderCancelDialog(Core::instance()->main_window()); @@ -203,11 +206,11 @@ RenderWorker *RenderBackend::GetInstanceFromPool(QVector &worker instance->Init(viewer_node_); } - connect(instance, - &RenderWorker::FinishedJob, - this, - &RenderBackend::WorkerFinished, - Qt::QueuedConnection); + connect(instance, &RenderWorker::FinishedJob, + this, &RenderBackend::WorkerFinished, Qt::QueuedConnection); + + connect(instance, &RenderWorker::AudioConformUnavailable, + this, &RenderBackend::AudioConformUnavailable, Qt::QueuedConnection); } else { instance = worker_pool.at(instance_queuer % worker_pool.size()); instance_queuer++; @@ -217,11 +220,128 @@ RenderWorker *RenderBackend::GetInstanceFromPool(QVector &worker return instance; } +void RenderBackend::ListenForConformSignal(AudioStreamPtr s) +{ + foreach (const ConformWaitInfo& info, conform_wait_info_) { + if (info.stream == s) { + // We've probably already connected to this one + return; + } + } + + connect(s.get(), &AudioStream::ConformAppended, this, &RenderBackend::AudioConformUpdated); +} + +void RenderBackend::StopListeningForConformSignal(AudioStream *s) +{ + foreach (const ConformWaitInfo& info, conform_wait_info_) { + if (info.stream.get() == s) { + // There are still conforms we're waiting for, don't disconnect + return; + } + } + + disconnect(s, &AudioStream::ConformAppended, this, &RenderBackend::AudioConformUpdated); +} + +void RenderBackend::AudioConformUnavailable(StreamPtr stream, TimeRange range, rational stream_time, AudioRenderingParams params) +{ + ConformWaitInfo info = {stream, params, range, stream_time}; + + if (conform_wait_info_.contains(info)) { + return; + } + + AudioStreamPtr audio_stream = std::static_pointer_cast(stream); + + if (audio_stream->try_start_conforming(params)) { + + // Start indexing process + ListenForConformSignal(audio_stream); + + conform_wait_info_.append(info); + + ConformTask* conform_task = new ConformTask(audio_stream, params); + + TaskManager::instance()->AddTask(conform_task); + + } else if (audio_stream->has_conformed_version(params)) { + + // Conform JUST finished, requeue this time + ic_from_conform_ = true; + AudioInvalidated(range); + ic_from_conform_ = false; + + } else { + + // A conform task is already running, so we'll just wait for it + ListenForConformSignal(audio_stream); + + conform_wait_info_.append(info); + + } +} + +void RenderBackend::AudioConformUpdated(AudioRenderingParams params) +{ + AudioStream *stream = static_cast(sender()); + + for (int i=0;i r.in()) { + info.affected_range.set_out(r.in()); + } else if (info.affected_range.in() < r.out() && info.affected_range.out() > r.out()) { + info.affected_range.set_in(r.out()); + } + } + } + QFutureWatcher* watcher = new QFutureWatcher(); connect(watcher, &QFutureWatcher::finished, this, &RenderBackend::AudioRendered); + audio_jobs_.insert(watcher, r); + watcher->setFuture(QtConcurrent::run(&video_thread_pool_, GetInstanceFromPool(audio_instance_pool_, audio_thread_pool_, audio_instance_queuer_), &RenderWorker::RenderAudio, @@ -232,10 +352,16 @@ void RenderBackend::AudioRendered() { QFutureWatcher* watcher = static_cast*>(sender()); - if (watcher->result()) { - qDebug() << "AUDIO: Received" << watcher->result()->sample_count_per_channel() << "samples!"; - } else { - qDebug() << "AUDIO: Received null"; + if (audio_jobs_.contains(watcher)) { + TimeRange r = audio_jobs_.take(watcher); + + if (watcher->result()) { + qDebug() << "Received" << watcher->result()->sample_count_per_channel() << "samples"; + viewer_node_->audio_playback_cache()->WritePCM(r, watcher->result()); + } else { + qDebug() << "Received null"; + viewer_node_->audio_playback_cache()->WriteSilence(r); + } } watcher->deleteLater(); diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index 45b4c0bb5..fd3918e8a 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -104,18 +104,31 @@ private: // AUDIO MEMBERS SampleFormat::Format sample_fmt_; + QHash< QFutureWatcher*, TimeRange > audio_jobs_; struct ConformWaitInfo { StreamPtr stream; + AudioRenderingParams params; TimeRange affected_range; rational stream_time; bool operator==(const ConformWaitInfo& rhs) const; }; - QList footage_wait_info_; + QList conform_wait_info_; + + void ListenForConformSignal(AudioStreamPtr s); + + void StopListeningForConformSignal(AudioStream *s); + + bool ic_from_conform_; private slots: + void AudioConformUnavailable(StreamPtr stream, TimeRange range, + rational stream_time, AudioRenderingParams params); + + void AudioConformUpdated(OLIVE_NAMESPACE::AudioRenderingParams params); + void AudioInvalidated(const TimeRange &r); void AudioRendered(); diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index f40636de5..ba81d8c96 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -109,6 +109,15 @@ SampleBufferPtr RenderWorker::RenderAudio(const TimeRange &range) UpdateData(true); + audio_render_time_ = range; + + qDebug() << "Audio deps:"; + + QList d = viewer_->samples_input()->GetDependencies(); + foreach (Node* n, d) { + qDebug() << " " << n; + } + NodeValueTable table = ProcessInput(viewer_->samples_input(), range); QVariant samples = table.Get(NodeParam::kSamples); @@ -132,12 +141,8 @@ void RenderWorker::UpdateData(bool block_for_update) NodeValueTable RenderWorker::GenerateBlockTable(const TrackOutput *track, const TimeRange &range) { - qDebug() << "Hello from track" << track << "type" << track->track_type(); - if (track->track_type() == Timeline::kTrackTypeAudio) { - qDebug() << "Hello?"; - QList active_blocks = track->BlocksAtTimeRange(range); // All these blocks will need to output to a buffer so we create one here @@ -309,6 +314,8 @@ void RenderWorker::ProcessNodeEvent(const Node *node, const TimeRange &range, No void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable *table) { + qDebug() << "Stream type" << stream->type(); + if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) { ImageStreamPtr video_stream = std::static_pointer_cast(stream); @@ -348,6 +355,7 @@ void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &inp } else if (stream->type() == Stream::kAudio) { + qDebug() << "Hello!"; table->Push(GetDataFromStream(stream, input_time)); } @@ -358,7 +366,25 @@ NodeValue RenderWorker::GetDataFromStream(StreamPtr stream, const TimeRange &inp DecoderPtr decoder = ResolveDecoderFromInput(stream); if (decoder) { - return FrameToTexture(decoder, stream, input_time); + if (stream->type() == Stream::kVideo || stream->type() == Stream::kImage) { + return FrameToTexture(decoder, stream, input_time); + } else if (stream->type() == Stream::kAudio) { + qDebug() << "Decoding audio!"; + if (decoder->HasConformedVersion(audio_params())) { + qDebug() << " Retrieving audio!"; + SampleBufferPtr frame = decoder->RetrieveAudio(input_time.in(), input_time.length(), + audio_params()); + + if (frame) { + qDebug() << " Returning audio!"; + return NodeValue(NodeParam::kSamples, QVariant::fromValue(frame)); + } + } else { + qDebug() << " Conform doesn't exist! AAAAA"; + emit AudioConformUnavailable(decoder->stream(), audio_render_time_, + input_time.out(), audio_params()); + } + } } return NodeValue(); @@ -498,6 +524,7 @@ void RenderWorker::Init(ViewerOutput* viewer) Queue(viewer->texture_input()); Queue(viewer->samples_input()); + Queue(viewer->track_input(Timeline::kTrackTypeAudio)); ProcessQueue(); } diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index 4461ca48f..bd70c5678 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -124,7 +124,8 @@ protected: } signals: - void AudioConformUnavailable(); + void AudioConformUnavailable(StreamPtr stream, TimeRange range, + rational stream_time, AudioRenderingParams params); void FinishedJob(); @@ -160,6 +161,7 @@ private: DecoderCache decoder_cache_; ViewerOutput* viewer_; + TimeRange audio_render_time_; QList queued_updates_; QHash copy_map_; bool available_;