From e68e89c40b9b50abefc6f8dfe43c8265f04b952a Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Wed, 19 Feb 2020 16:45:34 +1100 Subject: [PATCH] renderer/decoder: use same background mechanism to conform audio as well --- app/codec/decoder.cpp | 32 ++++++++++- app/codec/decoder.h | 5 ++ app/project/item/footage/audiostream.cpp | 24 +++++++++ app/project/item/footage/audiostream.h | 12 +++++ app/render/backend/audio/audiobackend.cpp | 2 + app/render/backend/audiorenderbackend.cpp | 65 +++++++++++++++++++++++ app/render/backend/audiorenderbackend.h | 18 +++++++ app/render/backend/audiorenderworker.cpp | 7 ++- app/render/backend/audiorenderworker.h | 3 ++ app/render/backend/renderbackend.cpp | 15 +++--- app/render/backend/renderbackend.h | 2 + app/render/indexmanager.cpp | 44 ++++++++++++--- app/render/indexmanager.h | 20 +++++-- app/task/conform/conform.cpp | 1 + app/task/index/index.cpp | 6 +++ app/task/taskmanager.cpp | 1 + 16 files changed, 239 insertions(+), 18 deletions(-) diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index 1bb15c657..69ec84038 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -180,8 +180,6 @@ void Decoder::Conform(const AudioRenderingParams ¶ms, const QAtomicInt* canc return; } - Index(cancelled); - // Get indexed WAV file WaveInput input(GetIndexFilename()); @@ -196,12 +194,14 @@ void Decoder::Conform(const AudioRenderingParams ¶ms, const QAtomicInt* canc } // Otherwise, let's start converting the format + QMutexLocker locker(stream()->index_process_lock()); // Generate destination filename for this conversion to see if it exists QString conformed_fn = GetConformedFilename(params); if (QFileInfo::exists(conformed_fn)) { // We must have already conformed this format + std::static_pointer_cast(stream())->append_conformed_version(params); input.close(); return; } @@ -259,6 +259,8 @@ void Decoder::Conform(const AudioRenderingParams ¶ms, const QAtomicInt* canc // If we cancelled, the conform didn't finish so remove it if (cancelled && *cancelled) { QFile(conformed_fn).remove(); + } else { + std::static_pointer_cast(stream())->append_conformed_version(params); } } else { qWarning() << "Failed to conform file:" << stream()->footage()->filename(); @@ -321,6 +323,32 @@ void Decoder::Index(const QAtomicInt *) { } +bool Decoder::HasConformedVersion(const AudioRenderingParams ¶ms) +{ + if (stream()->type() != Stream::kAudio) { + return false; + } + + AudioStreamPtr audio_stream = std::static_pointer_cast(stream()); + + if (audio_stream->has_conformed_version(params)) { + return true; + } + + // Get indexed WAV file + WaveInput input(GetIndexFilename()); + + bool index_already_matches = false; + + if (input.open()) { + index_already_matches = (input.params() == params); + + input.close(); + } + + return index_already_matches; +} + void Decoder::SignalIndexProgress(const int64_t &ts) { if (stream()->duration() != AV_NOPTS_VALUE && stream()->duration() != 0) { diff --git a/app/codec/decoder.h b/app/codec/decoder.h index 9a6d45ce8..22635c0f6 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -237,6 +237,11 @@ public: */ virtual void Index(const QAtomicInt* cancelled); + /** + * @brief AUDIO ONLY: Returns whether a cached transcode of this audio matching the specified params already exists + */ + bool HasConformedVersion(const AudioRenderingParams& params); + signals: /** * @brief While indexing, this signal will provide progress as a percentage (0-100 inclusive) if available diff --git a/app/project/item/footage/audiostream.cpp b/app/project/item/footage/audiostream.cpp index 6b9e20e9d..14369f983 100644 --- a/app/project/item/footage/audiostream.cpp +++ b/app/project/item/footage/audiostream.cpp @@ -106,3 +106,27 @@ void AudioStream::clear_index() index_done_ = false; index_length_ = 0; } + +bool AudioStream::has_conformed_version(const AudioRenderingParams ¶ms) +{ + QMutexLocker locker(&index_access_lock_); + + foreach (const AudioRenderingParams& p, conformed_) { + if (p == params) { + return true; + } + } + + return false; +} + +void AudioStream::append_conformed_version(const AudioRenderingParams ¶ms) +{ + { + QMutexLocker locker(&index_access_lock_); + + conformed_.append(params); + } + + emit ConformAppended(params); +} diff --git a/app/project/item/footage/audiostream.h b/app/project/item/footage/audiostream.h index 13e8f6202..f46e76746 100644 --- a/app/project/item/footage/audiostream.h +++ b/app/project/item/footage/audiostream.h @@ -21,7 +21,10 @@ #ifndef AUDIOSTREAM_H #define AUDIOSTREAM_H +#include + #include "common/rational.h" +#include "render/audioparams.h" #include "stream.h" /** @@ -29,6 +32,7 @@ */ class AudioStream : public Stream { + Q_OBJECT public: AudioStream(); @@ -51,6 +55,12 @@ public: void clear_index(); + bool has_conformed_version(const AudioRenderingParams& params); + void append_conformed_version(const AudioRenderingParams& params); + +signals: + void ConformAppended(const AudioRenderingParams& params); + private: int channels_; uint64_t layout_; @@ -60,6 +70,8 @@ private: rational index_length_; bool index_done_; + QVector conformed_; + }; using AudioStreamPtr = std::shared_ptr; diff --git a/app/render/backend/audio/audiobackend.cpp b/app/render/backend/audio/audiobackend.cpp index 9d4c3eb27..11e859ed5 100644 --- a/app/render/backend/audio/audiobackend.cpp +++ b/app/render/backend/audio/audiobackend.cpp @@ -49,6 +49,8 @@ void AudioBackend::DecompileInternal() void AudioBackend::ConnectWorkerToThis(RenderWorker *worker) { + AudioRenderBackend::ConnectWorkerToThis(worker); + connect(worker, &RenderWorker::CompletedCache, this, &AudioBackend::ThreadCompletedCache); } diff --git a/app/render/backend/audiorenderbackend.cpp b/app/render/backend/audiorenderbackend.cpp index 7765e1902..7ebee611e 100644 --- a/app/render/backend/audiorenderbackend.cpp +++ b/app/render/backend/audiorenderbackend.cpp @@ -5,10 +5,12 @@ #include "audiorenderworker.h" #include "common/filefunctions.h" +#include "render/indexmanager.h" AudioRenderBackend::AudioRenderBackend(QObject *parent) : RenderBackend(parent) { + connect(IndexManager::instance(), &IndexManager::StreamConformAppended, this, &AudioRenderBackend::ConformUpdated); } void AudioRenderBackend::SetParameters(const AudioRenderingParams ¶ms) @@ -74,3 +76,66 @@ bool AudioRenderBackend::CanRender() { return params_.is_valid(); } + +void AudioRenderBackend::ConnectWorkerToThis(RenderWorker *worker) +{ + AudioRenderWorker* arw = static_cast(worker); + + connect(arw, &AudioRenderWorker::ConformUnavailable, this, &AudioRenderBackend::ConformUnavailable, Qt::QueuedConnection); +} + +void AudioRenderBackend::ConformUnavailable(StreamPtr stream, const TimeRange &range, const rational &stream_time, const AudioRenderingParams& params) +{ + ConformWaitInfo info = {stream, params, range, stream_time}; + + if (conform_wait_info_.contains(info)) { + return; + } + + qDebug() << "Waiting for conformed" << stream.get() << "time" << stream_time.toDouble() << "for frame" << range.in(); + + AudioStreamPtr audio_stream = std::static_pointer_cast(stream); + + if (IndexManager::instance()->IsConforming(audio_stream, params)) { + + conform_wait_info_.append(info); + + } else if (audio_stream->has_conformed_version(params)) { + + // Index JUST finished, requeue this time + InvalidateCache(range); + + } else { + + // Start indexing process + conform_wait_info_.append(info); + IndexManager::instance()->StartConformingStream(audio_stream, params); + + } +} + +void AudioRenderBackend::ConformUpdated(Stream *stream, const AudioRenderingParams ¶ms) +{ + qDebug() << "Got conform updated in ARB"; + + for (int i=0;i conform_wait_info_; + AudioRenderingParams params_; +private slots: + void ConformUnavailable(StreamPtr stream, const TimeRange& range, const rational& stream_time, const AudioRenderingParams ¶ms); + + void ConformUpdated(Stream *stream, const AudioRenderingParams& params); + }; #endif // AUDIORENDERBACKEND_H diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index 53060cf8b..c480b7930 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -25,7 +25,12 @@ void AudioRenderWorker::CloseInternal() FramePtr AudioRenderWorker::RetrieveFromDecoder(DecoderPtr decoder, const TimeRange &range) { - return decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params_); + if (decoder->HasConformedVersion(audio_params_)) { + return decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params_); + } else { + emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params_); + return nullptr; + } } NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const TimeRange &range) diff --git a/app/render/backend/audiorenderworker.h b/app/render/backend/audiorenderworker.h index 67e345ac2..04aa7218e 100644 --- a/app/render/backend/audiorenderworker.h +++ b/app/render/backend/audiorenderworker.h @@ -11,6 +11,9 @@ public: void SetParameters(const AudioRenderingParams& audio_params); +signals: + void ConformUnavailable(StreamPtr stream, const TimeRange& range, const rational& stream_time, const AudioRenderingParams& params); + protected: virtual bool InitInternal() override; diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index e852b80c4..a5d3458b7 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -443,12 +443,8 @@ void RenderBackend::FootageUnavailable(StreamPtr stream, Decoder::RetrieveState FootageWaitInfo info = {stream, range, stream_time}; - foreach (const FootageWaitInfo& compare, footage_wait_info_) { - if (info.stream == compare.stream - && info.stream_time == compare.stream_time - && info.affected_range == compare.affected_range) { - return; - } + if (footage_wait_info_.contains(info)) { + return; } qDebug() << "Waiting for" << stream.get() << "time" << stream_time.toDouble() << "for frame" << range.in(); @@ -513,3 +509,10 @@ void RenderBackend::IndexUpdated(Stream* stream) } } } + +bool RenderBackend::FootageWaitInfo::operator==(const RenderBackend::FootageWaitInfo &rhs) const +{ + return rhs.stream == stream + && rhs.stream_time == stream_time + && rhs.affected_range == affected_range; +} diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index 9143d2f26..9b44c58b4 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -149,6 +149,8 @@ private: StreamPtr stream; TimeRange affected_range; rational stream_time; + + bool operator==(const FootageWaitInfo& rhs) const; }; QList footage_wait_info_; diff --git a/app/render/indexmanager.cpp b/app/render/indexmanager.cpp index 8dfda980e..c73787154 100644 --- a/app/render/indexmanager.cpp +++ b/app/render/indexmanager.cpp @@ -31,7 +31,7 @@ void IndexManager::StartIndexingStream(StreamPtr stream) } IndexTask* index_task = new IndexTask(stream); - threads_.append({stream, index_task}); + indexing_.append({stream, index_task}); connect(stream.get(), &Stream::IndexChanged, this, &IndexManager::StreamIndexUpdatedEvent, Qt::QueuedConnection); connect(index_task, &IndexTask::Succeeded, this, &IndexManager::IndexTaskFinished, Qt::QueuedConnection); @@ -39,9 +39,24 @@ void IndexManager::StartIndexingStream(StreamPtr stream) TaskManager::instance()->AddTask(index_task); } -bool IndexManager::IsIndexing(StreamPtr stream) +void IndexManager::StartConformingStream(AudioStreamPtr stream, const AudioRenderingParams ¶ms) { - foreach (const StreamThreadPair& stp, threads_) { + if (IsConforming(stream, params)) { + return; + } + + ConformTask* conform_task = new ConformTask(stream, params); + conforming_.append({stream, params, conform_task}); + + connect(stream.get(), &AudioStream::ConformAppended, this, &IndexManager::StreamConformAppendedEvent, Qt::QueuedConnection); + connect(conform_task, &ConformTask::Succeeded, this, &IndexManager::IndexTaskFinished, Qt::QueuedConnection); + + TaskManager::instance()->AddTask(conform_task); +} + +bool IndexManager::IsIndexing(StreamPtr stream) const +{ + foreach (const IndexPair& stp, indexing_) { if (stp.stream == stream) { return true; } @@ -50,14 +65,25 @@ bool IndexManager::IsIndexing(StreamPtr stream) return false; } +bool IndexManager::IsConforming(AudioStreamPtr stream, const AudioRenderingParams ¶ms) const +{ + foreach (const ConformPair& cfp, conforming_) { + if (cfp.stream == stream && cfp.params == params) { + return true; + } + } + + return false; +} + void IndexManager::IndexTaskFinished() { - for (int i=0;i(sender())); } + +void IndexManager::StreamConformAppendedEvent(const AudioRenderingParams ¶ms) +{ + qDebug() << "Got stream conform appended event"; + emit StreamConformAppended(static_cast(sender()), params); +} diff --git a/app/render/indexmanager.h b/app/render/indexmanager.h index 9ef4c1967..e94cf0862 100644 --- a/app/render/indexmanager.h +++ b/app/render/indexmanager.h @@ -4,6 +4,7 @@ #include #include "project/item/footage/stream.h" +#include "task/conform/conform.h" #include "task/index/index.h" class IndexManager : public QObject @@ -16,29 +17,42 @@ public: static IndexManager* instance(); static void DestroyInstance(); - bool IsIndexing(StreamPtr stream); + bool IsIndexing(StreamPtr stream) const; + bool IsConforming(AudioStreamPtr stream, const AudioRenderingParams& params) const; public slots: void StartIndexingStream(StreamPtr stream); + void StartConformingStream(AudioStreamPtr stream, const AudioRenderingParams& params); signals: void StreamIndexUpdated(Stream* stream); + void StreamConformAppended(Stream* stream, const AudioRenderingParams& params); private: static IndexManager* instance_; - struct StreamThreadPair { + struct IndexPair { StreamPtr stream; IndexTask* task; }; - QList threads_; + struct ConformPair { + StreamPtr stream; + AudioRenderingParams params; + ConformTask* task; + }; + + QList indexing_; + + QList conforming_; private slots: void IndexTaskFinished(); void StreamIndexUpdatedEvent(); + void StreamConformAppendedEvent(const AudioRenderingParams& params); + }; #endif // INDEXMANAGER_H diff --git a/app/task/conform/conform.cpp b/app/task/conform/conform.cpp index 24a8e17c8..a467398f5 100644 --- a/app/task/conform/conform.cpp +++ b/app/task/conform/conform.cpp @@ -6,6 +6,7 @@ ConformTask::ConformTask(AudioStreamPtr stream, const AudioRenderingParams& para stream_(stream), params_(params) { + SetTitle(tr("Conforming Audio %1:%2").arg(stream_->footage()->filename(), QString::number(stream_->index()))); } void ConformTask::Action() diff --git a/app/task/index/index.cpp b/app/task/index/index.cpp index aceb322b6..411f29c0e 100644 --- a/app/task/index/index.cpp +++ b/app/task/index/index.cpp @@ -1,6 +1,7 @@ #include "index.h" #include "codec/decoder.h" +#include "codec/ffmpeg/ffmpegdecoder.h" IndexTask::IndexTask(StreamPtr stream) : stream_(stream) @@ -17,6 +18,11 @@ void IndexTask::Action() decoder->set_stream(stream_); + // Force multithreading for faster indexing + if (decoder->id() == "ffmpeg") { + static_cast(decoder.get())->SetMultithreading(true); + } + connect(decoder.get(), &Decoder::IndexProgress, this, &IndexTask::ProgressChanged); decoder->Open(); diff --git a/app/task/taskmanager.cpp b/app/task/taskmanager.cpp index 5a84a1ce3..024d5f18f 100644 --- a/app/task/taskmanager.cpp +++ b/app/task/taskmanager.cpp @@ -164,6 +164,7 @@ void TaskManager::DeleteTask(Task *t) break; } } + emit t->Removed(); if (GetTaskStatus(t) != kWorking) {