From a1345166c79d66796c3122b1a05c709b530febb1 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Fri, 14 May 2021 12:20:53 +1000 Subject: [PATCH] renderer: move conform to task Improved rendering system and user interface Fixes #1375 --- app/codec/CMakeLists.txt | 18 +++---- app/codec/decoder.cpp | 82 ++++++-------------------------- app/codec/decoder.h | 39 ++++++++------- app/core.cpp | 6 +++ app/render/previewautocacher.cpp | 82 +++++++++++++++++++++++--------- app/render/previewautocacher.h | 6 +++ app/render/rendermanager.cpp | 7 +-- app/render/rendermanager.h | 4 +- app/render/renderprocessor.cpp | 14 +++--- app/task/conform/conform.cpp | 37 ++++++-------- app/task/conform/conform.h | 9 ++-- app/task/render/render.cpp | 2 +- app/task/task.h | 9 ++++ 13 files changed, 159 insertions(+), 156 deletions(-) diff --git a/app/codec/CMakeLists.txt b/app/codec/CMakeLists.txt index 3123856b5..94d6b4062 100644 --- a/app/codec/CMakeLists.txt +++ b/app/codec/CMakeLists.txt @@ -19,21 +19,23 @@ add_subdirectory(oiio) set(OLIVE_SOURCES ${OLIVE_SOURCES} - codec/decoder.h + codec/conformmanager.cpp + codec/conformmanager.h codec/decoder.cpp - codec/encoder.h + codec/decoder.h codec/encoder.cpp - codec/exportcodec.h + codec/encoder.h codec/exportcodec.cpp - codec/exportformat.h + codec/exportcodec.h codec/exportformat.cpp - codec/frame.h + codec/exportformat.h codec/frame.cpp - codec/samplebuffer.h + codec/frame.h codec/samplebuffer.cpp - codec/waveinput.h + codec/samplebuffer.h codec/waveinput.cpp - codec/waveoutput.h + codec/waveinput.h codec/waveoutput.cpp + codec/waveoutput.h PARENT_SCOPE ) diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index 2746c026e..5647d1519 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -30,18 +30,12 @@ #include "common/ffmpegutils.h" #include "common/filefunctions.h" #include "common/timecodefunctions.h" +#include "conformmanager.h" #include "node/project/project.h" -#ifdef USE_OTIO -#include "task/project/loadotio/loadotio.h" -#endif #include "task/taskmanager.h" namespace olive { -QMutex Decoder::currently_conforming_mutex_; -QWaitCondition Decoder::currently_conforming_wait_cond_; -QVector Decoder::currently_conforming_; - const rational Decoder::kAnyTimecode = RATIONAL_MIN; Decoder::Decoder() @@ -112,7 +106,7 @@ FramePtr Decoder::RetrieveVideo(const rational &timecode, const RetrieveVideoPar return RetrieveVideoInternal(timecode, divider); } -SampleBufferPtr Decoder::RetrieveAudio(const TimeRange &range, const AudioParams ¶ms, const QString& cache_path, Footage::LoopMode loop_mode, const QAtomicInt *cancelled) +Decoder::RetrieveAudioData Decoder::RetrieveAudio(const TimeRange &range, const AudioParams ¶ms, const QString& cache_path, Footage::LoopMode loop_mode, RenderMode::Mode mode) { QMutexLocker locker(&mutex_); @@ -120,58 +114,24 @@ SampleBufferPtr Decoder::RetrieveAudio(const TimeRange &range, const AudioParams if (!stream_.IsValid()) { qCritical() << "Can't retrieve audio on a closed decoder"; - return nullptr; + return {kInvalid, nullptr, nullptr}; } if (!SupportsAudio()) { qCritical() << "Decoder doesn't support audio"; - return nullptr; + return {kInvalid, nullptr, nullptr}; } - // Determine if we already have a conformed version - QString conform_filename = GetConformedFilename(cache_path, params); - CurrentlyConforming want_conform = {stream_, params}; - - currently_conforming_mutex_.lock(); - - // Wait for conform to complete - while (currently_conforming_.contains(want_conform)) { - currently_conforming_wait_cond_.wait(¤tly_conforming_mutex_); + // Get conform state from ConformManager + ConformManager::Conform conform = ConformManager::instance()->GetConformState(id(), cache_path, stream_, params, (mode == RenderMode::kOnline)); + if (conform.state == ConformManager::kConformGenerating) { + return {kWaitingForConform, nullptr, conform.task}; } // See if we got the conform - SampleBufferPtr buffer = RetrieveAudioFromConform(conform_filename, range, loop_mode); + SampleBufferPtr out_buffer = RetrieveAudioFromConform(conform.filename, range, loop_mode); - if (!buffer) { - // We'll need to conform this ourselves - currently_conforming_.append(want_conform); - currently_conforming_mutex_.unlock(); - - // We conform to a different filename until it's done to make it clear even across sessions - // whether this conform is ready or not - QString working_fn = conform_filename; - working_fn.append(QStringLiteral(".working")); - - if (ConformAudioInternal(working_fn, params, cancelled)) { - // Move file to standard conform name, making it clear this conform is ready for use - QFile::remove(conform_filename); - QFile::rename(working_fn, conform_filename); - - // Return audio as planned - buffer = RetrieveAudioFromConform(conform_filename, range, loop_mode); - } else { - // Failed - qCritical() << "Failed to conform audio"; - } - - currently_conforming_mutex_.lock(); - currently_conforming_.removeOne(want_conform); - currently_conforming_wait_cond_.wakeAll(); - } - - currently_conforming_mutex_.unlock(); - - return buffer; + return {kOK, out_buffer, nullptr}; } qint64 Decoder::GetLastAccessedTime() @@ -194,6 +154,11 @@ void Decoder::Close() } } +bool Decoder::ConformAudio(const QString &output_filename, const AudioParams ¶ms, const QAtomicInt *cancelled) +{ + return ConformAudioInternal(output_filename, params, cancelled); +} + /* * DECODER STATIC PUBLIC MEMBERS */ @@ -228,23 +193,6 @@ DecoderPtr Decoder::CreateFromID(const QString &id) return nullptr; } -QString Decoder::GetConformedFilename(const QString& cache_path, const AudioParams ¶ms) -{ - QString index_fn = QStringLiteral("%1.%2:%3").arg(FileFunctions::GetUniqueFileIdentifier(stream_.filename()), - QString::number(stream_.stream())); - - index_fn = QDir(cache_path).filePath(index_fn); - - index_fn.append('.'); - index_fn.append(QString::number(params.sample_rate())); - index_fn.append('.'); - index_fn.append(QString::number(params.format())); - index_fn.append('.'); - index_fn.append(QString::number(params.channel_layout())); - - return index_fn; -} - int64_t Decoder::GetTimeInTimebaseUnits(const rational &time, const rational &timebase, int64_t start_time) { return Timecode::time_to_timestamp(time, timebase) + start_time; diff --git a/app/codec/decoder.h b/app/codec/decoder.h index cea86c922..7b61c2b35 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -37,6 +37,7 @@ extern "C" { #include "common/rational.h" #include "node/project/footage/footage.h" #include "node/project/footage/footagedescription.h" +#include "task/task.h" namespace olive { @@ -184,6 +185,18 @@ public: */ FramePtr RetrieveVideo(const rational& timecode, const RetrieveVideoParams& divider); + enum RetrieveAudioStatus { + kInvalid = -1, + kOK, + kWaitingForConform + }; + + struct RetrieveAudioData { + RetrieveAudioStatus status; + SampleBufferPtr samples; + Task *task; + }; + /** * @brief Retrieve audio data from footage * @@ -192,7 +205,7 @@ public: * * This function is thread safe and can only run while the decoder is open. \see Open() */ - SampleBufferPtr RetrieveAudio(const TimeRange& range, const AudioParams& params, const QString &cache_path, Footage::LoopMode loop_mode, const QAtomicInt *cancelled); + RetrieveAudioData RetrieveAudio(const TimeRange& range, const AudioParams& params, const QString &cache_path, Footage::LoopMode loop_mode, RenderMode::Mode mode); /** * @brief Determine the last time this decoder instance was used in any way @@ -219,6 +232,11 @@ public: */ void Close(); + /** + * @brief Conform audio stream + */ + bool ConformAudio(const QString &output_filename, const AudioParams ¶ms, const QAtomicInt *cancelled = nullptr); + /** * @brief Create a Decoder instance using a Decoder ID * @@ -271,21 +289,6 @@ protected: void SignalProcessingProgress(int64_t ts, int64_t duration); - /** - * @brief Get the destination filename of an audio stream conformed to a set of parameters - */ - QString GetConformedFilename(const QString &cache_path, const AudioParams ¶ms); - - struct CurrentlyConforming { - CodecStream stream; - AudioParams params; - - bool operator==(const CurrentlyConforming& rhs) const - { - return this->stream == rhs.stream && this->params == rhs.params; - } - }; - /** * @brief Return currently open stream * @@ -298,10 +301,6 @@ protected: static int64_t GetTimeInTimebaseUnits(const rational& time, const rational& timebase, int64_t start_time); - static QMutex currently_conforming_mutex_; - static QWaitCondition currently_conforming_wait_cond_; - static QVector currently_conforming_; - signals: /** * @brief While indexing, this signal will provide progress as a percentage (0-100 inclusive) if diff --git a/app/core.cpp b/app/core.cpp index a42d98f91..9dcb7f96c 100644 --- a/app/core.cpp +++ b/app/core.cpp @@ -35,6 +35,7 @@ #include "audio/audiomanager.h" #include "cli/clitask/clitaskdialog.h" +#include "codec/conformmanager.h" #include "common/filefunctions.h" #include "common/xmlutils.h" #include "config/config.h" @@ -144,6 +145,9 @@ void Core::Start() // Initialize FrameManager FrameManager::CreateInstance(); + // Initialize ConformManager + ConformManager::CreateInstance(); + // // Start application // @@ -188,6 +192,8 @@ void Core::Stop() } } + ConformManager::DestroyInstance(); + FrameManager::DestroyInstance(); RenderManager::DestroyInstance(); diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index c2ccc968a..240c97b8a 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -3,6 +3,7 @@ #include #include +#include "codec/conformmanager.h" #include "node/project/project.h" #include "render/rendermanager.h" #include "render/renderprocessor.h" @@ -15,7 +16,8 @@ PreviewAutoCacher::PreviewAutoCacher() : use_custom_range_(false), single_frame_render_(nullptr), last_update_time_(0), - ignore_next_mouse_button_(false) + ignore_next_mouse_button_(false), + last_conform_task_(0) { paused_ = !Config::Current()[QStringLiteral("AutoCacheEnabled")].toBool(), @@ -24,6 +26,8 @@ PreviewAutoCacher::PreviewAutoCacher() : delayed_requeue_timer_.setInterval(Config::Current()[QStringLiteral("AutoCacheDelay")].toInt()); delayed_requeue_timer_.setSingleShot(true); connect(&delayed_requeue_timer_, &QTimer::timeout, this, &PreviewAutoCacher::RequeueFrames); + + connect(ConformManager::instance(), &ConformManager::ConformReady, this, &PreviewAutoCacher::ConformFinished); } PreviewAutoCacher::~PreviewAutoCacher() @@ -131,7 +135,7 @@ void PreviewAutoCacher::VideoInvalidated(const TimeRange &range) void PreviewAutoCacher::AudioInvalidated(const TimeRange &range) { - ClearAudioQueue(); +// ClearAudioQueue(); // Start jobs to re-render the audio at this range, split into 2 second chunks invalidated_audio_.insert(range); @@ -165,35 +169,52 @@ void PreviewAutoCacher::AudioRendered() if (audio_tasks_.contains(watcher)) { if (watcher->HasResult()) { - viewer_node_->audio_playback_cache()->WritePCM(audio_tasks_.value(watcher), + const TimeRange &range = audio_tasks_.value(watcher); + + viewer_node_->audio_playback_cache()->WritePCM(range, watcher->Get().value(), watcher->GetTicket()->GetJobTime()); - // Retrieve visual waveforms - QVector waveform_list = watcher->GetTicket()->property("waveforms").value< QVector >(); - foreach (const RenderProcessor::RenderedWaveform& waveform_info, waveform_list) { - // Find original track - Track* track = nullptr; + bool pcm_is_usable = true; - for (auto it=copy_map_.cbegin(); it!=copy_map_.cend(); it++) { - if (it.value() == waveform_info.track) { - track = static_cast(it.key()); - break; - } + if (watcher->GetTicket()->property("incomplete").toBool()) { + if (last_conform_task_ > watcher->GetTicket()->GetJobTime()) { + // Requeue now + viewer_node_->audio_playback_cache()->Invalidate(range, QDateTime::currentMSecsSinceEpoch()); + pcm_is_usable = false; + } else { + // Wait for conform + audio_needing_conform_.insert(range); } + } - if (track) { - QList valid_ranges = viewer_node_->audio_playback_cache()->GetValidRanges(waveform_info.range, - watcher->GetTicket()->GetJobTime()); - if (!valid_ranges.isEmpty()) { - // Generate visual waveform in this background thread - track->waveform().set_channel_count(viewer_node_->GetAudioParams().channel_count()); + if (pcm_is_usable) { + // Retrieve visual waveforms + QVector waveform_list = watcher->GetTicket()->property("waveforms").value< QVector >(); + foreach (const RenderProcessor::RenderedWaveform& waveform_info, waveform_list) { + // Find original track + Track* track = nullptr; - foreach (const TimeRange& r, valid_ranges) { - track->waveform().OverwriteSums(waveform_info.waveform, r.in(), r.in() - waveform_info.range.in(), r.length()); + for (auto it=copy_map_.cbegin(); it!=copy_map_.cend(); it++) { + if (it.value() == waveform_info.track) { + track = static_cast(it.key()); + break; } + } - emit track->PreviewChanged(); + if (track) { + QList valid_ranges = viewer_node_->audio_playback_cache()->GetValidRanges(waveform_info.range, + watcher->GetTicket()->GetJobTime()); + if (!valid_ranges.isEmpty()) { + // Generate visual waveform in this background thread + track->waveform().set_channel_count(viewer_node_->GetAudioParams().channel_count()); + + foreach (const TimeRange& r, valid_ranges) { + track->waveform().OverwriteSums(waveform_info.waveform, r.in(), r.in() - waveform_info.range.in(), r.length()); + } + + emit track->PreviewChanged(); + } } } } @@ -491,7 +512,7 @@ void PreviewAutoCacher::TryRender() RenderTicketWatcher* watcher = new RenderTicketWatcher(); connect(watcher, &RenderTicketWatcher::Finished, this, &PreviewAutoCacher::AudioRendered); audio_tasks_.insert(watcher, r); - watcher->SetTicket(RenderManager::instance()->RenderAudio(copied_viewer_node_, r, true)); + watcher->SetTicket(RenderManager::instance()->RenderAudio(copied_viewer_node_, r, RenderMode::kOffline, true)); } } @@ -581,6 +602,18 @@ void PreviewAutoCacher::RequeueFrames() } } +void PreviewAutoCacher::ConformFinished() +{ + last_conform_task_ = QDateTime::currentMSecsSinceEpoch(); + + if (viewer_node_) { + foreach (const TimeRange &range, audio_needing_conform_) { + viewer_node_->audio_playback_cache()->Invalidate(range, QDateTime::currentMSecsSinceEpoch()); + } + audio_needing_conform_.clear(); + } +} + void PreviewAutoCacher::IgnoreNextMouseButton() { ignore_next_mouse_button_ = true; @@ -624,6 +657,9 @@ void PreviewAutoCacher::SetViewerNode(ViewerOutput *viewer_node) // No more immediate passthroughts video_immediate_passthroughs_.clear(); + // No more audio conforms + audio_needing_conform_.clear(); + // Delete all of our copied nodes qDeleteAll(created_nodes_); created_nodes_.clear(); diff --git a/app/render/previewautocacher.h b/app/render/previewautocacher.h index a0832df89..63b0e76f3 100644 --- a/app/render/previewautocacher.h +++ b/app/render/previewautocacher.h @@ -167,6 +167,10 @@ private: QTimer delayed_requeue_timer_; + TimeRangeList audio_needing_conform_; + + qint64 last_conform_task_; + private slots: /** * @brief Handler for when the NodeGraph reports a video change over a certain time range @@ -213,6 +217,8 @@ private slots: */ void RequeueFrames(); + void ConformFinished(); + }; } diff --git a/app/render/rendermanager.cpp b/app/render/rendermanager.cpp index 7327f0650..8cd3ae7c4 100644 --- a/app/render/rendermanager.cpp +++ b/app/render/rendermanager.cpp @@ -181,12 +181,12 @@ RenderTicketPtr RenderManager::RenderFrame(ViewerOutput *viewer, ColorManager* c return ticket; } -RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange& r, bool generate_waveforms, bool prioritize) +RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange& r, RenderMode::Mode mode, bool generate_waveforms, bool prioritize) { - return RenderAudio(viewer, r, viewer->GetAudioParams(), generate_waveforms, prioritize); + return RenderAudio(viewer, r, viewer->GetAudioParams(), mode, generate_waveforms, prioritize); } -RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange &r, const AudioParams ¶ms, bool generate_waveforms, bool prioritize) +RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange &r, const AudioParams ¶ms, RenderMode::Mode mode, bool generate_waveforms, bool prioritize) { // Create ticket RenderTicketPtr ticket = std::make_shared(); @@ -194,6 +194,7 @@ RenderTicketPtr RenderManager::RenderAudio(ViewerOutput* viewer, const TimeRange ticket->setProperty("viewer", Node::PtrToValue(viewer)); ticket->setProperty("time", QVariant::fromValue(r)); ticket->setProperty("type", kTypeAudio); + ticket->setProperty("mode", mode); ticket->setProperty("enablewaveforms", generate_waveforms); ticket->setProperty("aparam", QVariant::fromValue(params)); diff --git a/app/render/rendermanager.h b/app/render/rendermanager.h index d65d1a73a..4069a8c07 100644 --- a/app/render/rendermanager.h +++ b/app/render/rendermanager.h @@ -105,8 +105,8 @@ public: * * This function is thread-safe. */ - RenderTicketPtr RenderAudio(ViewerOutput* viewer, const TimeRange& r, const AudioParams& params, bool generate_waveforms, bool prioritize = false); - RenderTicketPtr RenderAudio(ViewerOutput *viewer, const TimeRange& r, bool generate_waveforms, bool prioritize = false); + RenderTicketPtr RenderAudio(ViewerOutput* viewer, const TimeRange& r, const AudioParams& params, RenderMode::Mode mode, bool generate_waveforms, bool prioritize = false); + RenderTicketPtr RenderAudio(ViewerOutput *viewer, const TimeRange& r, RenderMode::Mode mode, bool generate_waveforms, bool prioritize = false); RenderTicketPtr SaveFrameToCache(FrameHashCache* cache, FramePtr frame, const QByteArray& hash, bool prioritize = false); diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index 3e4739a2a..db49bff6a 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -443,13 +443,15 @@ QVariant RenderProcessor::ProcessAudioFootage(const FootageJob &stream, const Ti if (decoder) { const AudioParams& audio_params = ticket_->property("aparam").value(); - SampleBufferPtr frame = decoder->RetrieveAudio(input_time, audio_params, - stream.cache_path(), - stream.loop_mode(), - &IsCancelled()); + Decoder::RetrieveAudioData status = decoder->RetrieveAudio(input_time, audio_params, + stream.cache_path(), + stream.loop_mode(), + static_cast(ticket_->property("mode").toInt())); - if (frame) { - value = QVariant::fromValue(frame); + if (status.status == Decoder::kOK && status.samples) { + value = QVariant::fromValue(status.samples); + } else if (status.status == Decoder::kWaitingForConform) { + ticket_->setProperty("incomplete", true); } } diff --git a/app/task/conform/conform.cpp b/app/task/conform/conform.cpp index 656d76578..44c0aa5a6 100644 --- a/app/task/conform/conform.cpp +++ b/app/task/conform/conform.cpp @@ -20,42 +20,33 @@ #include "conform.h" -#include "codec/decoder.h" - namespace olive { -ConformTask::ConformTask(Footage* footage, int index, const AudioParams& params) : - footage_(footage), - index_(index), - params_(params) +ConformTask::ConformTask(const QString &decoder_id, const Decoder::CodecStream &stream, const AudioParams& params, const QString &output_filename) : + decoder_id_(decoder_id), + stream_(stream), + params_(params), + output_filename_(output_filename) { - SetTitle(tr("Conforming Audio %1:%2").arg(footage_->filename(), QString::number(index_))); + SetTitle(tr("Conforming Audio %1:%2").arg(stream.filename(), QString::number(stream.stream()))); } bool ConformTask::Run() { - // Conforming is done by the renderer now, but I would like to use something like this just to - // show progress + DecoderPtr decoder = Decoder::CreateFromID(decoder_id_); - /*if (stream_->footage()->decoder().isEmpty()) { - SetError(tr("Failed to find decoder to conform audio stream")); + if (!decoder->Open(stream_)) { + SetError(tr("Failed to open decoder for audio conform")); return false; - } else { - DecoderPtr decoder = Decoder::CreateFromID(stream_->footage()->decoder()); + } - decoder->set_stream(stream_); + connect(decoder.get(), &Decoder::IndexProgress, this, &ConformTask::ProgressChanged); - connect(decoder.get(), &Decoder::IndexProgress, this, &ConformTask::ProgressChanged); + bool ret = decoder->ConformAudio(output_filename_, params_, &IsCancelled()); - if (!decoder->ConformAudio(&IsCancelled(), params_)) { - SetError(tr("Failed to conform audio")); - return false; - } else { - return true; - } - }*/ + decoder->Close(); - return true; + return ret; } } diff --git a/app/task/conform/conform.h b/app/task/conform/conform.h index f2268929a..64f998ac5 100644 --- a/app/task/conform/conform.h +++ b/app/task/conform/conform.h @@ -21,6 +21,7 @@ #ifndef CONFORMTASK_H #define CONFORMTASK_H +#include "codec/decoder.h" #include "node/project/footage/footage.h" #include "render/audioparams.h" #include "task/task.h" @@ -31,18 +32,20 @@ class ConformTask : public Task { Q_OBJECT public: - ConformTask(Footage* stream, int index, const AudioParams& params); + ConformTask(const QString &decoder_id, const Decoder::CodecStream &stream, const AudioParams& params, const QString &output_filename); protected: virtual bool Run() override; private: - Footage* footage_; + QString decoder_id_; - int index_; + Decoder::CodecStream stream_; AudioParams params_; + QString output_filename_; + }; } diff --git a/app/task/render/render.cpp b/app/task/render/render.cpp index 9ced40636..a4bd28aad 100644 --- a/app/task/render/render.cpp +++ b/app/task/render/render.cpp @@ -71,7 +71,7 @@ bool RenderTask::Render(ColorManager* manager, watcher->setProperty("range", QVariant::fromValue(this_range)); PrepareWatcher(watcher, &watcher_thread); IncrementRunningTickets(); - watcher->SetTicket(RenderManager::instance()->RenderAudio(viewer_, this_range, audio_params_, false)); + watcher->SetTicket(RenderManager::instance()->RenderAudio(viewer_, this_range, audio_params_, mode, false)); r = end; } diff --git a/app/task/task.h b/app/task/task.h index cf32cd206..b660c470d 100644 --- a/app/task/task.h +++ b/app/task/task.h @@ -99,6 +99,8 @@ public slots: // Print how long this task took for debugging purposes qDebug() << this << "took" << (QDateTime::currentMSecsSinceEpoch() - start_time_); + emit Finished(this, ret); + return ret; } @@ -159,6 +161,13 @@ signals: */ void ProgressChanged(double d); + /** + * @brief Emitted when task is finished + * + * Do NOT delete immediately after this signal, call deleteLater() instead. + */ + void Finished(Task *task, bool succeeded); + private: QString title_;