From 7a6dd02e5b44470baf9996777e0212313010b463 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 11 Nov 2019 11:35:01 +0900 Subject: [PATCH] furthered development of audio renderers Mostly reimplementing functions from other workers to produce audio samples --- app/render/backend/CMakeLists.txt | 5 + app/render/backend/audio/CMakeLists.txt | 22 +++ app/render/backend/audio/audiobackend.cpp | 28 ++++ app/render/backend/audio/audiobackend.h | 25 ++++ app/render/backend/audiorenderbackend.cpp | 81 ++++++++++- app/render/backend/audiorenderbackend.h | 35 ++++- app/render/backend/audiorenderworker.cpp | 22 +++ app/render/backend/audiorenderworker.h | 24 ++++ app/render/backend/opengl/openglbackend.cpp | 81 ++++++----- app/render/backend/opengl/openglbackend.h | 24 +--- app/render/backend/opengl/openglworker.cpp | 14 +- app/render/backend/opengl/openglworker.h | 6 +- app/render/backend/renderbackend.cpp | 52 +++++++ app/render/backend/renderbackend.h | 35 ++++- app/render/backend/renderworker.cpp | 125 +++++++++++++++++ app/render/backend/renderworker.h | 55 ++++++++ app/render/backend/videorenderbackend.cpp | 51 ++----- app/render/backend/videorenderbackend.h | 29 +--- app/render/backend/videorenderframecache.cpp | 39 ++++-- app/render/backend/videorenderframecache.h | 9 +- app/render/backend/videorenderworker.cpp | 136 ++++--------------- app/render/backend/videorenderworker.h | 51 +++---- app/widget/viewer/viewer.cpp | 1 + app/widget/viewer/viewer.h | 2 + 24 files changed, 673 insertions(+), 279 deletions(-) create mode 100644 app/render/backend/audio/CMakeLists.txt create mode 100644 app/render/backend/audio/audiobackend.cpp create mode 100644 app/render/backend/audio/audiobackend.h create mode 100644 app/render/backend/audiorenderworker.cpp create mode 100644 app/render/backend/audiorenderworker.h create mode 100644 app/render/backend/renderworker.cpp create mode 100644 app/render/backend/renderworker.h diff --git a/app/render/backend/CMakeLists.txt b/app/render/backend/CMakeLists.txt index 9b1776c2c..663d5c548 100644 --- a/app/render/backend/CMakeLists.txt +++ b/app/render/backend/CMakeLists.txt @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +add_subdirectory(audio) add_subdirectory(opengl) add_subdirectory(vulkan) @@ -25,9 +26,13 @@ set(OLIVE_SOURCES render/backend/renderbackend.h render/backend/renderbackend.cpp + render/backend/renderworker.h + render/backend/renderworker.cpp render/backend/audiorenderbackend.h render/backend/audiorenderbackend.cpp + render/backend/audiorenderworker.h + render/backend/audiorenderworker.cpp render/backend/videorenderbackend.h render/backend/videorenderbackend.cpp diff --git a/app/render/backend/audio/CMakeLists.txt b/app/render/backend/audio/CMakeLists.txt new file mode 100644 index 000000000..5fcb98547 --- /dev/null +++ b/app/render/backend/audio/CMakeLists.txt @@ -0,0 +1,22 @@ +# Olive - Non-Linear Video Editor +# Copyright (C) 2019 Olive Team +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set(OLIVE_SOURCES + ${OLIVE_SOURCES} + render/backend/audio/audiobackend.h + render/backend/audio/audiobackend.cpp + PARENT_SCOPE +) diff --git a/app/render/backend/audio/audiobackend.cpp b/app/render/backend/audio/audiobackend.cpp new file mode 100644 index 000000000..a19f186f8 --- /dev/null +++ b/app/render/backend/audio/audiobackend.cpp @@ -0,0 +1,28 @@ +#include "audiobackend.h" + +AudioBackend::AudioBackend(QObject *parent) : + AudioRenderBackend(parent) +{ +} + +bool AudioBackend::InitInternal() +{ + // This backend doesn't init anything yet + return true; +} + +void AudioBackend::CloseInternal() +{ + // This backend doesn't init anything yet +} + +bool AudioBackend::CompileInternal() +{ + // This backend doesn't compile anything yet + return true; +} + +void AudioBackend::DecompileInternal() +{ + // This backend doesn't compile anything yet +} diff --git a/app/render/backend/audio/audiobackend.h b/app/render/backend/audio/audiobackend.h new file mode 100644 index 000000000..f21a7cac8 --- /dev/null +++ b/app/render/backend/audio/audiobackend.h @@ -0,0 +1,25 @@ +#ifndef AUDIOBACKEND_H +#define AUDIOBACKEND_H + +#include "../audiorenderbackend.h" + +class AudioBackend : public AudioRenderBackend +{ + Q_OBJECT +public: + AudioBackend(QObject* parent = nullptr); + +protected: + virtual bool InitInternal() override; + + virtual void CloseInternal() override; + + virtual bool CompileInternal() override; + + virtual void DecompileInternal() override; + + + +}; + +#endif // AUDIOBACKEND_H diff --git a/app/render/backend/audiorenderbackend.cpp b/app/render/backend/audiorenderbackend.cpp index 44a1f82ee..2c3fe61aa 100644 --- a/app/render/backend/audiorenderbackend.cpp +++ b/app/render/backend/audiorenderbackend.cpp @@ -1,12 +1,87 @@ #include "audiorenderbackend.h" -AudioRenderBackend::AudioRenderBackend() +#include + +AudioRenderBackend::AudioRenderBackend(QObject *parent) : + RenderBackend(parent) { } +void AudioRenderBackend::SetParameters(const AudioRenderingParams ¶ms) +{ + // Since we're changing parameters, all the existing threads are invalid and must be removed. They will start again + // next time this Node has to process anything. + Close(); + + // Set new parameters + params_ = params; + + // Regenerate the cache ID + RegenerateCacheID(); +} + void AudioRenderBackend::InvalidateCache(const rational &start_range, const rational &end_range) { - Q_UNUSED(start_range) - Q_UNUSED(end_range) + // Add the range to the list + cache_queue_.append(TimeRange(start_range, end_range)); + + // Remove any overlaps so we don't render the same thing twice + ValidateRanges(); + + // Start caching cycle if it hasn't started already + CacheNext(); +} + +void AudioRenderBackend::ViewerNodeChangedEvent(ViewerOutput *node) +{ + if (node != nullptr) { + // FIXME: Hardcoded format + SetParameters(AudioRenderingParams(node->audio_params(), olive::SAMPLE_FMT_FLT)); + } +} + +bool AudioRenderBackend::GenerateCacheIDInternal(QCryptographicHash &hash) +{ + if (!params_.is_valid()) { + return false; + } + + // Generate an ID that is more or less guaranteed to be unique to this Sequence + hash.addData(QString::number(params_.sample_rate()).toUtf8()); + hash.addData(QString::number(params_.channel_layout()).toUtf8()); + hash.addData(QString::number(params_.format()).toUtf8()); + + return true; +} + +void AudioRenderBackend::ValidateRanges() +{ + for (int i=0;i b.out()); } diff --git a/app/render/backend/audiorenderbackend.h b/app/render/backend/audiorenderbackend.h index a57e61d1b..e2f0d87ca 100644 --- a/app/render/backend/audiorenderbackend.h +++ b/app/render/backend/audiorenderbackend.h @@ -1,16 +1,47 @@ #ifndef AUDIORENDERBACKEND_H #define AUDIORENDERBACKEND_H +#include "common/timerange.h" #include "renderbackend.h" class AudioRenderBackend : public RenderBackend { Q_OBJECT public: - AudioRenderBackend(); + AudioRenderBackend(QObject* parent = nullptr); + + /** + * @brief Set parameters of the Renderer + * + * The Renderer owns the buffers that are used in the rendering process and this function sets the kind of buffers + * to use. The Renderer must be stopped when calling this function. + */ + void SetParameters(const AudioRenderingParams ¶ms); public slots: - virtual void InvalidateCache(const rational &start_range, const rational &end_range); + virtual void InvalidateCache(const rational &start_range, const rational &end_range) override; + +protected: + virtual void ViewerNodeChangedEvent(ViewerOutput* node) override; + + /** + * @brief Internal function for generating the cache ID + */ + virtual bool GenerateCacheIDInternal(QCryptographicHash& hash) override; + + //virtual void CacheIDChangedEvent(const QString& id) override; + +private: + void ValidateRanges(); + + TimeRange CombineRange(const TimeRange& a, const TimeRange& b); + + bool RangesOverlap(const TimeRange& a, const TimeRange& b); + + AudioRenderingParams params_; + + QByteArray pcm_data_; + }; #endif // AUDIORENDERBACKEND_H diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp new file mode 100644 index 000000000..21b9934fb --- /dev/null +++ b/app/render/backend/audiorenderworker.cpp @@ -0,0 +1,22 @@ +#include "audiorenderworker.h" + +AudioRenderWorker::AudioRenderWorker(DecoderCache *decoder_cache, QObject *parent) : + RenderWorker(decoder_cache, parent) +{ +} + +void AudioRenderWorker::RenderAsSibling(NodeDependency dep) +{ + +} + +bool AudioRenderWorker::InitInternal() +{ + // Nothing to init yet + return true; +} + +void AudioRenderWorker::CloseInternal() +{ + // Nothing to init yet +} diff --git a/app/render/backend/audiorenderworker.h b/app/render/backend/audiorenderworker.h new file mode 100644 index 000000000..21d0e6b6f --- /dev/null +++ b/app/render/backend/audiorenderworker.h @@ -0,0 +1,24 @@ +#ifndef AUDIORENDERWORKER_H +#define AUDIORENDERWORKER_H + +#include "renderworker.h" + +class AudioRenderWorker : public RenderWorker +{ + Q_OBJECT +public: + AudioRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); + +public slots: + virtual void RenderAsSibling(NodeDependency dep) override; + +protected: + virtual bool InitInternal() override; + + virtual void CloseInternal() override; + + + +}; + +#endif // AUDIORENDERWORKER_H diff --git a/app/render/backend/opengl/openglbackend.cpp b/app/render/backend/opengl/openglbackend.cpp index e34bdde67..9d8ed95a0 100644 --- a/app/render/backend/opengl/openglbackend.cpp +++ b/app/render/backend/opengl/openglbackend.cpp @@ -7,8 +7,8 @@ OpenGLBackend::OpenGLBackend(QObject *parent) : VideoRenderBackend(parent), - push_time_(-1), - compiled_(false) + push_texture_(nullptr), + push_time_(-1) { } @@ -35,12 +35,15 @@ bool OpenGLBackend::InitInternal() QThread* thread = threads().at(i); // Create one processor object for each thread - OpenGLWorker* processor = new OpenGLWorker(share_ctx, &shader_cache_, &decoder_cache_); + OpenGLWorker* processor = new OpenGLWorker(share_ctx, &shader_cache_, decoder_cache(), frame_cache()); processor->SetParameters(params()); // Connect to it connect(processor, SIGNAL(RequestSibling(NodeDependency)), this, SLOT(ThreadRequestedSibling(NodeDependency))); - connect(processor, SIGNAL(CompletedFrame(NodeDependency)), this, SLOT(ThreadCompletedFrame(NodeDependency))); + connect(processor, SIGNAL(CompletedFrame(NodeDependency, QByteArray)), this, SLOT(ThreadCompletedFrame(NodeDependency, QByteArray))); + connect(processor, SIGNAL(HashAlreadyBeingCached()), this, SLOT(ThreadSkippedFrame())); + connect(processor, SIGNAL(CompletedDownload(NodeDependency, QByteArray)), this, SLOT(ThreadCompletedDownload(NodeDependency, QByteArray))); + connect(processor, SIGNAL(HashAlreadyExists(NodeDependency, QByteArray)), this, SLOT(ThreadHashAlreadyExists(NodeDependency, QByteArray))); // Finally, we can move it to its own thread processor->moveToThread(thread); @@ -56,54 +59,36 @@ bool OpenGLBackend::InitInternal() master_texture_ = std::make_shared(); master_texture_->Create(share_ctx, params().effective_width(), params().effective_height(), params().format()); + /* // Create internal FBO for copying textures copy_buffer_.Create(share_ctx); copy_buffer_.Attach(master_texture_); copy_pipeline_ = OpenGLShader::CreateDefault(); + */ return true; } -void OpenGLBackend::GenerateFrame(const rational &time) -{ - qDebug() << "Compiled state:" << compiled_; - if (!compiled_) { - Compile(); - } - - NodeDependency dep = NodeDependency(viewer_node()->texture_input()->get_connected_output(), time, time); - - foreach (OpenGLWorker* worker, processors_) { - if (worker->IsAvailable() || worker == processors_.last()) { - QMetaObject::invokeMethod(worker, - "Render", - Qt::QueuedConnection, - Q_ARG(NodeDependency, dep)); - } - } -} - void OpenGLBackend::CloseInternal() { Decompile(); - copy_buffer_.Destroy(); + //copy_buffer_.Destroy(); master_texture_ = nullptr; - copy_pipeline_ = nullptr; + push_texture_ = nullptr; + //copy_pipeline_ = nullptr; VideoRenderBackend::Close(); } OpenGLTexturePtr OpenGLBackend::GetCachedFrameAsTexture(const rational &time) { - last_time_requested_ = time; - if (push_time_ >= 0) { rational temp_push_time = push_time_; push_time_ = -1; if (time == temp_push_time) { - return master_texture_; + return push_texture_; } } @@ -201,23 +186,28 @@ bool OpenGLBackend::TraverseCompiling(Node *n) return true; } -void OpenGLBackend::ThreadCompletedFrame(NodeDependency path) +void OpenGLBackend::ThreadCompletedFrame(NodeDependency path, QByteArray hash) { caching_ = false; OpenGLTexturePtr texture = path.node()->get_cached_value(path.range()).value(); - if (texture != nullptr) { - QString cache_fn = frame_cache()->CachePathName(QStringLiteral("%1-%2").arg(QString::number(path.in().numerator()), QString::number(path.in().denominator())).toLatin1()); + if (texture == nullptr) { + // No frame received, we set hash to an empty + frame_cache()->RemoveHash(path.in()); + } else { + // Received a texture, let's download it + QString cache_fn = frame_cache()->CachePathName(hash); // Find an available worker to download this texture - foreach (OpenGLWorker* worker, processors_) { + foreach (RenderWorker* worker, processors_) { // Check if one is available, but worst case if none of them are available, just queue it on the last worker since // it's the least likely to get work if (worker->IsAvailable() || worker == processors_.last()) { QMetaObject::invokeMethod(worker, "Download", Q_ARG(NodeDependency, path), + Q_ARG(QByteArray, hash), Q_ARG(QVariant, QVariant::fromValue(texture)), Q_ARG(QString, cache_fn)); break; @@ -225,6 +215,11 @@ void OpenGLBackend::ThreadCompletedFrame(NodeDependency path) } } + // Set as push texture + push_time_ = path.in(); + push_texture_ = texture; + emit CachedFrameReady(push_time_); + CacheNext(); } @@ -279,7 +274,7 @@ void OpenGLBackend::ThreadCompletedFrame(NodeDependency path) void OpenGLBackend::ThreadRequestedSibling(NodeDependency dep) { // Try to queue another thread to run this dep in advance - foreach (OpenGLWorker* worker, processors_) { + foreach (RenderWorker* worker, processors_) { if (worker->IsAvailable()) { QMetaObject::invokeMethod(worker, "RenderAsSibling", @@ -290,6 +285,26 @@ void OpenGLBackend::ThreadRequestedSibling(NodeDependency dep) } } +void OpenGLBackend::ThreadCompletedDownload(NodeDependency dep, QByteArray hash) +{ + frame_cache()->SetHash(dep.in(), hash); + + emit CachedFrameReady(dep.in()); +} + +void OpenGLBackend::ThreadSkippedFrame() +{ + caching_ = false; + CacheNext(); +} + +void OpenGLBackend::ThreadHashAlreadyExists(NodeDependency dep, QByteArray hash) +{ + ThreadCompletedDownload(dep, hash); + + ThreadSkippedFrame(); +} + /*void OpenGLBackend::ThreadSkippedFrame(const rational& time, const QByteArray& hash) { caching_ = false; diff --git a/app/render/backend/opengl/openglbackend.h b/app/render/backend/opengl/openglbackend.h index 51dd8375a..51d414909 100644 --- a/app/render/backend/opengl/openglbackend.h +++ b/app/render/backend/opengl/openglbackend.h @@ -23,8 +23,6 @@ protected: virtual void CloseInternal() override; - virtual void GenerateFrame(const rational& time) override; - virtual bool CompileInternal() override; virtual void DecompileInternal() override; @@ -32,30 +30,22 @@ protected: private: bool TraverseCompiling(Node* n); - QVector processors_; - OpenGLTexturePtr master_texture_; + OpenGLTexturePtr push_texture_; rational push_time_; - OpenGLFramebuffer copy_buffer_; - OpenGLShaderPtr copy_pipeline_; + /*OpenGLFramebuffer copy_buffer_; + OpenGLShaderPtr copy_pipeline_;*/ OpenGLShaderCache shader_cache_; - bool compiled_; - private slots: - void ThreadCompletedFrame(NodeDependency path); + void ThreadCompletedFrame(NodeDependency path, QByteArray hash); void ThreadRequestedSibling(NodeDependency dep); + void ThreadCompletedDownload(NodeDependency dep, QByteArray hash); + void ThreadSkippedFrame(); + void ThreadHashAlreadyExists(NodeDependency dep, QByteArray hash); - - - //void ThreadCallback(OpenGLTexturePtr texture, const rational& time, const QByteArray& hash); - - - //void ThreadSkippedFrame(const rational &time, const QByteArray &hash); - - //void DownloadThreadComplete(const QByteArray &hash); }; #endif // OPENGLBACKEND_H diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index 701347461..686c5c9ec 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -4,8 +4,8 @@ #include "node/node.h" #include "render/pixelservice.h" -OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, QObject *parent) : - VideoRenderWorker(decoder_cache, parent), +OpenGLWorker::OpenGLWorker(QOpenGLContext *share_ctx, OpenGLShaderCache *shader_cache, DecoderCache *decoder_cache, VideoRenderFrameCache *frame_cache, QObject *parent) : + VideoRenderWorker(decoder_cache, frame_cache, parent), share_ctx_(share_ctx), ctx_(nullptr), functions_(nullptr), @@ -21,6 +21,10 @@ OpenGLWorker::~OpenGLWorker() bool OpenGLWorker::InitInternal() { + if (!VideoRenderWorker::InitInternal()) { + return false; + } + // Create context object ctx_ = new QOpenGLContext(); @@ -48,6 +52,12 @@ QVariant OpenGLWorker::FrameToTexture(FramePtr frame) OpenGLTexturePtr footage_tex = std::make_shared(); footage_tex->Create(ctx_, frame); + // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU + //if (video_params().mode() == olive::kOnline) { + // Convert frame to float for + //frame = PixelService::ConvertPixelFormat(frame, olive::PIX_FMT_RGBA32F); + //} + // FIXME: Alpha association and color management return QVariant::fromValue(footage_tex); diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index 85e8ec900..2702f61c3 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -11,7 +11,11 @@ class OpenGLWorker : public VideoRenderWorker { Q_OBJECT public: - OpenGLWorker(QOpenGLContext* share_ctx, OpenGLShaderCache* shader_cache, DecoderCache* decoder_cache, QObject* parent = nullptr); + OpenGLWorker(QOpenGLContext* share_ctx, + OpenGLShaderCache* shader_cache, + DecoderCache* decoder_cache, + VideoRenderFrameCache* frame_cache, + QObject* parent = nullptr); virtual ~OpenGLWorker() override; diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index dbd47245e..12af1eb5c 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -5,6 +5,8 @@ RenderBackend::RenderBackend(QObject *parent) : QObject(parent), + compiled_(false), + caching_(false), started_(false), viewer_node_(nullptr) { @@ -57,6 +59,11 @@ void RenderBackend::Close() thread->wait(); // FIXME: Maximum time in case a thread is stuck? } threads_.clear(); + + foreach (RenderWorker* processor, processors_) { + delete processor; + } + processors_.clear(); } const QString &RenderBackend::GetError() const @@ -148,11 +155,56 @@ void RenderBackend::ViewerNodeChangedEvent(ViewerOutput *node) Q_UNUSED(node) } +void RenderBackend::CacheNext() +{ + if (!Init() || cache_queue_.isEmpty() || viewer_node() == nullptr || caching_) { + return; + } + + TimeRange cache_frame = cache_queue_.takeFirst(); + + qDebug() << "Caching FRAME" << cache_frame.in() << "to" << cache_frame.out(); + + caching_ = GenerateData(cache_frame); +} + +bool RenderBackend::GenerateData(const TimeRange &range) +{ + if (!Compile()) { + qDebug() << "Graph remains uncompiled, nothing to be done"; + return false; + } + + NodeDependency dep = NodeDependency(viewer_node()->texture_input()->get_connected_output(), range.in(), range.out()); + + foreach (RenderWorker* worker, processors_) { + if (worker->IsAvailable() || worker == processors_.last()) { + QMetaObject::invokeMethod(worker, + "Render", + Qt::QueuedConnection, + Q_ARG(NodeDependency, dep)); + return true; + } + } + + return false; +} + ViewerOutput *RenderBackend::viewer_node() const { return viewer_node_; } +DecoderCache *RenderBackend::decoder_cache() +{ + return &decoder_cache_; +} + +const QString &RenderBackend::cache_id() const +{ + return cache_id_; +} + const QVector &RenderBackend::threads() { return threads_; diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index 16c84a9a8..d4c8467fa 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -1,8 +1,11 @@ #ifndef RENDERBACKEND_H #define RENDERBACKEND_H +#include + #include "decodercache.h" #include "node/output/viewer/viewer.h" +#include "renderworker.h" class RenderBackend : public QObject { @@ -43,8 +46,6 @@ protected: virtual void DecompileInternal() = 0; - DecoderCache decoder_cache_; - const QVector& threads(); /** @@ -58,11 +59,28 @@ protected: virtual void ViewerNodeChangedEvent(ViewerOutput* node); + /** + * @brief Function called when there are frames in the queue to cache + * + * This function is NOT thread-safe and should only be called in the main thread. + */ + void CacheNext(); + + bool GenerateData(const TimeRange& range); + ViewerOutput* viewer_node() const; - QString cache_name_; - qint64 cache_time_; - QString cache_id_; + DecoderCache* decoder_cache(); + + const QString& cache_id() const; + + QList cache_queue_; + + QVector processors_; + + bool compiled_; + + bool caching_; private: /** @@ -85,7 +103,12 @@ private: */ QString error_; - bool compiled_; + DecoderCache decoder_cache_; + + QString cache_name_; + qint64 cache_time_; + QString cache_id_; + }; #endif // RENDERBACKEND_H diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp new file mode 100644 index 000000000..19d600f3e --- /dev/null +++ b/app/render/backend/renderworker.cpp @@ -0,0 +1,125 @@ +#include "renderworker.h" + +#include "node/block/block.h" + +RenderWorker::RenderWorker(DecoderCache *decoder_cache, QObject *parent) : + QObject(parent), + working_(0), + started_(false), + decoder_cache_(decoder_cache) +{ + +} + +bool RenderWorker::IsAvailable() +{ + return (working_ == 0); +} + +bool RenderWorker::Init() +{ + if (started_) { + return true; + } + + if (!(started_ = InitInternal())) { + Close(); + } + + return started_; +} + +void RenderWorker::Close() +{ + CloseInternal(); + + started_ = false; +} + +void RenderWorker::Render(NodeDependency path) +{ + NodeOutput* output = path.node(); + Node* node = output->parent(); + + QList all_nodes_in_graph = ListNodeAndAllDependencies(node); + + // Lock all Nodes to prevent UI changes during this render + foreach (Node* dep, all_nodes_in_graph) { + dep->LockUserInput(); + } + + RenderInternal(path); + + // Unlock all Nodes so changes can be made again + foreach (Node* dep, all_nodes_in_graph) { + dep->UnlockUserInput(); + } +} + +DecoderCache *RenderWorker::decoder_cache() +{ + return decoder_cache_; +} + +Node *RenderWorker::ValidateBlock(Node *n, const rational& time) +{ + if (n->IsBlock()) { + Block* block = static_cast(n); + + while (block != nullptr && block->in() > time) { + // This Block is too late, find an earlier one + block = block->previous(); + } + + while (block != nullptr && block->out() <= time) { + // This block is too early, find a later one + block = block->next(); + } + + // By this point, we should have the correct Block or nullptr if there's no Block here + return block; + } + + return n; +} + +void RenderWorker::RenderInternal(const NodeDependency &path) +{ + RenderAsSibling(path); +} + +StreamPtr RenderWorker::ResolveStreamFromInput(NodeInput *input) +{ + return input->get_value_at_time(0).value(); +} + +DecoderPtr RenderWorker::ResolveDecoderFromInput(NodeInput *input) +{ + // Access a map of Node inputs and decoder instances and retrieve a frame! + StreamPtr stream = ResolveStreamFromInput(input); + DecoderPtr decoder = decoder_cache()->GetDecoder(stream.get()); + + if (decoder == nullptr && stream != nullptr) { + // Init decoder + decoder = Decoder::CreateFromID(stream->footage()->decoder()); + decoder->set_stream(stream); + decoder_cache()->AddDecoder(stream.get(), decoder); + } + + return decoder; +} + +QList RenderWorker::ListNodeAndAllDependencies(Node *n) +{ + QList node_list; + + node_list.append(n); + node_list.append(n->GetDependencies()); + + return node_list; +} + +bool RenderWorker::IsStarted() +{ + return started_; +} diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h new file mode 100644 index 000000000..f2a30044b --- /dev/null +++ b/app/render/backend/renderworker.h @@ -0,0 +1,55 @@ +#ifndef RENDERWORKER_H +#define RENDERWORKER_H + +#include + +#include "node/node.h" +#include "decodercache.h" + +class RenderWorker : public QObject +{ + Q_OBJECT +public: + RenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); + + Q_DISABLE_COPY_MOVE(RenderWorker) + + bool Init(); + + bool IsStarted(); + + bool IsAvailable(); + +public slots: + void Close(); + + void Render(NodeDependency path); + + virtual void RenderAsSibling(NodeDependency dep) = 0; + +protected: + Node* ValidateBlock(Node* n, const rational& time); + + virtual bool InitInternal() = 0; + + virtual void CloseInternal() = 0; + + virtual void RenderInternal(const NodeDependency& path); + + StreamPtr ResolveStreamFromInput(NodeInput* input); + DecoderPtr ResolveDecoderFromInput(NodeInput* input); + + QList ListNodeAndAllDependencies(Node* n); + + DecoderCache* decoder_cache(); + + QAtomicInt working_; + +private: + bool started_; + + DecoderCache* decoder_cache_; + +}; + +#endif // RENDERWORKER_H diff --git a/app/render/backend/videorenderbackend.cpp b/app/render/backend/videorenderbackend.cpp index 49a0d7b19..253593074 100644 --- a/app/render/backend/videorenderbackend.cpp +++ b/app/render/backend/videorenderbackend.cpp @@ -27,12 +27,10 @@ #include #include -#include "opengl/functions.h" #include "render/pixelservice.h" VideoRenderBackend::VideoRenderBackend(QObject *parent) : - RenderBackend(parent), - caching_(false) + RenderBackend(parent) { // FIXME: Cache name should actually be the name of the sequence SetCacheName("Test"); @@ -77,34 +75,28 @@ void VideoRenderBackend::InvalidateCache(const rational &start_range, const rati diff = qAbs(diff) * 5; } - bool contains = false; bool added = false; - QLinkedList::iterator insert_iterator; - for (QLinkedList::iterator i = cache_queue_.begin();i != cache_queue_.end();i++) { - rational compare = *i; + TimeRange new_range(r, r); - if (!added) { - rational compare_diff = compare - last_time; + for (int i=0;i diff) { - insert_iterator = i; - added = true; - } + if (compare_diff > diff) { + cache_queue_.insert(i, new_range); + added = true; + break; } if (compare == r) { - contains = true; + added = true; break; } } - if (!contains) { - if (added) { - cache_queue_.insert(insert_iterator, r); - } else { - cache_queue_.append(r); - } + if (!added) { + cache_queue_.append(new_range); } } @@ -150,7 +142,7 @@ void VideoRenderBackend::SetParameters(const VideoRenderingParams& params) bool VideoRenderBackend::GenerateCacheIDInternal(QCryptographicHash& hash) { - if (cache_name_.isEmpty() || !params_.is_valid()) { + if (!params_.is_valid()) { return false; } @@ -173,21 +165,6 @@ VideoRenderFrameCache *VideoRenderBackend::frame_cache() return &frame_cache_; } -void VideoRenderBackend::CacheNext() -{ - if (!Init() || cache_queue_.isEmpty() || viewer_node() == nullptr || caching_) { - return; - } - - rational cache_frame = cache_queue_.takeFirst(); - - qDebug() << "Caching" << cache_frame.toDouble(); - - GenerateFrame(cache_frame); - - caching_ = true; -} - const char *VideoRenderBackend::GetCachedFrame(const rational &time) { last_time_requested_ = time; @@ -197,7 +174,7 @@ const char *VideoRenderBackend::GetCachedFrame(const rational &time) return nullptr; } - if (cache_id_.isEmpty()) { + if (cache_id().isEmpty()) { qWarning() << "No cache ID"; return nullptr; } diff --git a/app/render/backend/videorenderbackend.h b/app/render/backend/videorenderbackend.h index 30276f35a..02c4e558c 100644 --- a/app/render/backend/videorenderbackend.h +++ b/app/render/backend/videorenderbackend.h @@ -51,18 +51,6 @@ public: * * The Renderer owns the buffers that are used in the rendering process and this function sets the kind of buffers * to use. The Renderer must be stopped when calling this function. - * - * @param width - * - * Buffer width - * - * @param height - * - * Buffer height - * - * @param format - * - * Buffer pixel format */ void SetParameters(const VideoRenderingParams ¶ms); @@ -87,25 +75,12 @@ protected: virtual void ViewerNodeChangedEvent(ViewerOutput* node) override; - virtual void GenerateFrame(const rational&) = 0; - const char *GetCachedFrame(const rational& time); VideoRenderFrameCache* frame_cache(); - /** - * @brief Function called when there are frames in the queue to cache - * - * This function is NOT thread-safe and should only be called in the main thread. - */ - void CacheNext(); - const VideoRenderingParams& params() const; - rational last_time_requested_; - - bool caching_; - /** * @brief Internal function for generating the cache ID */ @@ -119,12 +94,12 @@ signals: private: VideoRenderingParams params_; - QLinkedList cache_queue_; - QByteArray cache_frame_load_buffer_; VideoRenderFrameCache frame_cache_; + rational last_time_requested_; + private slots: diff --git a/app/render/backend/videorenderframecache.cpp b/app/render/backend/videorenderframecache.cpp index 627c31d00..5d6b24c1a 100644 --- a/app/render/backend/videorenderframecache.cpp +++ b/app/render/backend/videorenderframecache.cpp @@ -12,31 +12,31 @@ VideoRenderFrameCache::VideoRenderFrameCache() bool VideoRenderFrameCache::HasHash(const QByteArray &hash) { - return QFileInfo::exists(CachePathName(hash)); + return QFileInfo::exists(CachePathName(hash)) && !IsCaching(hash); } bool VideoRenderFrameCache::IsCaching(const QByteArray &hash) { - cache_hash_list_mutex_.lock(); + currently_caching_lock_.lock(); - bool is_caching = cache_hash_list_.contains(hash); + bool is_caching = currently_caching_list_.contains(hash); - cache_hash_list_mutex_.unlock(); + currently_caching_lock_.unlock(); return is_caching; } bool VideoRenderFrameCache::TryCache(const QByteArray &hash) { - cache_hash_list_mutex_.lock(); + currently_caching_lock_.lock(); - bool is_caching = cache_hash_list_.contains(hash); + bool is_caching = currently_caching_list_.contains(hash); if (!is_caching) { - cache_hash_list_.append(hash); + currently_caching_list_.append(hash); } - cache_hash_list_mutex_.unlock(); + currently_caching_lock_.unlock(); return !is_caching; } @@ -51,6 +51,29 @@ QByteArray VideoRenderFrameCache::TimeToHash(const rational &time) return time_hash_map_.value(time); } +void VideoRenderFrameCache::SetHash(const rational &time, const QByteArray &hash) +{ + // No longer currently caching this frame + RemoveHashFromCurrentlyCaching(hash); + + // Insert frame into map + time_hash_map_.insert(time, hash); +} + +void VideoRenderFrameCache::RemoveHash(const rational &time) +{ + RemoveHashFromCurrentlyCaching(time_hash_map_.value(time)); + + time_hash_map_.remove(time); +} + +void VideoRenderFrameCache::RemoveHashFromCurrentlyCaching(const QByteArray &hash) +{ + currently_caching_lock_.lock(); + currently_caching_list_.removeOne(hash); + currently_caching_lock_.unlock(); +} + QString VideoRenderFrameCache::CachePathName(const QByteArray &hash) { QDir this_cache_dir = QDir(GetMediaCacheLocation()).filePath(cache_id_); diff --git a/app/render/backend/videorenderframecache.h b/app/render/backend/videorenderframecache.h index 5e609bfc0..4f6fbbda0 100644 --- a/app/render/backend/videorenderframecache.h +++ b/app/render/backend/videorenderframecache.h @@ -34,11 +34,16 @@ public: QByteArray TimeToHash(const rational& time); + void SetHash(const rational& time, const QByteArray& hash); + void RemoveHash(const rational& time); + private: + void RemoveHashFromCurrentlyCaching(const QByteArray& hash); + QMap time_hash_map_; - QMutex cache_hash_list_mutex_; - QVector cache_hash_list_; + QMutex currently_caching_lock_; + QVector currently_caching_list_; QString cache_id_; }; diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index b3f261331..2df775198 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -3,38 +3,41 @@ #include #include "common/define.h" -#include "node/block/block.h" #include "node/node.h" #include "render/pixelservice.h" -VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, QObject *parent) : - QObject(parent), - decoder_cache_(decoder_cache), - started_(false) +VideoRenderWorker::VideoRenderWorker(DecoderCache *decoder_cache, VideoRenderFrameCache *frame_cache, QObject *parent) : + RenderWorker(decoder_cache, parent), + frame_cache_(frame_cache) { } -bool VideoRenderWorker::IsAvailable() -{ - return (working_ == 0); -} - -void VideoRenderWorker::Close() -{ - CloseInternal(); - - started_ = false; -} - const VideoRenderingParams &VideoRenderWorker::video_params() { return video_params_; } -DecoderCache *VideoRenderWorker::decoder_cache() +void VideoRenderWorker::RenderInternal(const NodeDependency& path) { - return decoder_cache_; + // Get hash of node graph + // We use SHA-1 for speed (benchmarks show it's the fastest hash available to us) + QCryptographicHash hasher(QCryptographicHash::Sha1); + HashNodeRecursively(&hasher, path.node()->parent(), path.in()); + QByteArray hash = hasher.result(); + + if (frame_cache_->HasHash(hash)) { + // We've already cached this hash, no need to continue + emit HashAlreadyExists(path, hash); + } else if (frame_cache_->TryCache(hash)) { + // This hash is available for us to cache, start traversing graph + RenderAsSibling(path); + + emit CompletedFrame(path, hash); + } else { + // Another thread must be caching this already, nothing to be done + emit HashAlreadyBeingCached(); + } } void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, const rational& time) @@ -94,49 +97,6 @@ void VideoRenderWorker::HashNodeRecursively(QCryptographicHash *hash, Node* n, c } } -StreamPtr VideoRenderWorker::ResolveStreamFromInput(NodeInput *input) -{ - return input->get_value_at_time(0).value(); -} - -DecoderPtr VideoRenderWorker::ResolveDecoderFromInput(NodeInput *input) -{ - // Access a map of Node inputs and decoder instances and retrieve a frame! - StreamPtr stream = ResolveStreamFromInput(input); - DecoderPtr decoder = decoder_cache()->GetDecoder(stream.get()); - - if (decoder == nullptr && stream != nullptr) { - // Init decoder - decoder = Decoder::CreateFromID(stream->footage()->decoder()); - decoder->set_stream(stream); - decoder_cache()->AddDecoder(stream.get(), decoder); - } - - return decoder; -} - -Node *VideoRenderWorker::ValidateBlock(Node *n, const rational& time) -{ - if (n->IsBlock()) { - Block* block = static_cast(n); - - while (block != nullptr && block->in() > time) { - // This Block is too late, find an earlier one - block = block->previous(); - } - - while (block != nullptr && block->out() <= time) { - // This block is too early, find a later one - block = block->next(); - } - - // By this point, we should have the correct Block or nullptr if there's no Block here - return block; - } - - return n; -} - void VideoRenderWorker::SetParameters(const VideoRenderingParams &video_params) { video_params_ = video_params; @@ -144,51 +104,15 @@ void VideoRenderWorker::SetParameters(const VideoRenderingParams &video_params) ParametersChangedEvent(); } -bool VideoRenderWorker::Init() +bool VideoRenderWorker::InitInternal() { - if (started_) { - return true; - } - - started_ = InitInternal(); - - if (started_) { - download_buffer_.resize(PixelService::GetBufferSize(video_params().format(), video_params().effective_width(), video_params().effective_height())); - } else { - Close(); - } - - return started_; + download_buffer_.resize(PixelService::GetBufferSize(video_params().format(), video_params().effective_width(), video_params().effective_height())); + return true; } -bool VideoRenderWorker::IsStarted() +void VideoRenderWorker::CloseInternal() { - return started_; -} - -void VideoRenderWorker::Render(NodeDependency path) -{ - NodeOutput* output = path.node(); - Node* node = output->parent(); - - QList all_nodes_in_graph; - all_nodes_in_graph.append(node); - all_nodes_in_graph.append(node->GetDependencies()); - - // Lock all Nodes to prevent UI changes during this render - foreach (Node* dep, all_nodes_in_graph) { - dep->LockUserInput(); - } - - // Start traversing graph - RenderAsSibling(path); - - // Unlock all Nodes so changes can be made again - foreach (Node* dep, all_nodes_in_graph) { - dep->UnlockUserInput(); - } - - emit CompletedFrame(path); + download_buffer_.clear(); } QList VideoRenderWorker::ProcessNodeInputsForTime(Node *n, const TimeRange &time) @@ -225,8 +149,6 @@ QList VideoRenderWorker::ProcessNodeInputsForTime(Node *n, const Tim QVariant value = FrameToTexture(frame); input->set_stored_value(value); - - qDebug() << "Placing texture" << value << "into input" << input; } } } @@ -366,7 +288,7 @@ end_render: -void VideoRenderWorker::Download(NodeDependency dep, QVariant texture, QString filename) +void VideoRenderWorker::Download(NodeDependency dep, QByteArray hash, QVariant texture, QString filename) { working_++; @@ -388,7 +310,7 @@ void VideoRenderWorker::Download(NodeDependency dep, QVariant texture, QString f out->write_image(format_info.oiio_desc, download_buffer_.data()); out->close(); - emit CompletedDownload(dep); + emit CompletedDownload(dep, hash); } else { qWarning() << "Failed to open output file:" << filename; } diff --git a/app/render/backend/videorenderworker.h b/app/render/backend/videorenderworker.h index ed3486731..682c8bc4b 100644 --- a/app/render/backend/videorenderworker.h +++ b/app/render/backend/videorenderworker.h @@ -2,56 +2,44 @@ #define VIDEORENDERWORKER_H #include -#include -#include "decodercache.h" #include "node/dependency.h" #include "render/videoparams.h" +#include "renderworker.h" +#include "videorenderframecache.h" -class VideoRenderWorker : public QObject { +class VideoRenderWorker : public RenderWorker { Q_OBJECT public: - VideoRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); - - Q_DISABLE_COPY_MOVE(VideoRenderWorker) - - bool IsStarted(); + VideoRenderWorker(DecoderCache* decoder_cache, VideoRenderFrameCache* frame_cache, QObject* parent = nullptr); void SetParameters(const VideoRenderingParams& video_params); - virtual bool Init(); - - bool IsAvailable(); - public slots: - void Close(); + virtual void RenderAsSibling(NodeDependency dep) override; - void Render(NodeDependency path); - - void RenderAsSibling(NodeDependency dep); - - void Download(NodeDependency dep, QVariant texture, QString filename); + void Download(NodeDependency dep, QByteArray hash, QVariant texture, QString filename); signals: void RequestSibling(NodeDependency path); - void CompletedFrame(NodeDependency path); + void CompletedFrame(NodeDependency path, QByteArray hash); - void CompletedDownload(NodeDependency path); + void CompletedDownload(NodeDependency path, QByteArray hash); + + void HashAlreadyBeingCached(); + + void HashAlreadyExists(NodeDependency path, QByteArray hash); protected: - virtual bool InitInternal() = 0; + virtual bool InitInternal() override; - virtual void CloseInternal() = 0; + virtual void CloseInternal() override; virtual QVariant FrameToTexture(FramePtr frame) = 0; const VideoRenderingParams& video_params(); - DecoderCache* decoder_cache(); - - Node* ValidateBlock(Node* n, const rational& time); - virtual void ParametersChangedEvent(){} virtual bool OutputIsShader(NodeOutput *output) = 0; @@ -60,26 +48,21 @@ protected: virtual void TextureToBuffer(const QVariant& texture, QByteArray& buffer) = 0; + virtual void RenderInternal(const NodeDependency& path) override; + private: void ProcessNode(); - StreamPtr ResolveStreamFromInput(NodeInput* input); - DecoderPtr ResolveDecoderFromInput(NodeInput* input); - QList ProcessNodeInputsForTime(Node* n, const TimeRange& time); void HashNodeRecursively(QCryptographicHash* hash, Node *n, const rational &time); VideoRenderingParams video_params_; - DecoderCache* decoder_cache_; - - QAtomicInt working_; + VideoRenderFrameCache* frame_cache_; QByteArray download_buffer_; - bool started_; - private slots: }; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index f3352f9a8..f4bce24d4 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -86,6 +86,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) : // Start background renderers video_renderer_ = new OpenGLBackend(this); connect(video_renderer_, SIGNAL(CachedFrameReady(const rational&)), this, SLOT(RendererCachedFrame(const rational&))); + audio_renderer_ = new AudioBackend(this); } void ViewerWidget::SetTimebase(const rational &r) diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index b9463341d..a4c608e95 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -32,6 +32,7 @@ #include "node/output/viewer/viewer.h" #include "render/backend/opengl/openglbackend.h" #include "render/backend/opengl/opengltexture.h" +#include "render/backend/audio/audiobackend.h" #include "viewerglwidget.h" #include "viewersizer.h" #include "widget/playbackcontrols/playbackcontrols.h" @@ -112,6 +113,7 @@ private: void PushScrubbedAudio(); OpenGLBackend* video_renderer_; + AudioBackend* audio_renderer_; ViewerSizer* sizer_;