From 9e7e2cc1a81b79f2e9180f9a77ac515f3e159b0d Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sun, 20 Oct 2019 00:02:14 +1100 Subject: [PATCH] completed separation of video renderer --- app/project/item/sequence/sequence.cpp | 24 +-- app/project/item/sequence/sequence.h | 1 - app/render/video/videorenderer.cpp | 211 ++++++++++++------------- app/render/video/videorenderer.h | 41 +++-- app/widget/viewer/viewer.cpp | 20 ++- app/widget/viewer/viewer.h | 5 +- 6 files changed, 145 insertions(+), 157 deletions(-) diff --git a/app/project/item/sequence/sequence.cpp b/app/project/item/sequence/sequence.cpp index e09fbd295..63d01bf0b 100644 --- a/app/project/item/sequence/sequence.cpp +++ b/app/project/item/sequence/sequence.cpp @@ -32,7 +32,6 @@ Sequence::Sequence() : timeline_output_(nullptr), - video_renderer_processor_(nullptr), viewer_output_(nullptr), video_track_output_(nullptr), audio_track_output_(nullptr) @@ -61,10 +60,6 @@ void Sequence::add_default_nodes() timeline_output_->SetCanBeDeleted(false); AddNode(timeline_output_); - video_renderer_processor_ = new VideoRendererProcessor(); - video_renderer_processor_->SetCanBeDeleted(false); - AddNode(video_renderer_processor_); - viewer_output_ = new ViewerOutput(); viewer_output_->SetCanBeDeleted(false); AddNode(viewer_output_); @@ -77,21 +72,15 @@ void Sequence::add_default_nodes() audio_track_output_->SetCanBeDeleted(false); AddNode(audio_track_output_); - // Connect track to renderer - NodeParam::ConnectEdge(video_track_output_->texture_output(), video_renderer_processor_->texture_input()); - - // Connect renderer to viewer - NodeParam::ConnectEdge(video_renderer_processor_->texture_output(), viewer_output_->texture_input()); + // Connect track to viewer + NodeParam::ConnectEdge(video_track_output_->texture_output(), viewer_output_->texture_input()); // Connect track to timeline NodeParam::ConnectEdge(video_track_output_->track_output(), timeline_output_->track_input(kTrackTypeVideo)); NodeParam::ConnectEdge(audio_track_output_->track_output(), timeline_output_->track_input(kTrackTypeAudio)); - // Connect timeline end point to renderer - NodeParam::ConnectEdge(timeline_output_->length_output(), video_renderer_processor_->length_input()); - // Update the timebase on these nodes - video_renderer_processor_->SetCacheName(name()); + //video_renderer_processor_->SetCacheName(name()); set_video_time_base(video_time_base_); update_video_parameters(); } @@ -162,9 +151,6 @@ void Sequence::set_video_time_base(const rational &time_base) if (viewer_output_ != nullptr) viewer_output_->SetTimebase(video_time_base_); - - if (video_renderer_processor_ != nullptr) - video_renderer_processor_->SetTimebase(video_time_base_); } const rational &Sequence::audio_time_base() @@ -200,7 +186,7 @@ void Sequence::SetDefaultParameters() void Sequence::update_video_parameters() { - if (video_renderer_processor_ != nullptr) { + /*if (video_renderer_processor_ != nullptr) { // Set renderer's parameters based on sequence's parameters video_renderer_processor_->SetParameters(video_width_, video_height_, @@ -210,7 +196,7 @@ void Sequence::update_video_parameters() // Set the "cache name" only here to aid the cache ID's uniqueness video_renderer_processor_->SetCacheName(name()); - } + }*/ if (viewer_output_ != nullptr) { viewer_output_->SetViewerSize(video_width_, video_height_); diff --git a/app/project/item/sequence/sequence.h b/app/project/item/sequence/sequence.h index dac92c432..7b525ab0a 100644 --- a/app/project/item/sequence/sequence.h +++ b/app/project/item/sequence/sequence.h @@ -78,7 +78,6 @@ private: void update_video_parameters(); TimelineOutput* timeline_output_; - VideoRendererProcessor* video_renderer_processor_; ViewerOutput* viewer_output_; TrackOutput* video_track_output_; TrackOutput* audio_track_output_; diff --git a/app/render/video/videorenderer.cpp b/app/render/video/videorenderer.cpp index 7015f163c..1cc67be79 100644 --- a/app/render/video/videorenderer.cpp +++ b/app/render/video/videorenderer.cpp @@ -29,47 +29,28 @@ #include #include "common/filefunctions.h" +#include "render/gl/functions.h" +#include "render/gl/shadergenerators.h" #include "render/pixelservice.h" -VideoRendererProcessor::VideoRendererProcessor() : +VideoRendererProcessor::VideoRendererProcessor(QObject *parent) : + QObject(parent), started_(false), width_(0), height_(0), divider_(1), caching_(false), - starting_(false) + push_time_(-1), + starting_(false), + viewer_node_(nullptr) { - texture_input_ = new NodeInput("tex_in"); - texture_input_->add_data_input(NodeInput::kTexture); - AddParameter(texture_input_); - - length_input_ = new NodeInput("length_in"); - length_input_->add_data_input(NodeInput::kRational); - AddParameter(length_input_); - - texture_output_ = new NodeOutput("tex_out"); - texture_output_->set_data_type(NodeInput::kTexture); - AddParameter(texture_output_); + // FIXME: Cache name should actually be the name of the sequence + SetCacheName("Test"); } -QString VideoRendererProcessor::Name() +VideoRendererProcessor::~VideoRendererProcessor() { - return tr("Video Renderer"); -} - -QString VideoRendererProcessor::Category() -{ - return tr("Processor"); -} - -QString VideoRendererProcessor::Description() -{ - return tr("A multi-threaded OpenGL hardware-accelerated node compositor."); -} - -QString VideoRendererProcessor::id() -{ - return "org.olivevideoeditor.Olive.renderervenus"; + Stop(); } void VideoRendererProcessor::SetCacheName(const QString &s) @@ -80,69 +61,18 @@ void VideoRendererProcessor::SetCacheName(const QString &s) GenerateCacheIDInternal(); } -QVariant VideoRendererProcessor::Value(NodeOutput* output, const rational& time) +void VideoRendererProcessor::InvalidateCache(const rational &start_range, const rational &end_range) { - if (output == texture_output_) { - if (!texture_input_->IsConnected()) { - // Nothing is connected - nothing to show or render - return 0; - } - - if (cache_id_.isEmpty()) { - qWarning() << "RendererProcessor has no cache ID"; - return 0; - } - - if (timebase_.isNull()) { - qWarning() << "RendererProcessor has no timebase"; - return 0; - } - - // Find frame in map - if (time_hash_map_.contains(time)) { - QString fn = CachePathName(time_hash_map_[time]); - - if (QFileInfo::exists(fn)) { - auto in = OIIO::ImageInput::open(fn.toStdString()); - - if (in) { - in->read_image(PixelService::GetPixelFormatInfo(format_).oiio_desc, cache_frame_load_buffer_.data()); - - in->close(); - - master_texture_->Upload(cache_frame_load_buffer_.data()); - - return QVariant::fromValue(master_texture_); - } else { - qWarning() << "OIIO Error:" << OIIO::geterror().c_str(); - } - } - } - } - - return 0; -} - -void VideoRendererProcessor::Release() -{ - Stop(); -} - -void VideoRendererProcessor::InvalidateCache(const rational &start_range, const rational &end_range, NodeInput *from) -{ - Q_UNUSED(from) - if (timebase_.isNull()) { return; } - //ClearCachedValuesInParameters(start_range, end_range); - texture_input_->ClearCachedValue(); - length_input_->ClearCachedValue(); - // Adjust range to min/max values rational start_range_adj = qMax(rational(0), start_range); - rational end_range_adj = qMin(length_input()->get_value(0).value(), end_range); + + // FIXME: Needs real length value + //rational end_range_adj = qMin(length_input()->get_value(0).value(), end_range); + rational end_range_adj = qMin(rational(60), end_range); qDebug() << "Cache invalidated between" << start_range_adj.toDouble() @@ -157,7 +87,7 @@ void VideoRendererProcessor::InvalidateCache(const rational &start_range, const for (rational r=true_start_range;r<=end_range_adj;r+=timebase_) { // Try to order the queue from closest to the playhead to furthest - rational last_time = texture_output()->LastRequestedTime(); + rational last_time = last_time_requested_; rational diff = r - last_time; @@ -313,6 +243,11 @@ void VideoRendererProcessor::Start() master_texture_ = std::make_shared(); master_texture_->Create(ctx, effective_width_, effective_height_, format_); + // Create internal FBO for copying textures + copy_buffer_.Create(ctx); + copy_buffer_.Attach(master_texture_); + copy_pipeline_ = olive::ShaderGenerator::DefaultPipeline(); + cache_frame_load_buffer_.resize(PixelService::GetBufferSize(format_, effective_width_, effective_height_)); started_ = true; @@ -336,7 +271,9 @@ void VideoRendererProcessor::Stop() } threads_.clear(); + copy_buffer_.Destroy(); master_texture_ = nullptr; + copy_pipeline_ = nullptr; cache_frame_load_buffer_.clear(); } @@ -362,7 +299,7 @@ void VideoRendererProcessor::GenerateCacheIDInternal() void VideoRendererProcessor::CacheNext() { - if (cache_queue_.isEmpty() || !texture_input_->IsConnected() || caching_) { + if (cache_queue_.isEmpty() || viewer_node_ == nullptr || caching_) { return; } @@ -373,7 +310,7 @@ void VideoRendererProcessor::CacheNext() qDebug() << "Caching" << cache_frame.toDouble(); - threads_.first()->Queue(NodeDependency(texture_input_->get_connected_output(), cache_frame), true, false); + threads_.first()->Queue(NodeDependency(viewer_node_->texture_input()->get_connected_output(), cache_frame), true, false); caching_ = true; } @@ -452,10 +389,20 @@ void VideoRendererProcessor::ThreadCallback(RenderTexturePtr texture, const rati } // If the connected output is using this time, signal it to update - if (texture_output_->IsConnected() - && texture_output_->LastRequestedTime() == time) { - texture_output_->push_value(QVariant::fromValue(texture), time); - SendInvalidateCache(time, time); + if (last_time_requested_ == time) { + copy_buffer_.Bind(); + texture->Bind(); + + QOpenGLContext::currentContext()->functions()->glViewport(0, 0, master_texture_->width(), master_texture_->height()); + + olive::gl::Blit(copy_pipeline_); + + texture->Release(); + copy_buffer_.Release(); + + push_time_ = time; + + emit CachedFrameReady(time); } CacheNext(); @@ -481,11 +428,7 @@ void VideoRendererProcessor::ThreadSkippedFrame(const rational& time, const QByt DownloadThreadComplete(hash); // Signal output to update value - if (texture_output_->IsConnected() - && texture_output_->LastRequestedTime() == time) { - texture_output_->ClearCachedValue(); - SendInvalidateCache(time, time); - } + emit CachedFrameReady(time); } CacheNext(); @@ -526,17 +469,73 @@ RenderInstance *VideoRendererProcessor::CurrentInstance() return nullptr; } -NodeInput *VideoRendererProcessor::texture_input() +RenderTexturePtr VideoRendererProcessor::GetCachedFrame(const rational &time) { - return texture_input_; + last_time_requested_ = time; + + if (push_time_ >= 0) { + rational temp_push_time = push_time_; + push_time_ = -1; + + if (time == temp_push_time) { + return master_texture_; + } + } + + if (viewer_node_ == nullptr) { + // Nothing is connected - nothing to show or render + return nullptr; + } + + if (cache_id_.isEmpty()) { + qWarning() << "RendererProcessor has no cache ID"; + return nullptr; + } + + if (timebase_.isNull()) { + qWarning() << "RendererProcessor has no timebase"; + return nullptr; + } + + // Find frame in map + if (time_hash_map_.contains(time)) { + QString fn = CachePathName(time_hash_map_[time]); + + if (QFileInfo::exists(fn)) { + auto in = OIIO::ImageInput::open(fn.toStdString()); + + if (in) { + in->read_image(PixelService::GetPixelFormatInfo(format_).oiio_desc, cache_frame_load_buffer_.data()); + + in->close(); + + master_texture_->Upload(cache_frame_load_buffer_.data()); + + return master_texture_; + } else { + qWarning() << "OIIO Error:" << OIIO::geterror().c_str(); + } + } + } + + return nullptr; } -NodeInput *VideoRendererProcessor::length_input() +void VideoRendererProcessor::SetViewerNode(ViewerOutput *viewer) { - return length_input_; -} + if (viewer_node_ != nullptr) { + disconnect(viewer_node_, SIGNAL(TextureChangedBetween(const rational&, const rational&)), this, SLOT(InvalidateCache(const rational&, const rational&))); + } -NodeOutput *VideoRendererProcessor::texture_output() -{ - return texture_output_; + viewer_node_ = viewer; + + if (viewer_node_ != nullptr) { + connect(viewer_node_, SIGNAL(TextureChangedBetween(const rational&, const rational&)), this, SLOT(InvalidateCache(const rational&, const rational&))); + + // FIXME: Hardcoded format and mode + SetParameters(viewer_node_->ViewerWidth(), + viewer_node_->ViewerHeight(), + olive::PIX_FMT_RGBA16F, + olive::kOffline); + } } diff --git a/app/render/video/videorenderer.h b/app/render/video/videorenderer.h index e7eab905a..d12f3f125 100644 --- a/app/render/video/videorenderer.h +++ b/app/render/video/videorenderer.h @@ -24,7 +24,7 @@ #include #include -#include "node/node.h" +#include "node/output/viewer/viewer.h" #include "render/pixelformat.h" #include "render/rendermodes.h" #include "videorendererdownloadthread.h" @@ -33,7 +33,7 @@ /** * @brief A multithreaded OpenGL based renderer for node systems */ -class VideoRendererProcessor : public Node +class VideoRendererProcessor : public QObject { Q_OBJECT public: @@ -43,19 +43,12 @@ public: * Constructing a Renderer object will not start any threads/backend on its own. Use Start() to do this and Stop() * when the Renderer is about to be destroyed. */ - VideoRendererProcessor(); + VideoRendererProcessor(QObject* parent); - virtual QString Name() override; - virtual QString Category() override; - virtual QString Description() override; - virtual QString id() override; + virtual ~VideoRendererProcessor() override; void SetCacheName(const QString& s); - virtual void Release() override; - - virtual void InvalidateCache(const rational &start_range, const rational &end_range, NodeInput *from = nullptr) override; - void SetTimebase(const rational& timebase); /** @@ -109,14 +102,12 @@ public: static RenderInstance* CurrentInstance(); - NodeInput* texture_input(); + RenderTexturePtr GetCachedFrame(const rational& time); - NodeInput* length_input(); + void SetViewerNode(ViewerOutput* viewer); - NodeOutput* texture_output(); - -protected: - virtual QVariant Value(NodeOutput* output, const rational& time) override; +signals: + void CachedFrameReady(const rational& time); private: struct HashTimeMapping { @@ -165,12 +156,6 @@ private: */ bool started_; - NodeInput* texture_input_; - - NodeInput* length_input_; - - NodeOutput* texture_output_; - int width_; int height_; @@ -184,6 +169,8 @@ private: olive::RenderMode mode_; + rational last_time_requested_; + rational timebase_; double timebase_dbl_; @@ -199,6 +186,10 @@ private: int last_download_thread_; RenderTexturePtr master_texture_; + rational push_time_; + + RenderFramebuffer copy_buffer_; + ShaderPtr copy_pipeline_; QMap time_hash_map_; @@ -209,7 +200,11 @@ private: bool starting_; + ViewerOutput* viewer_node_; + private slots: + void InvalidateCache(const rational &start_range, const rational &end_range); + void ThreadCallback(RenderTexturePtr texture, const rational& time, const QByteArray& hash); void ThreadRequestSibling(NodeDependency dep); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index ab2653618..7aa98d040 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -82,6 +82,10 @@ ViewerWidget::ViewerWidget(QWidget *parent) : // FIXME: Magic number ruler_->SetScale(48.0); + + // Start background renderers + video_renderer_ = new VideoRendererProcessor(this); + connect(video_renderer_, SIGNAL(CachedFrameReady(const rational&)), this, SLOT(RendererCachedFrame(const rational&))); } void ViewerWidget::SetTimebase(const rational &r) @@ -93,6 +97,8 @@ void ViewerWidget::SetTimebase(const rational &r) controls_->SetTimebase(r); playback_timer_.setInterval(qFloor(r.toDouble())); + + video_renderer_->SetTimebase(r); } const double &ViewerWidget::scale() @@ -136,7 +142,6 @@ void ViewerWidget::ConnectViewerNode(ViewerOutput *node) SetTimebase(0); disconnect(viewer_node_, SIGNAL(TimebaseChanged(const rational&)), this, SLOT(SetTimebase(const rational&))); - disconnect(viewer_node_, SIGNAL(TextureChangedBetween(const rational&, const rational&)), this, SLOT(ViewerNodeChangedBetween(const rational&, const rational&))); disconnect(viewer_node_, SIGNAL(SizeChanged(int, int)), this, SLOT(SizeChangedSlot(int, int))); // Effectively disables the viewer and clears the state @@ -152,11 +157,12 @@ void ViewerWidget::ConnectViewerNode(ViewerOutput *node) SetTimebase(viewer_node_->Timebase()); connect(viewer_node_, SIGNAL(TimebaseChanged(const rational&)), this, SLOT(SetTimebase(const rational&))); - connect(viewer_node_, SIGNAL(TextureChangedBetween(const rational&, const rational&)), this, SLOT(ViewerNodeChangedBetween(const rational&, const rational&))); connect(viewer_node_, SIGNAL(SizeChanged(int, int)), this, SLOT(SizeChangedSlot(int, int))); SizeChangedSlot(viewer_node_->ViewerWidth(), viewer_node_->ViewerHeight()); } + + video_renderer_->SetViewerNode(viewer_node_); } void ViewerWidget::DisconnectViewerNode() @@ -193,7 +199,7 @@ void ViewerWidget::UpdateTextureFromNode(const rational& time) if (viewer_node_ == nullptr) { SetTexture(nullptr); } else { - SetTexture(viewer_node_->GetTexture(time)); + SetTexture(video_renderer_->GetCachedFrame(time)); } } @@ -222,14 +228,14 @@ void ViewerWidget::PlayInternal(int speed) void ViewerWidget::PushScrubbedAudio() { - // FIXME: Test code if (Config::Current()["AudioScrubbing"].toBool() && !IsPlaying()) { + // FIXME: Test code int size_of_sample = qFloor(2 * 48000 * time_base_dbl_) * static_cast(sizeof(float)); test_file_.seek(static_cast(qFloor(GetTime().toDouble() * 48000 * 2)) * static_cast(sizeof(float))); QByteArray frame_audio = test_file_.read(size_of_sample); AudioManager::instance()->PushToOutput(frame_audio); + // End test code } - // End test code } void ViewerWidget::RulerTimeChange(int64_t i) @@ -333,9 +339,9 @@ void ViewerWidget::PlaybackTimerUpdate() SetTime(current_time); } -void ViewerWidget::ViewerNodeChangedBetween(const rational &start, const rational &end) +void ViewerWidget::RendererCachedFrame(const rational &time) { - if (GetTime() >= start && GetTime() <= end) { + if (GetTime() == time) { UpdateTextureFromNode(GetTime()); } } diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 5817a95c9..378e58768 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -30,6 +30,7 @@ #include "common/rational.h" #include "node/output/viewer/viewer.h" +#include "render/video/videorenderer.h" #include "viewerglwidget.h" #include "viewersizer.h" #include "widget/playbackcontrols/playbackcontrols.h" @@ -109,6 +110,8 @@ private: void PushScrubbedAudio(); + VideoRendererProcessor* video_renderer_; + ViewerSizer* sizer_; ViewerGLWidget* gl_widget_; @@ -141,7 +144,7 @@ private slots: void PlaybackTimerUpdate(); - void ViewerNodeChangedBetween(const rational& start, const rational& end); + void RendererCachedFrame(const rational& time); void SizeChangedSlot(int width, int height);