From 3845c31b372d0eaa6c7afcf9b4a5dc41f495ca62 Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Sat, 16 May 2026 16:17:21 +0800 Subject: [PATCH] =?UTF-8?q?perf(plugin):=20eliminate=20GPU=E2=86=94CPU=20p?= =?UTF-8?q?ing-pong=20in=20OFX=20render=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit removes redundant readbacks and uploads in the OpenFX plugin pipeline, achieving zero-copy rendering for GL-capable plugins and reducing CPU path overhead. PluginRenderer (OpenGL path): - Skip ReadbackTextureToFrame + ConvertFrameIfNeeded + Upload after plugin render. The destination texture is already valid on GPU. - Pass readback_cpu=false to setInputTexture() so input textures are provided via loadTexture() (GL texture IDs) instead of being downloaded to CPU Image buffers. - Remove duplicate ReadbackTextureToFrame block between getClipPreferences and the second setInputTexture call. OliveClipInstance: - Add optional bool readback_cpu=true to setInputTexture(). When false, only params and input_textures_ are updated; CPU readback and memcpy into Image are skipped. - Add pruneImagesCache() to prevent unbounded growth of images_. Input clips are limited to 8 cached frames; output clips are left untouched. Micro-optimizations: - Replace per-row memcpy loops with single block memcpy when src/dst strides are contiguous (common case in Olive pipeline). - Remove dead GL_PREAMBLE macro definition. fix(viewer): resolve playback head lag, frozen frames, and pause delay Three playback pipeline behavioral issues are fixed: 1. Prequeue blocked playhead start: Reduce kVideoPlaybackInterval from 0.5s to 0.1s. This lowers prequeue length from 15–30 frames to 3–6 frames, so the playback timer starts much sooner after pressing Play. 2. Frozen display during playback: Relax the hard frame-drop logic in RendererGeneratedFrameForQueue. When the queue has fewer than 2 frames, keep late frames instead of dropping them, preventing the viewer from freezing entirely when rendering cannot keep up with playback speed. 3. Delayed frame update after pause: Cancel in-flight render tickets in PauseInternal() before deleting queue watchers. Previously the render thread continued processing stale playback frames, blocking the single-frame render requested by UpdateTextureFromNode(). --- app/pluginSupport/OliveClip.cpp | 36 +++++++++++++-- app/pluginSupport/OliveClip.h | 8 +++- app/render/plugin/pluginrenderer.cpp | 66 +++++++++++----------------- app/widget/viewer/viewer.cpp | 15 ++++++- tests/gtest/plugin_ofx_misc_test.cpp | 16 +++---- 5 files changed, 86 insertions(+), 55 deletions(-) diff --git a/app/pluginSupport/OliveClip.cpp b/app/pluginSupport/OliveClip.cpp index 864890a84..9629d3d22 100644 --- a/app/pluginSupport/OliveClip.cpp +++ b/app/pluginSupport/OliveClip.cpp @@ -559,6 +559,21 @@ void olive::plugin::OliveClipInstance::setDefaultRegionOfDefinition( defaultRegionOfDefinitions_ = regionOfDefinition; } +void olive::plugin::OliveClipInstance::pruneImagesCache() +{ + // Do not prune output clip images; they may have external references + // added by getImage()/addReference() and are typically single-frame. + if (name_ == kOfxImageEffectOutputClipName) { + return; + } + while (images_.size() > kMaxInputImageCache) { + auto it = images_.begin(); + Image *img = it.value(); + images_.erase(it); + delete img; + } +} + void olive::plugin::OliveClipInstance::setParams(const VideoParams ¶ms) { params_ = params; @@ -567,7 +582,7 @@ void olive::plugin::OliveClipInstance::setParams(const VideoParams ¶ms) setComponents(getUnmappedComponents()); } -void olive::plugin::OliveClipInstance::setInputTexture(TexturePtr texture, OfxTime time){ +void olive::plugin::OliveClipInstance::setInputTexture(TexturePtr texture, OfxTime time, bool readback_cpu){ if (!texture) { return; } @@ -593,6 +608,14 @@ void olive::plugin::OliveClipInstance::setInputTexture(TexturePtr texture, OfxTi input_textures_.insert(time, texture); #endif + // In OpenGL render path, skip CPU readback entirely. + // The plugin will fetch input via loadTexture() using GPU texture IDs. + // If the plugin falls back to getImage(), it will be created on-demand + // in getImage() with zero-initialized data. + if (!readback_cpu) { + return; + } + AVFramePtr frame = texture->frame(); if (!frame || !frame->data[0]) { frame = ReadbackTextureToFrame(texture, params_); @@ -620,6 +643,7 @@ void olive::plugin::OliveClipInstance::setInputTexture(TexturePtr texture, OfxTi regionOfDefinition, false); images_.insert(time, image); } + pruneImagesCache(); uint8_t *dst = (uint8_t*)image->data(); if (!dst) { @@ -679,9 +703,13 @@ copy_pixels: int copy_height = std::min(image->height(), src_frame->height); const uint8_t *src = src_frame->data[0]; - for (int y = 0; y < copy_height; ++y) { - std::memcpy(dst + y * dst_row_bytes, src + y * src_row_bytes, - copy_bytes); + if (dst_row_bytes == src_row_bytes && src_row_bytes == copy_bytes) { + std::memcpy(dst, src, copy_bytes * copy_height); + } else { + for (int y = 0; y < copy_height; ++y) { + std::memcpy(dst + y * dst_row_bytes, src + y * src_row_bytes, + copy_bytes); + } } diff --git a/app/pluginSupport/OliveClip.h b/app/pluginSupport/OliveClip.h index acea6d125..12fb7bd49 100644 --- a/app/pluginSupport/OliveClip.h +++ b/app/pluginSupport/OliveClip.h @@ -70,12 +70,18 @@ public: const OfxRectD *optionalBounds) override; # endif - void setInputTexture(TexturePtr texture, OfxTime time); + void setInputTexture(TexturePtr texture, OfxTime time, bool readback_cpu = true); void setOutputTexture(TexturePtr texture, OfxTime time); // Get the plugin-preferred VideoParams based on base class _pixelDepth/_components VideoParams getPluginPreferredParams() const; + // Prune old entries from the images_ cache to prevent unbounded growth. + // Output clip images are not pruned (they are typically single-frame). + void pruneImagesCache(); + + static constexpr int kMaxInputImageCache = 8; + private: VideoParams params_; diff --git a/app/render/plugin/pluginrenderer.cpp b/app/render/plugin/pluginrenderer.cpp index c9810f6d9..cb9b96bde 100644 --- a/app/render/plugin/pluginrenderer.cpp +++ b/app/render/plugin/pluginrenderer.cpp @@ -40,7 +40,6 @@ #include #include #include -#define GL_PREAMBLE //QMutexLocker __l(&global_opengl_mutex); #include "pluginrenderer.h" #include "core.h" #include "undo/undostack.h" @@ -700,10 +699,14 @@ static olive::AVFramePtr create_avframe_from_ofx_image(OFX::Host::ImageEffect::I } const int copy_bytes = width * bytes_per_pixel; - for (int y = 0; y < height; ++y) { - std::memcpy(frame->data[0] + y * frame->linesize[0], - src + y * row_bytes, - copy_bytes); + if (frame->linesize[0] == row_bytes && row_bytes == copy_bytes) { + std::memcpy(frame->data[0], src, copy_bytes * height); + } else { + for (int y = 0; y < height; ++y) { + std::memcpy(frame->data[0] + y * frame->linesize[0], + src + y * row_bytes, + copy_bytes); + } } return frame; @@ -789,10 +792,15 @@ static olive::AVFramePtr create_avframe_from_ofx_image_with_params( src_frame->format = src_fmt; if (av_frame_get_buffer(src_frame.get(), 0) >= 0) { - // Copy source data row by row - for (int y = 0; y < height; ++y) { - memcpy(src_frame->data[0] + y * src_frame->linesize[0], - src + y * row_bytes, width * src_bytes_per_pixel); + // Copy source data row by row (or as a single block if strides match) + const int copy_bytes = width * src_bytes_per_pixel; + if (src_frame->linesize[0] == row_bytes && row_bytes == copy_bytes) { + memcpy(src_frame->data[0], src, copy_bytes * height); + } else { + for (int y = 0; y < height; ++y) { + memcpy(src_frame->data[0] + y * src_frame->linesize[0], + src + y * row_bytes, copy_bytes); + } } // Convert to destination format return ConvertFrameIfNeeded(src_frame, params, renderer); @@ -1471,7 +1479,9 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: if (is_usable_input(input_tex)) { input_textures[entry.first] = input_tex; olive::VideoParams params = input_tex->params(); - input_clip->setInputTexture(input_tex, frame); + // First pass: set params only, no CPU readback yet. + // Readback will happen below (CPU path) or be skipped entirely (GL path). + input_clip->setInputTexture(input_tex, frame, false); input_clips[entry.first] = input_clip; } } @@ -1549,11 +1559,6 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: } const QString clip_key = QString::fromStdString(entry.first); TexturePtr input_tex = input_textures[entry.first]; - if (!use_opengl) { - AVFramePtr ptr = - ReadbackTextureToFrame(input_tex, input_tex->params()); - input_tex->handleFrame(ptr); - } if (is_usable_input(input_tex)) { // Query the plugin descriptor for supported pixel depths and pick // the best one according to our priority: F32 > U16 > U8 > F16. @@ -1572,7 +1577,7 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: } } - input_clip->setInputTexture(input_tex, frame); + input_clip->setInputTexture(input_tex, frame, !use_opengl); OfxRectD rod; rod.x1 = 0; rod.y1 = 0; @@ -1738,34 +1743,15 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: << PluginIdForInstance(instance); } } else { - AVFramePtr frame_ptr = - ReadbackTextureToFrame(destination, destination_params); + // OpenGL path: plugin has already rendered directly into the destination + // texture via FBO/GL. No CPU readback or conversion needed. #ifdef OFX_SUPPORTS_OPENGLRENDER DetachOutputTexture(); instance->contextDetachedAction(); #endif - if (frame_ptr && destination) { - AVFramePtr converted = - ConvertFrameIfNeeded(frame_ptr, destination_params, this); - const AVPixelFormat expected_fmt = - GetDestinationAVPixelFormat(destination_params); - destination->handleFrame(converted); - if (destination->renderer() && converted && converted->data[0] && - (expected_fmt == AV_PIX_FMT_NONE || - converted->format == expected_fmt)) { - int linesize_pixels = LinesizeToPixels(destination_params, - converted->linesize[0]); - if (linesize_pixels <= 0) { - linesize_pixels = destination_params.effective_width(); - } - destination->Upload(converted->data[0], linesize_pixels); - } else if (destination->renderer() && converted && - converted->data[0]) { - qWarning().noquote() - << "OFX output pixel format mismatch for plugin=" - << PluginIdForInstance(instance); - } - } + instance->endRenderAction(frame, numFramesToRender, 1.0, interactive, + renderScale, true, interactive); + return; } instance->endRenderAction(frame, numFramesToRender, 1.0, interactive, renderScale, true,interactive diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 09e030a1a..23229b0a3 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -63,7 +63,7 @@ QVector ViewerWidget::instances_; // changing values. 1/4 second seems to be a good middleground. const rational ViewerWidget::kAudioPlaybackInterval = rational(1, 4); -const rational kVideoPlaybackInterval = rational(1, 2); +const rational kVideoPlaybackInterval = rational(1, 10); ViewerWidget::ViewerWidget(ViewerDisplayWidget *display, QWidget *parent) : super(false, true, parent) @@ -1146,6 +1146,12 @@ void ViewerWidget::PauseInternal() dw->Pause(); } + // Cancel in-flight render tickets before deleting watchers, + // otherwise the render thread keeps working on stale frames + // and blocks the single-frame render requested by UpdateTextureFromNode(). + foreach (RenderTicketWatcher *watcher, queue_watchers_) { + watcher->Cancel(); + } qDeleteAll(queue_watchers_); queue_watchers_.clear(); RenderManager::instance()->GetCacher()->ClearSingleFrameRenders(); @@ -1480,7 +1486,12 @@ void ViewerWidget::RendererGeneratedFrameForQueue() static_cast(playback_step)); if (start_ms > 0 && (now_ms - start_ms) > frame_interval_ms) { - drop_frame = true; + // If the queue is nearly empty, keep the frame anyway + // to prevent the viewer from freezing entirely when + // rendering can't keep up with playback speed. + if (display_widget_->queue()->size() >= 2) { + drop_frame = true; + } } rational ts = watcher->property("time").value(); diff --git a/tests/gtest/plugin_ofx_misc_test.cpp b/tests/gtest/plugin_ofx_misc_test.cpp index 68b6cd282..c08ae5e1c 100644 --- a/tests/gtest/plugin_ofx_misc_test.cpp +++ b/tests/gtest/plugin_ofx_misc_test.cpp @@ -604,25 +604,25 @@ TEST(PluginMisc, ListAvailablePlugins) SUCCEED(); } -TEST(PluginMisc, CImgGuided_MultiInput) +TEST(PluginMisc, CImgBilateralGuided_MultiInput) { if (ShouldSkipTest()) GTEST_SKIP() << "OFX integration test not enabled"; - // CImgGuided is a multi-input plugin (Source + Mask). + // CImgBilateralGuided is a multi-input plugin (Source + Guide). // This test verifies that connecting both inputs does not trigger // the frame-rate mismatch exception in setupClipPreferencesArgs. VideoParams params(320, 240, core::PixelFormat::F32, 4); TexturePtr source = CreateSolidTexture(params, 0x80); - TexturePtr mask = CreateSolidTexture(params, 0x40); + TexturePtr guide = CreateSolidTexture(params, 0x40); ASSERT_NE(source, nullptr); - ASSERT_NE(mask, nullptr); + ASSERT_NE(guide, nullptr); NodeValueRow row; row.insert(QString::fromStdString(kOfxImageEffectSimpleSourceClipName), NodeValue(NodeValue::kTexture, source)); - row.insert(QStringLiteral("Mask"), - NodeValue(NodeValue::kTexture, mask)); - bool result = RenderPlugin("net.sf.cimg.CImgGuided", params, row, true); - EXPECT_TRUE(result) << "CImgGuided plugin should produce output with both Source and Mask connected"; + row.insert(QStringLiteral("Guide"), + NodeValue(NodeValue::kTexture, guide)); + bool result = RenderPlugin("net.sf.cimg.CImgBilateralGuided", params, row, true); + EXPECT_TRUE(result) << "CImgBilateralGuided plugin should produce output with both Source and Guide connected"; } } // namespace test