From f6211f97a51637144e664674852e07c4f23de927 Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Sat, 20 Jun 2026 10:20:35 +0800 Subject: [PATCH] Fix render worker reuse and stabilize out-of-process video rendering --- app/codec/ffmpeg/ffmpegdecoder.cpp | 26 + app/config/config.cpp | 9 +- app/node/generator/solid/solid.cpp | 4 + app/node/traverser.cpp | 4 +- app/render/backend/dynamicrenderer.cpp | 12 +- app/render/backend/dynamicrenderer.h | 2 + app/render/colormanagement.cpp | 16 +- app/render/ipc/frameslotpool.h | 5 +- app/render/opengl/openglrenderer.cpp | 77 ++- app/render/previewautocacher.cpp | 24 +- app/render/renderer.h | 5 + app/render/rendermanager.cpp | 27 +- app/render/rendermanager.h | 1 - app/render/renderprocessor.cpp | 55 +- app/render/renderworkerpool.cpp | 608 +++++++++++++----- app/render/renderworkerpool.h | 50 +- app/render/vulkan/vulkanrenderer.cpp | 442 +++++++++++-- app/render/vulkan/vulkanrenderer.h | 25 + app/render/worker/workermain.cpp | 65 +- app/widget/viewer/viewer.cpp | 37 +- app/widget/viewer/viewer.h | 1 + app/widget/viewer/viewerdisplay.cpp | 254 ++++++-- app/widget/viewer/viewerdisplay.h | 14 + docs/zh/render-process-isolation-plan.md | 2 +- ...olor-audio-performance-manual-test-plan.md | 39 +- tests/gtest/CMakeLists.txt | 1 + tests/gtest/dynamic_render_backend_test.cpp | 59 +- tests/gtest/render_worker_footage_test.cpp | 440 +++++++++++++ 28 files changed, 1918 insertions(+), 386 deletions(-) create mode 100644 tests/gtest/render_worker_footage_test.cpp diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 64774046f..5a486a180 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -21,6 +21,10 @@ #include "ffmpegdecoder.h" +extern "C" { +#include +} + namespace olive { static FramePtr CopyPackedAVFrameToFrame(const AVFramePtr &src, @@ -491,6 +495,28 @@ FramePtr FFmpegDecoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p) return nullptr; } + // sws_scale does not initialize the alpha channel when converting + // from non-alpha source formats (e.g. YUV). av_frame_get_buffer + // zero-initializes the destination, leaving alpha at 0. The color + // management shader later multiplies RGB by alpha, producing black. + // Ensure alpha is opaque for source formats that have no alpha. + const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get( + static_cast(f->format)); + if (src_desc && !(src_desc->flags & AV_PIX_FMT_FLAG_ALPHA)) { + const int bpc = (dest->format == AV_PIX_FMT_RGBA) ? 1 : 2; + const int stride = dest->linesize[0]; + for (int y = 0; y < dest->height; ++y) { + uchar *row = dest->data[0] + y * stride; + for (int x = 0; x < dest->width; ++x) { + if (bpc == 1) { + row[x * 4 + 3] = 0xFF; + } else { + *reinterpret_cast(row + x * 8 + 6) = 0xFFFF; + } + } + } + } + return CopyPackedAVFrameToFrame(dest, dest->format == AV_PIX_FMT_RGBA ? PixelFormat::U8 diff --git a/app/config/config.cpp b/app/config/config.cpp index 56056b144..32169dd59 100644 --- a/app/config/config.cpp +++ b/app/config/config.cpp @@ -146,8 +146,6 @@ void Config::SetDefaults() SetEntryInternal(QStringLiteral("UseGLFinish"), NodeValue::kBoolean, false); SetEntryInternal(QStringLiteral("GraphicsBackend"), NodeValue::kText, QStringLiteral("opengl")); - SetEntryInternal(QStringLiteral("RenderProcessIsolationEnabled"), - NodeValue::kBoolean, false); SetEntryInternal(QStringLiteral("TimelineThumbnailMode"), NodeValue::kInt, Timeline::kThumbnailInOut); @@ -334,8 +332,13 @@ void Config::Load() } if (reader.hasError()) { + // Config::Load() is called before Core (and therefore the main window) + // is constructed, so we cannot use Core::instance()->main_window() as + // the message box parent. Passing nullptr creates a top-level dialog. + QWidget *parent = Core::instance() ? Core::instance()->main_window() + : nullptr; QMessageBox::critical( - Core::instance()->main_window(), + parent, QCoreApplication::translate("Config", "Error loading settings"), QCoreApplication::translate( "Config", diff --git a/app/node/generator/solid/solid.cpp b/app/node/generator/solid/solid.cpp index 11e7e3137..3c857d015 100644 --- a/app/node/generator/solid/solid.cpp +++ b/app/node/generator/solid/solid.cpp @@ -66,6 +66,10 @@ void SolidGenerator::Value(const NodeValueRow &value, const NodeGlobals &globals, NodeValueTable *table) const { + Color c = value[kColorInput].toColor(); + fprintf(stderr, + "SolidGenerator::Value color=%f %f %f %f\n", + c.red(), c.green(), c.blue(), c.alpha()); table->Push(NodeValue::kTexture, Texture::Job(globals.vparams(), ShaderJob(value)), this); } diff --git a/app/node/traverser.cpp b/app/node/traverser.cpp index d8b0fc6ff..5a7fed0a8 100644 --- a/app/node/traverser.cpp +++ b/app/node/traverser.cpp @@ -496,7 +496,9 @@ void NodeTraverser::ResolveJobs(NodeValue &val) GetCacheVideoParams().format()); tex = CreateTexture(managed_params); - ProcessVideoFootage(tex, fj, footage_time); + if (tex) { + ProcessVideoFootage(tex, fj, footage_time); + } } val.set_value(tex); diff --git a/app/render/backend/dynamicrenderer.cpp b/app/render/backend/dynamicrenderer.cpp index a81eb3fd5..eaeb8ab55 100644 --- a/app/render/backend/dynamicrenderer.cpp +++ b/app/render/backend/dynamicrenderer.cpp @@ -97,7 +97,12 @@ bool DynamicRenderer::Load() return false; } - handle_ = create_(this->parent()); + // Pass this (rather than this->parent()) so the backend renderer becomes a + // child QObject of the adapter. That ensures it follows DynamicRenderer when + // the latter is moved to the render thread; otherwise it stays in the thread + // where Load() was called and every GL operation is rejected as "wrong + // thread", producing a black screen. + handle_ = create_(this); if (!handle_) { library_.unload(); return false; @@ -319,6 +324,11 @@ bool DynamicRenderer::IsOpenGL() const return backend_ == QStringLiteral("opengl"); } +bool DynamicRenderer::IsVulkan() const +{ + return backend_ == QStringLiteral("vulkan"); +} + // Dispatches a shader blit to the loaded backend. void DynamicRenderer::Blit(QVariant shader, AcceleratedJob &job, Texture *destination, VideoParams destination_params, diff --git a/app/render/backend/dynamicrenderer.h b/app/render/backend/dynamicrenderer.h index f25794924..2a0705a53 100644 --- a/app/render/backend/dynamicrenderer.h +++ b/app/render/backend/dynamicrenderer.h @@ -67,6 +67,8 @@ public: // Reports whether the effective backend is OpenGL. virtual bool IsOpenGL() const override; + // Reports whether the effective backend is Vulkan. + virtual bool IsVulkan() const override; // Attaches a texture for OFX OpenGL output when supported. virtual void AttachOutputTexture(Texture *texture) override; diff --git a/app/render/colormanagement.cpp b/app/render/colormanagement.cpp index 006a68b5f..2a7751f9a 100644 --- a/app/render/colormanagement.cpp +++ b/app/render/colormanagement.cpp @@ -43,6 +43,8 @@ bool Renderer::GetColorContext(const ColorTransformJob &color_job, color_ctx = color_cache_.value(proc_id); return true; } else { + locker.unlock(); + // Create shader description QString ocio_func_name; if (color_job.GetFunctionName().isEmpty()) { @@ -147,19 +149,19 @@ bool Renderer::GetColorContext(const ColorTransformJob &color_job, } // Allocate 1D LUT - color_ctx.lut1d_textures[i].texture = CreateTexture( - VideoParams(width, height, PixelFormat::F32, - (channel == - OCIO::GpuShaderDesc::TEXTURE_RED_CHANNEL) ? - 1 : - VideoParams::kRGBChannelCount), - values); + int lut_channels = (channel == + OCIO::GpuShaderDesc::TEXTURE_RED_CHANNEL) ? + 1 : + VideoParams::kRGBChannelCount; + VideoParams lut_params(width, height, PixelFormat::F32, lut_channels); + color_ctx.lut1d_textures[i].texture = CreateTexture(lut_params, values); color_ctx.lut1d_textures[i].name = sampler_name; color_ctx.lut1d_textures[i].interpolation = (interpolation == OCIO::INTERP_NEAREST) ? Texture::kNearest : Texture::kLinear; } + locker.relock(); color_cache_.insert(proc_id, color_ctx); return true; diff --git a/app/render/ipc/frameslotpool.h b/app/render/ipc/frameslotpool.h index 48c48d85a..be08131ed 100644 --- a/app/render/ipc/frameslotpool.h +++ b/app/render/ipc/frameslotpool.h @@ -49,6 +49,7 @@ struct FrameSlotMeta { int32_t channel_count; int32_t linesize; ///< Bytes per scanline (stride). int32_t data_size; ///< Valid bytes written into the slot's data block. + char colorspace[128]; ///< Input colorspace name for color-managed footage. }; /** @@ -143,9 +144,11 @@ public: const FrameSlotMeta *Meta(uint32_t index) const; const void *SlotData(uint32_t index) const; -private: +public: FrameSlotPool() = default; +private: + struct Header { uint32_t magic; uint32_t slot_count; diff --git a/app/render/opengl/openglrenderer.cpp b/app/render/opengl/openglrenderer.cpp index 243691881..b900cdb04 100644 --- a/app/render/opengl/openglrenderer.cpp +++ b/app/render/opengl/openglrenderer.cpp @@ -82,6 +82,8 @@ private: OpenGLRenderer::OpenGLRenderer(QObject *parent) : Renderer(parent) , context_(nullptr) + , functions_(nullptr) + , surface_(nullptr, this) , framebuffer_(0) { } @@ -111,8 +113,6 @@ bool OpenGLRenderer::Init() return false; } - surface_.create(); - context_ = new QOpenGLContext(this); context_->setShareContext(QOpenGLContext::globalShareContext()); if (!context_->create()) { @@ -137,20 +137,29 @@ void OpenGLRenderer::PostInit() { GL_PREAMBLE; - // Make context current on that surface - if (context_->parent() == this && !context_->makeCurrent(&surface_)) { - qCritical() << "Failed to makeCurrent() on offscreen surface in thread" - << thread(); + if (!context_) { + qWarning() << __FUNCTION__ << "called without an OpenGL context"; return; } - functions_ = context_->functions(); + if (context_->thread() != QThread::currentThread()) { + qWarning() << __FUNCTION__ + << "called from the wrong thread for this OpenGL context"; + return; + } - // Store OpenGL functions instance - functions_->glBlendFunc(GL_ONE, GL_ZERO); + // Create the offscreen surface in the thread that will actually use it. + // When OpenGLRenderer is moved to a render thread, surface_ follows as a + // child QObject; creating it here avoids making the context current on a + // surface whose platform backing still belongs to the construction thread, + // which crashes drivers on the first GL call. + if (context_->parent() == this && !surface_.isValid()) { + surface_.create(); + } - // Set up framebuffer used for various things - functions_->glGenFramebuffers(1, &framebuffer_); + if (QOpenGLContext::currentContext() == context_) { + functions_ = context_->functions(); + } } void OpenGLRenderer::DestroyInternal() @@ -177,6 +186,10 @@ void OpenGLRenderer::ClearDestination(Texture *texture, double r, double g, { GL_PREAMBLE; + if (!EnsureContextCurrent(__FUNCTION__)) { + return; + } + if (texture) { AttachTextureAsDestination(texture->id()); } @@ -239,6 +252,10 @@ void OpenGLRenderer::AttachTextureAsDestination(const QVariant &texture) { PRINT_GL_ERRORS; + if (!framebuffer_) { + functions_->glGenFramebuffers(1, &framebuffer_); + } + functions_->glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); functions_->glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture.value(), @@ -255,6 +272,10 @@ void OpenGLRenderer::DetachTextureAsDestination() void OpenGLRenderer::DestroyNativeTexture(QVariant texture) { + if (!EnsureContextCurrent(__FUNCTION__)) { + return; + } + GLuint t = texture.value(); if (t > 0) { @@ -266,6 +287,10 @@ QVariant OpenGLRenderer::CreateNativeShader(ShaderCode code) { GL_PREAMBLE; + if (!EnsureContextCurrent(__FUNCTION__)) { + return QVariant(); + } + PRINT_GL_ERRORS; GLuint vert = CompileShader(GL_VERTEX_SHADER, code.vert_code()); @@ -298,6 +323,10 @@ void OpenGLRenderer::DestroyNativeShader(QVariant shader) { GL_PREAMBLE; + if (!EnsureContextCurrent(__FUNCTION__)) { + return; + } + GLuint program = shader.value(); functions_->glDeleteProgram(program); } @@ -396,6 +425,10 @@ void OpenGLRenderer::Flush() { GL_PREAMBLE; + if (!EnsureContextCurrent(__FUNCTION__)) { + return; + } + #if !defined(OAK_RENDER_BACKEND_PLUGIN) if (OLIVE_CONFIG("UseGLFinish").toBool()) { functions_->glFinish(); @@ -418,6 +451,10 @@ void OpenGLRenderer::Flush() // attachment path used by OFX OpenGL rendering. void OpenGLRenderer::AttachOutputTexture(olive::Texture *texture) { + if (!EnsureContextCurrent(__FUNCTION__)) { + return; + } + if (texture) { AttachTextureAsDestination(texture->id()); } @@ -426,11 +463,19 @@ void OpenGLRenderer::AttachOutputTexture(olive::Texture *texture) // Clears the framebuffer attachment installed by AttachOutputTexture(). void OpenGLRenderer::DetachOutputTexture() { + if (!EnsureContextCurrent(__FUNCTION__)) { + return; + } + DetachTextureAsDestination(); } Color OpenGLRenderer::GetPixelFromTexture(Texture *texture, const QPointF &pt) { + if (!texture || !EnsureContextCurrent(__FUNCTION__)) { + return Color(); + } + AttachTextureAsDestination(texture->id()); QByteArray data(VideoParams::GetBytesPerPixel(texture->format(), @@ -464,6 +509,9 @@ void OpenGLRenderer::Blit(QVariant s, AcceleratedJob& a_job, Texture *destinatio bool clear_destination) { GL_PREAMBLE; + if (!EnsureContextCurrent(__FUNCTION__)) { + return; + } try { if (!destination) { // Ensure we're drawing to the default framebuffer for this context. @@ -897,6 +945,9 @@ void OpenGLRenderer::PrepareInputTexture(GLenum target, void OpenGLRenderer::ClearDestinationInternal(double r, double g, double b, double a) { + if (!functions_) { + return; + } functions_->glClearColor(r, g, b, a); functions_->glClear(GL_COLOR_BUFFER_BIT); } @@ -1025,10 +1076,6 @@ bool OpenGLRenderer::EnsureContextCurrent(const char *caller) return false; } - if (!framebuffer_) { - functions_->glGenFramebuffers(1, &framebuffer_); - } - return true; } diff --git a/app/render/previewautocacher.cpp b/app/render/previewautocacher.cpp index eb490ac75..ae5e45ce0 100644 --- a/app/render/previewautocacher.cpp +++ b/app/render/previewautocacher.cpp @@ -110,11 +110,15 @@ void PreviewAutoCacher::ClearSingleFrameRenders() QMap> copy = video_immediate_passthroughs_; for (auto it = copy.cbegin(); it != copy.cend(); it++) { - it.key()->Cancel(); - if (!it.key()->IsRunning()) { - RenderManager::instance()->RemoveTicket(it.key()->GetTicket()); - emit it.key()->GetTicket()->Finished(); + // Keep already-running workers alive: cancelling an in-flight render + // forces the worker process to be torn down, which defeats the process + // pool. Frames that finish late are simply ignored by the viewer. + if (it.key()->IsRunning()) { + continue; } + it.key()->Cancel(); + RenderManager::instance()->RemoveTicket(it.key()->GetTicket()); + emit it.key()->GetTicket()->Finished(); } } @@ -652,17 +656,23 @@ RenderTicketWatcher *PreviewAutoCacher::RenderFrame(Node *node, VideoParams::GetDividerForTargetResolution( rvp.video_params.width(), rvp.video_params.height(), 160, 120)); - rvp.force_color_output = display_color_processor_; - rvp.force_format = PixelFormat::U8; + rvp.force_format = PixelFormat::F32; + rvp.force_channel_count = VideoParams::kRGBAChannelCount; } else { frame_cache->SetTimebase( context->GetVideoParams().frame_rate_as_time_base()); } rvp.AddCache(frame_cache); + } else { + rvp.force_format = PixelFormat::F32; + rvp.force_channel_count = VideoParams::kRGBAChannelCount; } - rvp.return_type = dry ? RenderManager::kNull : RenderManager::kTexture; + // Video playback frames are rendered out-of-process. GPU textures cannot be + // shared across worker processes (or across independent Vulkan instances), + // so we always request CPU frames. + rvp.return_type = dry ? RenderManager::kNull : RenderManager::kFrame; // Allow using cached images for this render job rvp.use_cache = true; diff --git a/app/render/renderer.h b/app/render/renderer.h index 3484342e2..9d7ce2c75 100644 --- a/app/render/renderer.h +++ b/app/render/renderer.h @@ -125,6 +125,11 @@ public: return false; } + virtual bool IsVulkan() const + { + return false; + } + /** * @brief Attach a texture as the current output destination for OFX plugin * OpenGL rendering. diff --git a/app/render/rendermanager.cpp b/app/render/rendermanager.cpp index 1fd9826b2..14cd0c3de 100644 --- a/app/render/rendermanager.cpp +++ b/app/render/rendermanager.cpp @@ -124,7 +124,6 @@ RenderManager::RenderManager(QObject *parent) } if (context_) { - video_thread_ = CreateThread(context_); dry_run_thread_ = CreateThread(); audio_thread_ = CreateThread(); @@ -135,11 +134,10 @@ RenderManager::RenderManager(QObject *parent) auto_cacher_ = new PreviewAutoCacher(this); - if (OLIVE_CONFIG("RenderProcessIsolationEnabled").toBool()) { - worker_pool_ = new RenderWorkerPool(decoder_cache_, this); - worker_pool_->start(QThread::NormalPriority); - backend_ = kMultiProcess; - } + worker_pool_ = new RenderWorkerPool( + decoder_cache_, BackendToString(requested_backend_), this); + worker_pool_->start(QThread::NormalPriority); + backend_ = kMultiProcess; } decoder_clear_timer_ = new QTimer(this); @@ -208,15 +206,22 @@ RenderTicketPtr RenderManager::RenderFrame(const RenderVideoParams ¶ms) ticket->setProperty("cacheid", QVariant::fromValue(params.cache_id)); ticket->setProperty("multicam", QtUtils::PtrToValue(params.multicam)); - if (worker_pool_ && params.return_type == ReturnType::kFrame && - worker_pool_->SubmitFrame(ticket, params)) { - return ticket; + // Video frames are always rendered by the worker pool. GPU textures cannot + // be shared across the process boundary (or across independent Vulkan + // instances), so texture-return requests are downgraded to CPU frames. + RenderVideoParams worker_params = params; + if (worker_params.return_type == ReturnType::kTexture) { + worker_params.return_type = ReturnType::kFrame; } - if (params.return_type == ReturnType::kNull) { + if (worker_params.return_type == ReturnType::kNull) { dry_run_thread_->AddTicket(ticket); + } else if (worker_pool_ && worker_pool_->SubmitFrame(ticket, worker_params)) { + return ticket; } else { - video_thread_->AddTicket(ticket); + qWarning() << "RenderManager: worker pool unavailable, finishing ticket " + "without result"; + ticket->Finish(); } return ticket; diff --git a/app/render/rendermanager.h b/app/render/rendermanager.h index e335d034f..a701007e4 100644 --- a/app/render/rendermanager.h +++ b/app/render/rendermanager.h @@ -252,7 +252,6 @@ private: QTimer *decoder_clear_timer_; - RenderThread *video_thread_; RenderThread *dry_run_thread_; RenderThread *audio_thread_; diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index 9cd7a39f4..7b13e5f89 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -28,6 +28,7 @@ #include #include + #include "audio/audioprocessor.h" #include "node/block/clip/clip.h" #include "node/block/transition/transition.h" @@ -148,22 +149,17 @@ FramePtr RenderProcessor::GenerateFrame(TexturePtr texture, texture = blit_tex; } - render_ctx_->Flush(); - render_ctx_->DownloadFromTexture(texture->id(), texture->params(), frame->data(), frame->linesize_pixels()); - - // Diagnostic: check if downloaded frame is all black - bool all_black = true; - const uint8_t *pixels = reinterpret_cast(frame->data()); - size_t total_bytes = frame->allocated_size(); - for (size_t i = 0; i < std::min(total_bytes, size_t(1024)); ++i) { - if (pixels[i] != 0) { - all_black = false; - break; - } + if (output_color_transform) { + VideoParams display_params = frame->video_params(); + display_params.set_colorspace( + QStringLiteral("display:") + + QString::fromUtf8(output_color_transform->id())); + frame->set_video_params(display_params); } + } return frame; @@ -232,7 +228,6 @@ void RenderProcessor::Run() if (HeardCancel()) { // Finish cancelled ticket with nothing since we can't guarantee the frame we generated // is actually "complete - qDebug() << "[RENDER] HeardCancel, finishing empty"; ticket_->Finish(); } else { FramePtr frame; @@ -416,13 +411,16 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination, QString using_colorspace = stream_data.colorspace(); + if (using_colorspace.isEmpty() && color_manager) { + using_colorspace = color_manager->GetDefaultInputColorSpace(); + } + if (using_colorspace.isEmpty()) { - // FIXME: - qWarning() << "HAVEN'T GOTTEN DEFAULT INPUT COLORSPACE"; + qWarning() << "RenderProcessor ProcessVideoFootage: no input colorspace available"; } auto blit_color_managed = [&](const TexturePtr &unmanaged_texture, - const VideoParams &texture_params) { + const VideoParams &texture_params) { if (!render_ctx_ || !unmanaged_texture || IsCancelled()) { return; } @@ -482,13 +480,32 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination, input_params.set_height(meta->height); input_params.set_format(PixelFormat::Format(meta->format)); input_params.set_channel_count(meta->channel_count); + // The decoder may leave depth at 0 for 2D frames, but the renderer + // needs depth >= 1 to compute image size and upload the texture. + if (input_params.depth() <= 0) { + input_params.set_depth(1); + } + + + // Prefer the colorspace that the main process used when decoding this + // frame. The FootageJob reconstructed in the worker may have stale or + // empty colorspace if the project snapshot was saved before stream + // metadata was fully resolved. + const QString ipc_colorspace = QString::fromUtf8(meta->colorspace); + if (!ipc_colorspace.isEmpty()) { + input_params.set_colorspace(ipc_colorspace); + using_colorspace = ipc_colorspace; + } const int bytes_per_pixel = input_params.GetBytesPerPixel(); const int linesize_pixels = bytes_per_pixel > 0 - ? meta->linesize / bytes_per_pixel - : input_params.effective_width(); + ? meta->linesize / bytes_per_pixel + : input_params.effective_width(); + + const void *slot_data = input_pool->SlotData(uint32_t(input_slot)); TexturePtr unmanaged_texture = render_ctx_->CreateTexture( - input_params, input_pool->SlotData(uint32_t(input_slot)), linesize_pixels); + input_params, slot_data, linesize_pixels); + blit_color_managed(unmanaged_texture, input_params); return; } diff --git a/app/render/renderworkerpool.cpp b/app/render/renderworkerpool.cpp index 5e17f2a68..fdd66987f 100644 --- a/app/render/renderworkerpool.cpp +++ b/app/render/renderworkerpool.cpp @@ -21,6 +21,7 @@ #include "renderworkerpool.h" #include +#include #include #include #include @@ -30,8 +31,10 @@ #include #include #include +#include #include #include +#include #include #if defined(Q_OS_WIN) #include @@ -183,6 +186,17 @@ FramePtr DecodeInputFrame(DecoderCache *decoder_cache, FramePtr frame = decoder->RetrieveVideoFrame(retrieve); if (frame) { frame->set_timestamp(input.time); + + // Ensure the frame carries the colorspace the color manager expects. + // Decoders do not always set this on the returned frame, but the worker + // needs it to build the correct OCIO transform. + VideoParams frame_params = frame->video_params(); + if (frame_params.colorspace().isEmpty() && + !stream_data.colorspace().isEmpty()) { + frame_params.set_colorspace(stream_data.colorspace()); + frame->set_video_params(frame_params); + } + } return frame; } @@ -254,6 +268,26 @@ bool KillProcessById(qint64 process_id) #endif } +bool IsProcessAlive(qint64 process_id) +{ + if (process_id <= 0) { + return false; + } + +#if defined(Q_OS_WIN) + HANDLE handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, DWORD(process_id)); + if (!handle) { + return false; + } + DWORD exit_code = 0; + const bool alive = GetExitCodeProcess(handle, &exit_code) && exit_code == STILL_ACTIVE; + CloseHandle(handle); + return alive; +#else + return ::kill(pid_t(process_id), 0) == 0; +#endif +} + QString WorkerProcessDetails(const QProcess *process) { if (!process) { @@ -323,9 +357,11 @@ bool ReadControlMessage(QProcess *process, QJsonObject *out, QString *error, } // namespace RenderWorkerPool::RenderWorkerPool(DecoderCache *decoder_cache, + const QString &gpu_backend, QObject *parent) : QThread(parent) , decoder_cache_(decoder_cache) + , gpu_backend_(gpu_backend) { } @@ -393,7 +429,7 @@ bool RenderWorkerPool::RemoveTicket(RenderTicketPtr ticket) void RenderWorkerPool::Shutdown() { - QVector queued_graph_paths; + QVector graph_paths_to_clean; { QMutexLocker locker(&mutex_); @@ -402,7 +438,6 @@ void RenderWorkerPool::Shutdown() if (job.ticket) { job.ticket->Cancel(); } - queued_graph_paths.append(job.graph_path); } queue_.clear(); for (ActiveJob &active : active_jobs_) { @@ -411,10 +446,14 @@ void RenderWorkerPool::Shutdown() CancelActiveProcess(active.process_id); } } + for (auto it = graph_cache_.begin(); it != graph_cache_.end(); ++it) { + graph_paths_to_clean.append(it->path); + } + graph_cache_.clear(); wait_.wakeAll(); } - for (const QString &path : queued_graph_paths) { + for (const QString &path : graph_paths_to_clean) { CleanupGraphFile(path); } @@ -431,11 +470,12 @@ void RenderWorkerPool::run() active_jobs_.resize(worker_count); } + std::vector>> local_pools(worker_count); std::vector workers; workers.reserve(size_t(worker_count)); for (int i = 0; i < worker_count; i++) { - workers.emplace_back([this, i]() { - WorkerLoop(i); + workers.emplace_back([this, i, &local_pools]() { + WorkerLoop(i, &local_pools[i]); }); } @@ -443,11 +483,19 @@ void RenderWorkerPool::run() worker.join(); } + for (auto &local_pool : local_pools) { + ShutdownLocalPool(&local_pool); + } + + ClearGraphCache(); + QMutexLocker locker(&mutex_); active_jobs_.clear(); } -void RenderWorkerPool::WorkerLoop(int worker_index) +void RenderWorkerPool::WorkerLoop( + int worker_index, + std::vector> *local_pool) { while (true) { mutex_.lock(); @@ -463,8 +511,7 @@ void RenderWorkerPool::WorkerLoop(int worker_index) queue_.pop_front(); mutex_.unlock(); - ProcessJob(job, worker_index); - CleanupGraphFile(job.graph_path); + ProcessJob(job, worker_index, local_pool); } } @@ -491,8 +538,26 @@ bool RenderWorkerPool::PrepareJob(RenderTicketPtr ticket, } QString graph_path; - if (!WriteGraphSnapshot(project, &graph_path)) { - return false; + bool wrote_new_snapshot = false; + { + const QUuid project_uuid = project->GetUuid(); + QMutexLocker locker(&mutex_); + auto it = graph_cache_.find(project_uuid); + if (it != graph_cache_.end() && !project->is_modified()) { + graph_path = it->path; + } else { + if (it != graph_cache_.end()) { + CleanupGraphFile(it->path); + graph_cache_.erase(it); + } + locker.unlock(); + if (!WriteGraphSnapshot(project, &graph_path)) { + return false; + } + wrote_new_snapshot = true; + locker.relock(); + graph_cache_.insert(project_uuid, {graph_path}); + } } job->ticket = ticket; @@ -500,6 +565,7 @@ bool RenderWorkerPool::PrepareJob(RenderTicketPtr ticket, job->graph_path = graph_path; job->node_token = QString::number(reinterpret_cast(params.node)); job->input_frames = input_frames; + Q_UNUSED(wrote_new_snapshot) return true; } @@ -535,7 +601,9 @@ bool RenderWorkerPool::IsSupported(const RenderManager::RenderVideoParams ¶m params.video_params.is_valid(); } -void RenderWorkerPool::ProcessJob(const Job &job, int worker_index) +void RenderWorkerPool::ProcessJob( + const Job &job, int worker_index, + std::vector> *local_pool) { const qint64 ticket_id = qint64(reinterpret_cast(job.ticket.get())); SetActiveWorker(worker_index, job.ticket, nullptr, ticket_id); @@ -547,8 +615,39 @@ void RenderWorkerPool::ProcessJob(const Job &job, int worker_index) return; } + std::unique_ptr worker = AcquireWorker(local_pool, job.graph_path); + if (!worker) { + qWarning() << "RenderWorkerPool failed to acquire worker for ticket" + << ticket_id; + job.ticket->Finish(); + ClearActiveWorker(worker_index, 0); + return; + } + for (int attempt = 0; attempt < kMaxAttempts; attempt++) { - const JobResult result = ProcessJobAttempt(job, worker_index, attempt); + if (attempt > 0) { + worker = AcquireWorker(local_pool, job.graph_path); + if (!worker) { + qWarning() << "RenderWorkerPool failed to acquire worker for retry" + << ticket_id; + break; + } + } + + const JobResult result = ProcessJobAttempt(job, worker_index, attempt, + worker.get()); + const qint64 worker_pid = worker && worker->process + ? worker->process->processId() + : 0; + const bool process_state_running = worker && worker->process && + worker->process->state() == QProcess::Running; + const bool os_alive = worker_pid > 0 && IsProcessAlive(worker_pid); + const bool worker_healthy = process_state_running || os_alive; + const bool keep_alive = (result == JobResult::kFinished) && worker_healthy; + + ReturnWorker(local_pool, std::move(worker), keep_alive); + worker.reset(); + if (result == JobResult::kFinished) { ClearActiveWorker(worker_index, 0); return; @@ -578,198 +677,241 @@ void RenderWorkerPool::ProcessJob(const Job &job, int worker_index) } RenderWorkerPool::JobResult RenderWorkerPool::ProcessJobAttempt( - const Job &job, int worker_index, int attempt_index) + const Job &job, int worker_index, int attempt_index, + PooledWorker *worker) { const qint64 ticket_id = qint64(reinterpret_cast(job.ticket.get())); if (job.ticket->IsCancelled()) { return JobResult::kCancelled; } - const int linesize = Frame::generate_linesize_bytes( - kMaxWidth, PixelFormat::F32, VideoParams::kRGBAChannelCount); - const size_t slot_bytes = size_t(linesize) * kMaxHeight; - const size_t region_bytes = ipc::FrameSlotPool::BytesNeeded(kOutputSlots, slot_bytes); - const QString shm_key = - ipc::SharedMemoryRegion::MakeKey(QCoreApplication::applicationPid(), - int((reinterpret_cast(job.ticket.get()) + - attempt_index * 2) & 0xFFFF)); - - ipc::SharedMemoryRegion region; - if (!region.Open(shm_key, region_bytes, ipc::SharedMemoryRegion::kCreate)) { - qWarning() << "RenderWorkerPool failed to create shared memory" - << region.error(); - return JobResult::kFatalFailure; + if (!worker || !worker->process) { + return JobResult::kRetryableFailure; } - ipc::FrameSlotPool output_pool = - ipc::FrameSlotPool::Create(region.data(), kOutputSlots, slot_bytes); - const QString input_shm_key = - job.input_frames.isEmpty() - ? QString() - : ipc::SharedMemoryRegion::MakeKey( - QCoreApplication::applicationPid(), - int((reinterpret_cast(job.ticket.get()) + - attempt_index * 2 + 1) & 0xFFFF)); - ipc::SharedMemoryRegion input_region; - std::optional input_pool; - QVector input_slots; - if (!job.input_frames.isEmpty()) { - const uint32_t input_slot_count = uint32_t(job.input_frames.size()); - const size_t input_region_bytes = - ipc::FrameSlotPool::BytesNeeded(input_slot_count, slot_bytes); - if (!input_region.Open(input_shm_key, input_region_bytes, - ipc::SharedMemoryRegion::kCreate)) { - qWarning() << "RenderWorkerPool failed to create input shared memory" - << input_region.error(); + const qint64 worker_process_id = worker->process->processId(); + + const int output_width = job.params.force_size.width() > 0 + ? job.params.force_size.width() + : job.params.video_params.effective_width(); + const int output_height = job.params.force_size.height() > 0 + ? job.params.force_size.height() + : job.params.video_params.effective_height(); + const PixelFormat::Format output_format = + job.params.force_format != PixelFormat::INVALID + ? PixelFormat::Format(job.params.force_format) + : PixelFormat::F32; + const int output_channels = job.params.force_channel_count > 0 + ? job.params.force_channel_count + : VideoParams::kRGBAChannelCount; + const int output_linesize = + Frame::generate_linesize_bytes(output_width, output_format, + output_channels); + const size_t estimated_output_slot_bytes = + size_t(output_linesize) * size_t(output_height); + const int f32_rgba_linesize = + Frame::generate_linesize_bytes(output_width, PixelFormat::F32, + VideoParams::kRGBAChannelCount); + const size_t f32_rgba_slot_bytes = + size_t(f32_rgba_linesize) * size_t(output_height); + const size_t output_slot_bytes = + std::max(estimated_output_slot_bytes, f32_rgba_slot_bytes); + size_t input_slot_bytes = 0; + for (const FramePtr &frame : job.input_frames) { + if (frame && frame->is_allocated()) { + input_slot_bytes = + std::max(input_slot_bytes, size_t(frame->allocated_size())); + } + } + const size_t output_region_bytes = + ipc::FrameSlotPool::BytesNeeded(kOutputSlots, output_slot_bytes); + + if (!worker->output_region.IsValid() || + worker->output_slot_bytes < output_slot_bytes) { + if (worker->output_region.IsValid()) { + worker->output_region.Close(); + worker->output_pool = ipc::FrameSlotPool(); + } + if (worker->output_shm_key.isEmpty()) { + worker->output_shm_key = + ipc::SharedMemoryRegion::MakeKey(worker_process_id, 0) + + QStringLiteral("-out"); + } + if (!worker->output_region.Open(worker->output_shm_key, + output_region_bytes, + ipc::SharedMemoryRegion::kCreate)) { + qWarning() << "RenderWorkerPool failed to create output shared memory" + << worker->output_region.error(); return JobResult::kFatalFailure; - } else { - input_pool = ipc::FrameSlotPool::Create(input_region.data(), - input_slot_count, - slot_bytes); - for (const FramePtr &frame : job.input_frames) { - if (frame->allocated_size() > int(slot_bytes)) { - qWarning() << "RenderWorkerPool decoded input frame exceeds slot size"; - return JobResult::kFatalFailure; - } + } + worker->output_pool = ipc::FrameSlotPool::Create( + worker->output_region.data(), kOutputSlots, output_slot_bytes); + worker->output_slot_bytes = output_slot_bytes; + } + const QString shm_key = worker->output_shm_key; + ipc::FrameSlotPool &output_pool = worker->output_pool; - uint32_t slot = 0; - if (!input_pool->Acquire(&slot)) { - qWarning() << "RenderWorkerPool input pool had no free slot"; - return JobResult::kFatalFailure; - } - - memcpy(input_pool->SlotData(slot), frame->const_data(), - size_t(frame->allocated_size())); - ipc::FrameSlotMeta *meta = input_pool->Meta(slot); - meta->id = qint64(input_slots.size()); - meta->time_num = frame->timestamp().numerator(); - meta->time_den = frame->timestamp().denominator(); - meta->width = frame->width(); - meta->height = frame->height(); - meta->format = int32_t(frame->format()); - meta->channel_count = frame->channel_count(); - meta->linesize = frame->linesize_bytes(); - meta->data_size = frame->allocated_size(); - if (!input_pool->Publish(slot)) { - qWarning() << "RenderWorkerPool failed to publish input slot"; - return JobResult::kFatalFailure; - } - input_slots.append(int(slot)); + const uint32_t input_slot_count = + job.input_frames.isEmpty() ? 0 : uint32_t(job.input_frames.size()); + if (input_slot_count > 0) { + if (!worker->input_region.IsValid() || + worker->input_slot_bytes < input_slot_bytes || + worker->input_pool.slot_count() < input_slot_count) { + if (worker->input_region.IsValid()) { + worker->input_region.Close(); + worker->input_pool = ipc::FrameSlotPool(); } - - if (input_slots.size() != job.input_frames.size()) { - qWarning() << "RenderWorkerPool failed to publish all input frames;" - << "aborting worker render"; + if (worker->input_shm_key.isEmpty()) { + worker->input_shm_key = + ipc::SharedMemoryRegion::MakeKey(worker_process_id, 1) + + QStringLiteral("-in"); + } + const size_t input_region_bytes = + ipc::FrameSlotPool::BytesNeeded(input_slot_count, input_slot_bytes); + if (!worker->input_region.Open(worker->input_shm_key, + input_region_bytes, + ipc::SharedMemoryRegion::kCreate)) { + qWarning() << "RenderWorkerPool failed to create input shared memory" + << worker->input_region.error(); return JobResult::kFatalFailure; } + worker->input_pool = ipc::FrameSlotPool::Create( + worker->input_region.data(), input_slot_count, input_slot_bytes); + worker->input_slot_bytes = input_slot_bytes; + } + } + const QString input_shm_key = worker->input_shm_key; + ipc::FrameSlotPool &input_pool = worker->input_pool; + QVector input_slots; + if (input_slot_count > 0) { + for (const FramePtr &frame : job.input_frames) { + if (frame->allocated_size() > int(worker->input_slot_bytes)) { + qWarning() << "RenderWorkerPool decoded input frame exceeds slot size"; + return JobResult::kFatalFailure; + } + + uint32_t slot = 0; + if (!input_pool.Acquire(&slot)) { + qWarning() << "RenderWorkerPool input pool had no free slot"; + return JobResult::kFatalFailure; + } + + memcpy(input_pool.SlotData(slot), frame->const_data(), + size_t(frame->allocated_size())); + ipc::FrameSlotMeta *meta = input_pool.Meta(slot); + meta->id = qint64(input_slots.size()); + meta->time_num = frame->timestamp().numerator(); + meta->time_den = frame->timestamp().denominator(); + meta->width = frame->width(); + meta->height = frame->height(); + meta->format = int32_t(frame->format()); + meta->channel_count = frame->channel_count(); + meta->linesize = frame->linesize_bytes(); + meta->data_size = frame->allocated_size(); + memset(meta->colorspace, 0, sizeof(meta->colorspace)); + const QString cs = frame->video_params().colorspace(); + if (!cs.isEmpty()) { + const QByteArray cs_utf8 = cs.toUtf8(); + const size_t copy_len = qMin( + static_cast(cs_utf8.size()), + sizeof(meta->colorspace) - 1); + memcpy(meta->colorspace, cs_utf8.constData(), copy_len); + meta->colorspace[copy_len] = '\0'; + } + if (!input_pool.Publish(slot)) { + qWarning() << "RenderWorkerPool failed to publish input slot"; + return JobResult::kFatalFailure; + } + input_slots.append(int(slot)); + } + + if (input_slots.size() != job.input_frames.size()) { + qWarning() << "RenderWorkerPool failed to publish all input frames;" + << "aborting worker render"; + return JobResult::kFatalFailure; } } - QProcess worker; - worker.setProgram(WorkerProgramPath()); - worker.start(); - if (!worker.waitForStarted(10000)) { - qWarning() << "RenderWorkerPool failed to start worker" - << worker.errorString(); - return JobResult::kRetryableFailure; - } - const qint64 worker_process_id = worker.processId(); - - SetActiveWorker(worker_index, job.ticket, &worker, ticket_id); + SetActiveWorker(worker_index, job.ticket, worker->process, ticket_id); if (job.ticket->IsCancelled()) { ipc::CancelMsg cancel; cancel.ticket_id = ticket_id; - TryWriteControlMessage(&worker, cancel.ToJson()); - worker.kill(); - worker.waitForFinished(); + TryWriteControlMessage(worker->process, cancel.ToJson()); ClearActiveWorker(worker_index, worker_process_id); return JobResult::kCancelled; } - QString error; - QJsonObject response; - if (!ReadControlMessage(&worker, &response, &error)) { - if (!job.ticket->IsCancelled()) { - qWarning() << "RenderWorkerPool did not receive startup handshake" - << error << worker.readAllStandardError(); - } - worker.kill(); - worker.waitForFinished(); - ClearActiveWorker(worker_index, worker_process_id); - return job.ticket->IsCancelled() ? JobResult::kCancelled - : JobResult::kRetryableFailure; - } - ipc::HandshakeMsg handshake; handshake.protocol_version = kProtocolVersion; handshake.shm_key = shm_key; handshake.input_shm_key = input_slots.isEmpty() ? QString() : input_shm_key; handshake.input_slots = input_slots.size(); handshake.output_slots = int(kOutputSlots); - handshake.slot_data_bytes = qint64(slot_bytes); - handshake.input_slot_data_bytes = input_slots.isEmpty() ? 0 : qint64(slot_bytes); - if (!WriteControlMessage(&worker, handshake.ToJson())) { + handshake.slot_data_bytes = qint64(output_slot_bytes); + handshake.input_slot_data_bytes = input_slots.isEmpty() + ? 0 + : qint64(input_slot_bytes); + if (!WriteControlMessage(worker->process, handshake.ToJson())) { if (!job.ticket->IsCancelled()) { qWarning() << "RenderWorkerPool failed to send shared-memory handshake"; } - worker.kill(); - worker.waitForFinished(); ClearActiveWorker(worker_index, worker_process_id); return job.ticket->IsCancelled() ? JobResult::kCancelled - : JobResult::kRetryableFailure; + : JobResult::kRetryableFailure; } - ipc::LoadGraphMsg load; - load.path = job.graph_path; - if (!WriteControlMessage(&worker, load.ToJson()) || - !ReadControlMessage(&worker, &response, &error)) { - if (!job.ticket->IsCancelled()) { - qWarning() << "RenderWorkerPool failed to load graph in worker" - << error << worker.readAllStandardError(); + if (worker->loaded_graph_path != job.graph_path) { + ipc::LoadGraphMsg load; + load.path = job.graph_path; + QString error; + QJsonObject response; + if (!WriteControlMessage(worker->process, load.ToJson()) || + !ReadControlMessage(worker->process, &response, &error)) { + if (!job.ticket->IsCancelled()) { + qWarning() << "RenderWorkerPool failed to load graph in worker" + << error << worker->process->readAllStandardError(); + } + ClearActiveWorker(worker_index, worker_process_id); + return job.ticket->IsCancelled() ? JobResult::kCancelled + : JobResult::kRetryableFailure; } - worker.kill(); - worker.waitForFinished(); - ClearActiveWorker(worker_index, worker_process_id); - return job.ticket->IsCancelled() ? JobResult::kCancelled - : JobResult::kRetryableFailure; - } + worker->loaded_graph_path = job.graph_path; +} ipc::RenderFrameMsg render; render.ticket_id = ticket_id; render.node_uuid = job.node_token; render.time_num = job.params.time.numerator(); render.time_den = job.params.time.denominator(); - render.width = job.params.force_size.width(); - render.height = job.params.force_size.height(); - render.format = int(job.params.force_format); - render.channel_count = job.params.force_channel_count; + render.width = output_width; + render.height = output_height; + render.format = int(output_format); + render.channel_count = output_channels; render.mode = int(job.params.mode); render.input_slot = input_slots.isEmpty() ? -1 : input_slots.front(); render.input_slots = input_slots; - if (!WriteControlMessage(&worker, render.ToJson())) { + if (!WriteControlMessage(worker->process, render.ToJson())) { if (!job.ticket->IsCancelled()) { qWarning() << "RenderWorkerPool failed to send render_frame"; } - worker.kill(); - worker.waitForFinished(); ClearActiveWorker(worker_index, worker_process_id); return job.ticket->IsCancelled() ? JobResult::kCancelled - : JobResult::kRetryableFailure; + : JobResult::kRetryableFailure; } + QString error; + QJsonObject response; ipc::FrameReadyMsg ready; while (true) { - if (!ReadControlMessage(&worker, &response, &error, 30000)) { + if (!ReadControlMessage(worker->process, &response, &error, 30000)) { if (!job.ticket->IsCancelled()) { qWarning() << "RenderWorkerPool failed waiting for frame_ready" - << error << worker.readAllStandardError(); + << error << worker->process->readAllStandardError(); } - worker.kill(); - worker.waitForFinished(); ClearActiveWorker(worker_index, worker_process_id); return job.ticket->IsCancelled() ? JobResult::kCancelled - : JobResult::kRetryableFailure; + : JobResult::kRetryableFailure; } if (ipc::FrameReadyMsg::FromJson(response, &ready)) { @@ -780,19 +922,22 @@ RenderWorkerPool::JobResult RenderWorkerPool::ProcessJobAttempt( if (job.ticket->IsCancelled()) { ClearActiveWorker(worker_index, worker_process_id); return JobResult::kCancelled; - } else { - FinishWithFrame(job.ticket, output_pool, uint32_t(ready.output_slot)); } + + uint32_t consumed_slot = 0; + if (!output_pool.Consume(&consumed_slot)) { + qWarning() << "RenderWorkerPool failed to consume output slot"; + ClearActiveWorker(worker_index, worker_process_id); + return JobResult::kRetryableFailure; + } + if (int(consumed_slot) != ready.output_slot) { + qWarning() << "RenderWorkerPool output slot mismatch: consumed" + << consumed_slot << "expected" << ready.output_slot; + } + FinishWithFrame(job.ticket, output_pool, consumed_slot); + output_pool.Release(consumed_slot); ClearActiveWorker(worker_index, worker_process_id); - QJsonObject shutdown; - shutdown[QStringLiteral("type")] = ipc::msgtype::kShutdown; - WriteControlMessage(&worker, shutdown); - worker.closeWriteChannel(); - if (!worker.waitForFinished(5000)) { - worker.kill(); - worker.waitForFinished(); - } return JobResult::kFinished; } @@ -834,8 +979,163 @@ void RenderWorkerPool::ClearActiveWorker(int worker_index, qint64 process_id) int RenderWorkerPool::WorkerCount() const { + // GPU rendering is the bottleneck for video frames; too many workers just + // multiply first-frame warmup (shader/OCIO cache creation) and compete for + // the same GPU. Cap at a small number while still leaving cores free. const int ideal = QThread::idealThreadCount(); - return std::max(1, ideal - 2); + return std::max(1, std::min(ideal - 2, 4)); +} + +std::unique_ptr RenderWorkerPool::AcquireWorker( + std::vector> *local_pool, + const QString &graph_path) +{ + if (!local_pool) { + return nullptr; + } + + const qint64 now = QDateTime::currentMSecsSinceEpoch(); + + // Prefer an idle worker that already has the requested graph loaded. + int best_index = -1; + for (size_t i = 0; i < local_pool->size();) { + PooledWorker *candidate = (*local_pool)[i].get(); + if (!candidate || !candidate->process) { + local_pool->erase(local_pool->begin() + i); + continue; + } + const bool candidate_state_running = + candidate->process->state() == QProcess::Running; + const bool candidate_os_alive = + IsProcessAlive(candidate->process->processId()); + if (!candidate_state_running && !candidate_os_alive) { + ShutdownWorker(candidate); + local_pool->erase(local_pool->begin() + i); + continue; + } + if (now - candidate->last_used_ms > kWorkerIdleTimeoutMs) { + ShutdownWorker(candidate); + local_pool->erase(local_pool->begin() + i); + continue; + } + if (best_index < 0 || + (!candidate->loaded_graph_path.isEmpty() && + candidate->loaded_graph_path == graph_path && + ((*local_pool)[size_t(best_index)]->loaded_graph_path != graph_path))) { + best_index = int(i); + } + ++i; + } + + if (best_index >= 0) { + std::unique_ptr worker = + std::move((*local_pool)[size_t(best_index)]); + local_pool->erase(local_pool->begin() + best_index); + worker->last_used_ms = now; + ++worker->use_count; + return worker; + } + + + // No idle worker available: start a new one. + auto *process = new QProcess(); + process->setProgram(WorkerProgramPath()); + process->setArguments({QStringLiteral("--backend"), gpu_backend_}); + + const QString worker_stderr_path = QDir(QDir::tempPath()).filePath( + QStringLiteral("oak-render-worker-%1-%2.stderr.log") + .arg(QCoreApplication::applicationPid()) + .arg(QDateTime::currentMSecsSinceEpoch())); + process->setStandardErrorFile(worker_stderr_path); + + process->start(); + if (!process->waitForStarted(10000)) { + qWarning() << "RenderWorkerPool failed to start worker" + << process->errorString(); + delete process; + return nullptr; + } + + QString error; + QJsonObject response; + if (!ReadControlMessage(process, &response, &error)) { + qWarning() << "RenderWorkerPool did not receive startup handshake" + << error << process->readAllStandardError(); + process->kill(); + process->waitForFinished(); + delete process; + return nullptr; + } + + auto worker = std::make_unique(); + worker->process = process; + worker->last_used_ms = now; + worker->use_count = 1; + return worker; +} + +void RenderWorkerPool::ReturnWorker( + std::vector> *local_pool, + std::unique_ptr worker, + bool keep_alive) +{ + if (!worker || !worker->process) { + return; + } + + const bool pool_full = worker->use_count >= kWorkerMaxUses; + if (!keep_alive || stopping_ || pool_full) { + ShutdownWorker(worker.get()); + return; + } + + worker->last_used_ms = QDateTime::currentMSecsSinceEpoch(); + local_pool->push_back(std::move(worker)); +} + +void RenderWorkerPool::ShutdownWorker(PooledWorker *worker) +{ + if (!worker || !worker->process) { + return; + } + + QProcess *process = worker->process; + worker->process = nullptr; + worker->loaded_graph_path.clear(); + worker->use_count = 0; + + if (process->state() == QProcess::Running) { + QJsonObject shutdown; + shutdown[QStringLiteral("type")] = ipc::msgtype::kShutdown; + TryWriteControlMessage(process, shutdown); + process->closeWriteChannel(); + if (!process->waitForFinished(5000)) { + process->kill(); + process->waitForFinished(); + } + } + delete process; +} + +void RenderWorkerPool::ShutdownLocalPool( + std::vector> *local_pool) +{ + if (!local_pool) { + return; + } + for (std::unique_ptr &worker : *local_pool) { + ShutdownWorker(worker.get()); + } + local_pool->clear(); +} + +void RenderWorkerPool::ClearGraphCache() +{ + QMutexLocker locker(&mutex_); + for (auto it = graph_cache_.begin(); it != graph_cache_.end(); ++it) { + CleanupGraphFile(it->path); + } + graph_cache_.clear(); } void RenderWorkerPool::FinishWithFrame(RenderTicketPtr ticket, diff --git a/app/render/renderworkerpool.h b/app/render/renderworkerpool.h index 799a398a4..929a52151 100644 --- a/app/render/renderworkerpool.h +++ b/app/render/renderworkerpool.h @@ -21,11 +21,13 @@ #ifndef RENDERWORKERPOOL_H #define RENDERWORKERPOOL_H +#include #include #include #include #include #include +#include #include "codec/frame.h" #include "node/project/serializer/serializer.h" @@ -39,10 +41,13 @@ class QProcess; namespace olive { +class Project; + class RenderWorkerPool : public QThread { Q_OBJECT public: explicit RenderWorkerPool(DecoderCache *decoder_cache, + const QString &gpu_backend, QObject *parent = nullptr); ~RenderWorkerPool() override; @@ -84,16 +89,42 @@ private: qint64 ticket_id = 0; }; + struct PooledWorker { + QProcess *process = nullptr; + QString loaded_graph_path; + qint64 last_used_ms = 0; + int use_count = 0; + + // Persistent shared memory for this worker. Reusing regions across frames + // avoids the cost of creating/destroying large shm segments every render. + ipc::SharedMemoryRegion output_region; + ipc::FrameSlotPool output_pool; + size_t output_slot_bytes = 0; + QString output_shm_key; + + ipc::SharedMemoryRegion input_region; + ipc::FrameSlotPool input_pool; + size_t input_slot_bytes = 0; + QString input_shm_key; + }; + + struct CachedGraph { + QString path; + }; + bool PrepareJob(RenderTicketPtr ticket, const RenderManager::RenderVideoParams ¶ms, Job *job); bool WriteGraphSnapshot(Project *project, QString *path); bool IsSupported(const RenderManager::RenderVideoParams ¶ms) const; - void WorkerLoop(int worker_index); - void ProcessJob(const Job &job, int worker_index); + void WorkerLoop(int worker_index, + std::vector> *local_pool); + void ProcessJob(const Job &job, int worker_index, + std::vector> *local_pool); JobResult ProcessJobAttempt(const Job &job, int worker_index, - int attempt_index); + int attempt_index, + PooledWorker *worker); void FinishWithFrame(RenderTicketPtr ticket, const ipc::FrameSlotPool &pool, uint32_t slot); void CleanupGraphFile(const QString &path); @@ -103,17 +134,30 @@ private: void ClearActiveWorker(int worker_index, qint64 process_id); int WorkerCount() const; + std::unique_ptr AcquireWorker( + std::vector> *local_pool, + const QString &graph_path); + void ReturnWorker(std::vector> *local_pool, + std::unique_ptr worker, bool keep_alive); + void ShutdownWorker(PooledWorker *worker); + void ShutdownLocalPool(std::vector> *local_pool); + void ClearGraphCache(); + DecoderCache *decoder_cache_; + QString gpu_backend_; QMutex mutex_; QWaitCondition wait_; std::deque queue_; bool stopping_ = false; QVector active_jobs_; + QHash graph_cache_; static constexpr uint32_t kOutputSlots = 2; static constexpr int kMaxAttempts = 2; static constexpr int kMaxWidth = 4096; static constexpr int kMaxHeight = 2160; + static constexpr int kWorkerIdleTimeoutMs = 30000; + static constexpr int kWorkerMaxUses = 100; }; } diff --git a/app/render/vulkan/vulkanrenderer.cpp b/app/render/vulkan/vulkanrenderer.cpp index 3964dd6dd..a11210be1 100644 --- a/app/render/vulkan/vulkanrenderer.cpp +++ b/app/render/vulkan/vulkanrenderer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "node/value.h" #include "render/job/shaderjob.h" @@ -49,6 +50,16 @@ struct VulkanRenderer::VulkanShader { QVector uniforms; VkDeviceSize ubo_size = 0; int sampler_count = 0; + // Maps sampler uniform names to the descriptor binding assigned in + // RewriteShaderWithUbo. Used when updating descriptor sets so textures are + // bound to the sampler they belong to regardless of job iteration order. + QHash sampler_bindings; +}; + +struct VulkanRenderer::StagingBuffer { + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize size = 0; }; static const float kBlitVertices[] = { @@ -168,6 +179,27 @@ void VulkanRenderer::DestroyInternal() vertex_buffer_memory_ = VK_NULL_HANDLE; } + if (staging_buffer_) { + if (staging_buffer_->buffer != VK_NULL_HANDLE) { + vkDestroyBuffer(device_, staging_buffer_->buffer, nullptr); + } + if (staging_buffer_->memory != VK_NULL_HANDLE) { + vkFreeMemory(device_, staging_buffer_->memory, nullptr); + } + delete staging_buffer_; + staging_buffer_ = nullptr; + } + + if (reusable_fence_ != VK_NULL_HANDLE) { + vkDestroyFence(device_, reusable_fence_, nullptr); + reusable_fence_ = VK_NULL_HANDLE; + } + if (reusable_command_buffer_ != VK_NULL_HANDLE) { + vkFreeCommandBuffers(device_, command_pool_, 1, + &reusable_command_buffer_); + reusable_command_buffer_ = VK_NULL_HANDLE; + } + for (auto it = render_pass_cache_.begin(); it != render_pass_cache_.end(); ++it) { if (it.value() != VK_NULL_HANDLE) { vkDestroyRenderPass(device_, it.value(), nullptr); @@ -179,6 +211,7 @@ void VulkanRenderer::DestroyInternal() vkDestroyDescriptorPool(device_, descriptor_pool_, nullptr); descriptor_pool_ = VK_NULL_HANDLE; } + descriptor_sets_since_reset_ = 0; if (command_pool_ != VK_NULL_HANDLE) { vkDestroyCommandPool(device_, command_pool_, nullptr); @@ -189,8 +222,10 @@ void VulkanRenderer::DestroyInternal() vkDestroyDevice(device_, nullptr); device_ = VK_NULL_HANDLE; } + device_lost_ = false; if (instance_ != VK_NULL_HANDLE) { + DestroyDebugMessenger(); vkDestroyInstance(instance_, nullptr); instance_ = VK_NULL_HANDLE; } @@ -207,19 +242,131 @@ bool VulkanRenderer::CreateInstance() app_info.engineVersion = VK_MAKE_VERSION(0, 3, 0); app_info.apiVersion = VK_API_VERSION_1_2; + const bool enable_validation = + qEnvironmentVariableIsSet("OAK_VULKAN_VALIDATION"); + const char *validation_layer = "VK_LAYER_KHRONOS_validation"; + const char *debug_extension = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; + bool has_validation = false; + bool has_debug_extension = false; + + if (enable_validation) { + uint32_t layer_count = 0; + vkEnumerateInstanceLayerProperties(&layer_count, nullptr); + QVector layers(layer_count); + vkEnumerateInstanceLayerProperties(&layer_count, layers.data()); + for (const VkLayerProperties &layer : layers) { + if (strcmp(layer.layerName, validation_layer) == 0) { + has_validation = true; + break; + } + } + + uint32_t extension_count = 0; + vkEnumerateInstanceExtensionProperties(nullptr, &extension_count, nullptr); + QVector extensions(extension_count); + vkEnumerateInstanceExtensionProperties(nullptr, &extension_count, + extensions.data()); + for (const VkExtensionProperties &ext : extensions) { + if (strcmp(ext.extensionName, debug_extension) == 0) { + has_debug_extension = true; + break; + } + } + } + VkInstanceCreateInfo create_info = {}; create_info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; create_info.pApplicationInfo = &app_info; + if (has_validation) { + create_info.enabledLayerCount = 1; + create_info.ppEnabledLayerNames = &validation_layer; + } + if (has_debug_extension) { + create_info.enabledExtensionCount = 1; + create_info.ppEnabledExtensionNames = &debug_extension; + } VkResult result = vkCreateInstance(&create_info, nullptr, &instance_); if (result != VK_SUCCESS) { qWarning() << "Failed to create Vulkan instance:" << result; return false; } + + if (has_validation && has_debug_extension) { + CreateDebugMessenger(); + } + qDebug() << "Vulkan instance created successfully"; return true; } +// Logs validation errors/warnings from the Vulkan validation layers. These are +// the first signal of missing barriers or invalid usage that would otherwise +// become a GPU hang. +VKAPI_ATTR VkBool32 VKAPI_CALL +VulkanRenderer::DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity, + VkDebugUtilsMessageTypeFlagsEXT messageType, + const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData, + void *pUserData) +{ + Q_UNUSED(messageType) + Q_UNUSED(pUserData) + + if (!pCallbackData || !pCallbackData->pMessage) { + return VK_FALSE; + } + + // Only emit errors/warnings. Verbose validation messages are useful during + // bring-up but flood the log and degrade playback performance. + if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) { + qWarning() << "Vulkan validation error:" << pCallbackData->pMessage; + } else if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) { + qWarning() << "Vulkan validation warning:" << pCallbackData->pMessage; + } + + return VK_FALSE; +} + +bool VulkanRenderer::CreateDebugMessenger() +{ + auto create_fn = reinterpret_cast( + vkGetInstanceProcAddr(instance_, "vkCreateDebugUtilsMessengerEXT")); + if (!create_fn) { + qWarning() << "Failed to load vkCreateDebugUtilsMessengerEXT"; + return false; + } + + VkDebugUtilsMessengerCreateInfoEXT create_info = {}; + create_info.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT; + create_info.messageSeverity = + VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT; + create_info.messageType = + VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT; + create_info.pfnUserCallback = DebugCallback; + + VkResult result = create_fn(instance_, &create_info, nullptr, &debug_messenger_); + if (result != VK_SUCCESS) { + qWarning() << "Failed to create Vulkan debug messenger:" << result; + return false; + } + return true; +} + +void VulkanRenderer::DestroyDebugMessenger() +{ + if (debug_messenger_ == VK_NULL_HANDLE || instance_ == VK_NULL_HANDLE) { + return; + } + auto destroy_fn = reinterpret_cast( + vkGetInstanceProcAddr(instance_, "vkDestroyDebugUtilsMessengerEXT")); + if (destroy_fn) { + destroy_fn(instance_, debug_messenger_, nullptr); + } + debug_messenger_ = VK_NULL_HANDLE; +} + // Selects the first physical device with a graphics queue and creates a logical // device without swapchain extensions because viewer output is CPU readback. bool VulkanRenderer::CreateDevice() @@ -374,15 +521,35 @@ VkRenderPass VulkanRenderer::GetOrCreateRenderPass(VkFormat format, bool clear) // correctly synchronized. The destination image is brought in by the // pipeline barrier before the render pass; here we synchronize the render // pass output with whatever stage reads it next. - VkSubpassDependency dependency = {}; - dependency.srcSubpass = 0; - dependency.dstSubpass = VK_SUBPASS_EXTERNAL; - dependency.srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; - dependency.dstStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_TRANSFER_BIT; - dependency.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; - dependency.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | - VK_ACCESS_TRANSFER_READ_BIT; + // + // Two dependencies are required: + // 1) EXTERNAL -> 0: whatever produced the image before the render pass must + // finish before the color attachment output stage starts. + // 2) 0 -> EXTERNAL: the render pass write must complete before the image is + // read again by shaders or transfer commands. + // Without (1), drivers may start the subpass before prior transfer/shader + // writes finish, causing GPU hangs. + VkSubpassDependency dependencies[2] = {}; + + // Use conservative ALL_COMMANDS / MEMORY_READ|WRITE masks. The render pass + // is used after many different prior operations (transfers, shader reads, + // layout transitions, etc.) and an overly narrow dependency is the most + // common cause of VK_ERROR_DEVICE_LOST on the first draw. + dependencies[0].srcSubpass = VK_SUBPASS_EXTERNAL; + dependencies[0].dstSubpass = 0; + dependencies[0].srcStageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + dependencies[0].dstStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + dependencies[0].srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | + VK_ACCESS_MEMORY_WRITE_BIT; + dependencies[0].dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + + dependencies[1].srcSubpass = 0; + dependencies[1].dstSubpass = VK_SUBPASS_EXTERNAL; + dependencies[1].srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + dependencies[1].dstStageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + dependencies[1].srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + dependencies[1].dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | + VK_ACCESS_MEMORY_WRITE_BIT; VkRenderPassCreateInfo render_pass_info = {}; render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO; @@ -390,8 +557,8 @@ VkRenderPass VulkanRenderer::GetOrCreateRenderPass(VkFormat format, bool clear) render_pass_info.pAttachments = &color_attachment; render_pass_info.subpassCount = 1; render_pass_info.pSubpasses = &subpass; - render_pass_info.dependencyCount = 1; - render_pass_info.pDependencies = &dependency; + render_pass_info.dependencyCount = 2; + render_pass_info.pDependencies = dependencies; VkRenderPass render_pass = VK_NULL_HANDLE; VkResult result = vkCreateRenderPass(device_, &render_pass_info, nullptr, @@ -414,7 +581,7 @@ bool VulkanRenderer::CreateVertexBuffer() VkBufferCreateInfo buffer_info = {}; buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; buffer_info.size = buffer_size; - buffer_info.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + buffer_info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; VkBuffer staging_buffer; @@ -495,6 +662,7 @@ bool VulkanRenderer::CreateVertexBuffer() // Copy from staging to device local VkCommandBuffer cmd = BeginOneTimeCommands(); + if (cmd == VK_NULL_HANDLE) { return false; } VkBufferCopy copy_region = {}; copy_region.size = buffer_size; vkCmdCopyBuffer(cmd, staging_buffer, vertex_buffer_, 1, ©_region); @@ -573,23 +741,53 @@ VkSampler VulkanRenderer::GetSampler(Texture::Interpolation interpolation) const } } -// Allocates host-visible coherent memory for one upload/download transfer. +// Returns a renderer-owned host-visible buffer for upload/download transfers. +// Vulkan allocations are expensive and some drivers fragment host-visible heaps +// under repeated 4K/F32 readback. Reusing one submit-and-wait staging buffer +// keeps peak allocation count low while the renderer mutex serializes callers. bool VulkanRenderer::CreateStagingBuffer(VkDeviceSize size, VkBuffer *out_buffer, VkDeviceMemory *out_memory) { + if (size == 0) { + return false; + } + + if (staging_buffer_ && staging_buffer_->size >= size) { + *out_buffer = staging_buffer_->buffer; + *out_memory = staging_buffer_->memory; + return true; + } + + if (staging_buffer_) { + vkDeviceWaitIdle(device_); + if (staging_buffer_->buffer != VK_NULL_HANDLE) { + vkDestroyBuffer(device_, staging_buffer_->buffer, nullptr); + } + if (staging_buffer_->memory != VK_NULL_HANDLE) { + vkFreeMemory(device_, staging_buffer_->memory, nullptr); + } + delete staging_buffer_; + staging_buffer_ = nullptr; + } + VkBufferCreateInfo buffer_info = {}; buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; buffer_info.size = size; - buffer_info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + buffer_info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - VkResult result = vkCreateBuffer(device_, &buffer_info, nullptr, out_buffer); + VkBuffer buffer = VK_NULL_HANDLE; + VkResult result = vkCreateBuffer(device_, &buffer_info, nullptr, &buffer); if (result != VK_SUCCESS) { + qWarning() << "Failed to create Vulkan staging buffer:" << result + << "size=" << qulonglong(size); return false; } VkMemoryRequirements mem_req; - vkGetBufferMemoryRequirements(device_, *out_buffer, &mem_req); + vkGetBufferMemoryRequirements(device_, buffer, &mem_req); VkMemoryAllocateInfo alloc_info = {}; alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; @@ -599,30 +797,45 @@ bool VulkanRenderer::CreateStagingBuffer(VkDeviceSize size, VkBuffer *out_buffer VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); if (alloc_info.memoryTypeIndex == UINT32_MAX) { qWarning() << "Failed to find host-visible memory type for Vulkan staging buffer"; - vkDestroyBuffer(device_, *out_buffer, nullptr); + vkDestroyBuffer(device_, buffer, nullptr); return false; } - result = vkAllocateMemory(device_, &alloc_info, nullptr, out_memory); + VkDeviceMemory memory = VK_NULL_HANDLE; + result = vkAllocateMemory(device_, &alloc_info, nullptr, &memory); if (result != VK_SUCCESS) { - qWarning() << "Failed to allocate Vulkan staging buffer memory:" << result; - vkDestroyBuffer(device_, *out_buffer, nullptr); + qWarning() << "Failed to allocate Vulkan staging buffer memory:" << result + << "size=" << qulonglong(size) + << "allocation=" << qulonglong(mem_req.size); + vkDestroyBuffer(device_, buffer, nullptr); return false; } - result = vkBindBufferMemory(device_, *out_buffer, *out_memory, 0); + result = vkBindBufferMemory(device_, buffer, memory, 0); if (result != VK_SUCCESS) { qWarning() << "Failed to bind Vulkan staging buffer memory:" << result; - vkFreeMemory(device_, *out_memory, nullptr); - vkDestroyBuffer(device_, *out_buffer, nullptr); + vkFreeMemory(device_, memory, nullptr); + vkDestroyBuffer(device_, buffer, nullptr); return false; } + + staging_buffer_ = new StagingBuffer(); + staging_buffer_->buffer = buffer; + staging_buffer_->memory = memory; + staging_buffer_->size = mem_req.size; + *out_buffer = buffer; + *out_memory = memory; return true; } -// Releases a staging buffer and its memory allocation. +// Kept for existing call sites; runtime staging buffers are renderer-owned and +// released in DestroyInternal() or when a larger staging allocation is required. void VulkanRenderer::DestroyStagingBuffer(VkBuffer buffer, VkDeviceMemory memory) { + if (staging_buffer_ && buffer == staging_buffer_->buffer && + memory == staging_buffer_->memory) { + return; + } if (buffer != VK_NULL_HANDLE) { vkDestroyBuffer(device_, buffer, nullptr); } @@ -634,37 +847,94 @@ void VulkanRenderer::DestroyStagingBuffer(VkBuffer buffer, VkDeviceMemory memory // Starts a primary command buffer intended for immediate submit-and-wait use. VkCommandBuffer VulkanRenderer::BeginOneTimeCommands() { - VkCommandBufferAllocateInfo alloc_info = {}; - alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; - alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - alloc_info.commandPool = command_pool_; - alloc_info.commandBufferCount = 1; + if (reusable_command_buffer_ == VK_NULL_HANDLE) { + VkCommandBufferAllocateInfo alloc_info = {}; + alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + alloc_info.commandPool = command_pool_; + alloc_info.commandBufferCount = 1; - VkCommandBuffer cmd; - vkAllocateCommandBuffers(device_, &alloc_info, &cmd); + VkResult result = vkAllocateCommandBuffers( + device_, &alloc_info, &reusable_command_buffer_); + if (result != VK_SUCCESS || reusable_command_buffer_ == VK_NULL_HANDLE) { + qWarning() << "Failed to allocate Vulkan command buffer:" << result; + return VK_NULL_HANDLE; + } + } else { + vkResetCommandBuffer(reusable_command_buffer_, 0); + } VkCommandBufferBeginInfo begin_info = {}; begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - vkBeginCommandBuffer(cmd, &begin_info); - return cmd; + VkResult result = vkBeginCommandBuffer(reusable_command_buffer_, &begin_info); + if (result != VK_SUCCESS) { + qWarning() << "Failed to begin Vulkan command buffer:" << result; + return VK_NULL_HANDLE; + } + return reusable_command_buffer_; } -// Submits a one-time command buffer and waits synchronously for completion. +// Submits a one-time command buffer and waits with a timeout. Using a fence +// instead of vkQueueWaitIdle prevents the CPU thread from blocking forever if +// a bad barrier/shader causes the GPU to hang. void VulkanRenderer::EndOneTimeCommands(VkCommandBuffer cmd) { + if (cmd == VK_NULL_HANDLE) { + return; + } + + if (device_lost_) { + return; + } + vkEndCommandBuffer(cmd); + if (reusable_fence_ == VK_NULL_HANDLE) { + VkFenceCreateInfo fence_info = {}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + + VkResult result = + vkCreateFence(device_, &fence_info, nullptr, &reusable_fence_); + if (result != VK_SUCCESS) { + qWarning() << "Failed to create Vulkan fence:" << result; + return; + } + } else { + vkResetFences(device_, 1, &reusable_fence_); + } + VkSubmitInfo submit_info = {}; submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submit_info.commandBufferCount = 1; submit_info.pCommandBuffers = &cmd; - vkQueueSubmit(graphics_queue_, 1, &submit_info, VK_NULL_HANDLE); - vkQueueWaitIdle(graphics_queue_); + VkResult result = vkQueueSubmit(graphics_queue_, 1, &submit_info, + reusable_fence_); + if (result != VK_SUCCESS) { + if (result == VK_ERROR_DEVICE_LOST) { + if (!device_lost_) { + device_lost_ = true; + qCritical() << "Vulkan device lost during vkQueueSubmit; stopping " + "further GPU submissions"; + } + } else { + qWarning() << "vkQueueSubmit failed:" << result; + } + return; + } - vkFreeCommandBuffers(device_, command_pool_, 1, &cmd); + // 10 second timeout. If the GPU is hung, the process can report it instead + // of blocking forever. Note: a true GPU hang may still freeze the display + // before this timeout is reached, but the CPU-side wait will not deadlock. + constexpr uint64_t kTimeoutNs = 10ULL * 1000ULL * 1000ULL * 1000ULL; + result = vkWaitForFences(device_, 1, &reusable_fence_, VK_TRUE, kTimeoutNs); + if (result == VK_TIMEOUT) { + qCritical() << "Vulkan GPU wait timed out; the GPU may be hung"; + } else if (result != VK_SUCCESS) { + qWarning() << "vkWaitForFences failed:" << result; + } } // Emits a conservative barrier for the image layout transitions used by this @@ -688,24 +958,28 @@ void VulkanRenderer::TransitionImageLayout(VkCommandBuffer cmd, VkImage image, VkPipelineStageFlags source_stage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; VkPipelineStageFlags destination_stage = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; + bool handled = false; auto set_transfer = [&]() { barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; source_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; destination_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + handled = true; }; auto set_shader_read = [&]() { barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; source_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; destination_stage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + handled = true; }; auto set_color_attachment = [&]() { barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; source_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; destination_stage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + handled = true; }; if (old_layout == VK_IMAGE_LAYOUT_UNDEFINED) { @@ -714,15 +988,19 @@ void VulkanRenderer::TransitionImageLayout(VkCommandBuffer cmd, VkImage image, if (new_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; destination_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; destination_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; destination_stage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; destination_stage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + handled = true; } } else if (old_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) { if (new_layout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) { @@ -738,9 +1016,11 @@ void VulkanRenderer::TransitionImageLayout(VkCommandBuffer cmd, VkImage image, if (new_layout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; destination_stage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; destination_stage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) { set_transfer(); } @@ -750,12 +1030,15 @@ void VulkanRenderer::TransitionImageLayout(VkCommandBuffer cmd, VkImage image, if (new_layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; destination_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; destination_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; destination_stage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + handled = true; } } else if (old_layout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) { barrier.srcAccessMask = VK_ACCESS_SHADER_READ_BIT; @@ -763,15 +1046,28 @@ void VulkanRenderer::TransitionImageLayout(VkCommandBuffer cmd, VkImage image, if (new_layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; destination_stage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; destination_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + handled = true; } else if (new_layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) { barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; destination_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + handled = true; } } + if (!handled) { + qWarning() << "Unhandled Vulkan layout transition from" << old_layout + << "to" << new_layout + << "- using conservative ALL_COMMANDS barrier"; + barrier.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + source_stage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + destination_stage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + } + vkCmdPipelineBarrier(cmd, source_stage, destination_stage, 0, 0, nullptr, 0, nullptr, 1, &barrier); } @@ -1045,7 +1341,10 @@ QVariant VulkanRenderer::CreateNativeTexture(int width, int height, int depth, image_info.imageType = depth > 1 ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D; image_info.extent.width = static_cast(width); image_info.extent.height = static_cast(height); - image_info.extent.depth = static_cast(depth); + // Vulkan requires extent.depth >= 1 for all image types; for 2D images it + // must be exactly 1. Some callers pass 0 for 2D textures, which would + // otherwise produce validation errors and device lost. + image_info.extent.depth = static_cast(depth > 0 ? depth : 1); image_info.mipLevels = 1; image_info.arrayLayers = 1; image_info.format = vk_format; @@ -1133,8 +1432,9 @@ QVariant VulkanRenderer::CreateNativeTexture(int width, int height, int depth, VkDeviceSize image_size = static_cast(width) * height * depth * gpu_bytes_per_pixel; if (linesize == 0) { - linesize = width * cpu_bytes_per_pixel; + linesize = width; } + const int row_stride_bytes = linesize * cpu_bytes_per_pixel; VkBuffer staging_buffer; VkDeviceMemory staging_memory; @@ -1142,14 +1442,14 @@ QVariant VulkanRenderer::CreateNativeTexture(int width, int height, int depth, void *mapped; vkMapMemory(device_, staging_memory, 0, image_size, 0, &mapped); if (cpu_bytes_per_pixel == gpu_bytes_per_pixel) { - if (linesize == width * cpu_bytes_per_pixel) { + if (linesize == width) { memcpy(mapped, data, static_cast(image_size)); } else { char *dst = static_cast(mapped); const char *src = static_cast(data); for (int row = 0; row < height * depth; row++) { memcpy(dst + row * width * cpu_bytes_per_pixel, - src + row * linesize, + src + row * row_stride_bytes, static_cast(width * cpu_bytes_per_pixel)); } } @@ -1159,14 +1459,14 @@ QVariant VulkanRenderer::CreateNativeTexture(int width, int height, int depth, // in the staging buffer so the copy uses the GPU texel layout. QByteArray tmp(width * height * depth * cpu_bytes_per_pixel, Qt::Uninitialized); - if (linesize == width * cpu_bytes_per_pixel) { + if (linesize == width) { memcpy(tmp.data(), data, static_cast(tmp.size())); } else { char *dst = tmp.data(); const char *src = static_cast(data); for (int row = 0; row < height * depth; row++) { memcpy(dst + row * width * cpu_bytes_per_pixel, - src + row * linesize, + src + row * row_stride_bytes, static_cast(width * cpu_bytes_per_pixel)); } } @@ -1180,6 +1480,8 @@ QVariant VulkanRenderer::CreateNativeTexture(int width, int height, int depth, vkUnmapMemory(device_, staging_memory); VkCommandBuffer cmd = BeginOneTimeCommands(); + + if (cmd == VK_NULL_HANDLE) { return QVariant(); } TransitionImageLayout(cmd, tex->image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); CopyBufferToImage(cmd, staging_buffer, tex->image, @@ -1196,6 +1498,7 @@ QVariant VulkanRenderer::CreateNativeTexture(int width, int height, int depth, } } else { VkCommandBuffer cmd = BeginOneTimeCommands(); + if (cmd == VK_NULL_HANDLE) { return QVariant(); } TransitionImageLayout(cmd, tex->image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); EndOneTimeCommands(cmd); @@ -1255,8 +1558,9 @@ void VulkanRenderer::UploadToTexture(const QVariant &handle, VkDeviceSize image_size = static_cast(width) * height * depth * gpu_bytes_per_pixel; if (linesize == 0) { - linesize = width * cpu_bytes_per_pixel; + linesize = width; } + const int row_stride_bytes = linesize * cpu_bytes_per_pixel; VkBuffer staging_buffer; VkDeviceMemory staging_memory; @@ -1267,28 +1571,28 @@ void VulkanRenderer::UploadToTexture(const QVariant &handle, void *mapped; vkMapMemory(device_, staging_memory, 0, image_size, 0, &mapped); if (cpu_bytes_per_pixel == gpu_bytes_per_pixel) { - if (linesize == width * cpu_bytes_per_pixel) { + if (linesize == width) { memcpy(mapped, data, static_cast(image_size)); } else { char *dst = static_cast(mapped); const char *src = static_cast(data); for (int row = 0; row < height * depth; row++) { memcpy(dst + row * width * cpu_bytes_per_pixel, - src + row * linesize, + src + row * row_stride_bytes, static_cast(width * cpu_bytes_per_pixel)); } } } else { QByteArray tmp(width * height * depth * cpu_bytes_per_pixel, Qt::Uninitialized); - if (linesize == width * cpu_bytes_per_pixel) { + if (linesize == width) { memcpy(tmp.data(), data, static_cast(tmp.size())); } else { char *dst = tmp.data(); const char *src = static_cast(data); for (int row = 0; row < height * depth; row++) { memcpy(dst + row * width * cpu_bytes_per_pixel, - src + row * linesize, + src + row * row_stride_bytes, static_cast(width * cpu_bytes_per_pixel)); } } @@ -1302,6 +1606,8 @@ void VulkanRenderer::UploadToTexture(const QVariant &handle, vkUnmapMemory(device_, staging_memory); VkCommandBuffer cmd = BeginOneTimeCommands(); + + if (cmd == VK_NULL_HANDLE) { return; } if (tex->current_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) { TransitionImageLayout(cmd, tex->image, tex->current_layout, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); @@ -1340,8 +1646,9 @@ void VulkanRenderer::DownloadFromTexture(const QVariant &handle, gpu_bytes_per_pixel = cpu_bytes_per_pixel; } if (linesize == 0) { - linesize = width * cpu_bytes_per_pixel; + linesize = width; } + const int row_stride_bytes = linesize * cpu_bytes_per_pixel; VkDeviceSize image_size = static_cast(width) * height * gpu_bytes_per_pixel; @@ -1352,6 +1659,8 @@ void VulkanRenderer::DownloadFromTexture(const QVariant &handle, } VkCommandBuffer cmd = BeginOneTimeCommands(); + + if (cmd == VK_NULL_HANDLE) { return; } if (tex->current_layout != VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) { TransitionImageLayout(cmd, tex->image, tex->current_layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); @@ -1364,13 +1673,13 @@ void VulkanRenderer::DownloadFromTexture(const QVariant &handle, void *mapped; vkMapMemory(device_, staging_memory, 0, image_size, 0, &mapped); if (cpu_bytes_per_pixel == gpu_bytes_per_pixel) { - if (linesize == width * cpu_bytes_per_pixel) { + if (linesize == width) { memcpy(data, mapped, static_cast(image_size)); } else { char *dst = static_cast(data); const char *src = static_cast(mapped); for (int row = 0; row < height; row++) { - memcpy(dst + row * linesize, + memcpy(dst + row * row_stride_bytes, src + row * width * cpu_bytes_per_pixel, static_cast(width * cpu_bytes_per_pixel)); } @@ -1384,13 +1693,13 @@ void VulkanRenderer::DownloadFromTexture(const QVariant &handle, width, height, 1, gpu_channels, params.channel_count(), params.format()); - if (linesize != width * cpu_bytes_per_pixel) { + if (linesize != width) { // Repack from tight CPU layout to caller's stride in-place. QByteArray tight(static_cast(data), width * height * cpu_bytes_per_pixel); char *dst = static_cast(data); for (int row = 0; row < height; row++) { - memcpy(dst + row * linesize, + memcpy(dst + row * row_stride_bytes, tight.constData() + row * width * cpu_bytes_per_pixel, static_cast(width * cpu_bytes_per_pixel)); } @@ -1418,6 +1727,8 @@ void VulkanRenderer::ClearDestination(olive::Texture *texture, double r, double QMutexLocker lock(&mutex_); VkCommandBuffer cmd = BeginOneTimeCommands(); + + if (cmd == VK_NULL_HANDLE) { return; } if (texture) { quint64 id = texture->id().value(); VulkanTexture *tex = textures_.value(id); @@ -1482,6 +1793,8 @@ Color VulkanRenderer::GetPixelFromTexture(olive::Texture *texture, } VkCommandBuffer cmd = BeginOneTimeCommands(); + + if (cmd == VK_NULL_HANDLE) { return Color(); } if (tex->current_layout != VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) { TransitionImageLayout(cmd, tex->image, tex->current_layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); @@ -1873,6 +2186,7 @@ QVariant VulkanRenderer::CreateNativeShader(olive::ShaderCode code) sh->id = next_shader_id_++; sh->uniforms = all_uniforms; sh->sampler_count = all_samplers.size(); + sh->sampler_bindings = sampler_bindings; sh->ubo_size = 0; for (const UniformInfo &u : all_uniforms) { sh->ubo_size = qMax(sh->ubo_size, u.offset + u.size); @@ -2182,6 +2496,11 @@ void VulkanRenderer::BlitPass(VulkanShader *shader, VulkanTexture *dest_tex, QVector image_infos; bool descriptors_needed = (shader->ubo_size > 0 || !bindings.isEmpty()); if (descriptors_needed) { + if (descriptor_sets_since_reset_ >= kMaxDescriptorSets - 16) { + vkResetDescriptorPool(device_, descriptor_pool_, 0); + descriptor_sets_since_reset_ = 0; + } + VkDescriptorSetAllocateInfo ds_alloc = {}; ds_alloc.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; ds_alloc.descriptorPool = descriptor_pool_; @@ -2196,6 +2515,7 @@ void VulkanRenderer::BlitPass(VulkanShader *shader, VulkanTexture *dest_tex, } return; } + descriptor_sets_since_reset_++; QVector writes; @@ -2227,10 +2547,19 @@ void VulkanRenderer::BlitPass(VulkanShader *shader, VulkanTexture *dest_tex, img_info.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; image_infos.append(img_info); + // Use the binding assigned to this sampler name when the shader + // was compiled. This keeps descriptor writes in sync with the + // rewritten layout() bindings even when job value iteration + // orders the samplers differently. + int binding = shader->sampler_bindings.value(tb.name, -1); + if (binding < 0) { + binding = 1 + i; + } + VkWriteDescriptorSet write = {}; write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; write.dstSet = descriptor_set; - write.dstBinding = 1 + i; + write.dstBinding = static_cast(binding); write.dstArrayElement = 0; write.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; write.descriptorCount = 1; @@ -2247,6 +2576,8 @@ void VulkanRenderer::BlitPass(VulkanShader *shader, VulkanTexture *dest_tex, VkCommandBuffer cmd = BeginOneTimeCommands(); + if (cmd == VK_NULL_HANDLE) { return; } + if (dest_tex->current_layout != VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL) { TransitionImageLayout(cmd, dest_tex->image, dest_tex->current_layout, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); @@ -2322,9 +2653,6 @@ void VulkanRenderer::BlitPass(VulkanShader *shader, VulkanTexture *dest_tex, EndOneTimeCommands(cmd); - if (descriptor_set != VK_NULL_HANDLE) { - vkFreeDescriptorSets(device_, descriptor_pool_, 1, &descriptor_set); - } if (ubo_buffer != VK_NULL_HANDLE) { DestroyStagingBuffer(ubo_buffer, ubo_memory); } diff --git a/app/render/vulkan/vulkanrenderer.h b/app/render/vulkan/vulkanrenderer.h index b1e65f22d..c0d66a17e 100644 --- a/app/render/vulkan/vulkanrenderer.h +++ b/app/render/vulkan/vulkanrenderer.h @@ -73,6 +73,11 @@ public: // Waits for outstanding device work to complete. virtual void Flush() override; + virtual bool IsVulkan() const override + { + return true; + } + // Reads a single texture pixel using a one-pixel transfer readback. virtual Color GetPixelFromTexture(olive::Texture *texture, const QPointF &pt) override; @@ -102,9 +107,21 @@ private: struct VulkanTexture; struct VulkanShader; struct UniformInfo; + struct StagingBuffer; // Creates the Vulkan instance used for all offscreen work. bool CreateInstance(); + // Creates the debug messenger when validation layers are available. + bool CreateDebugMessenger(); + // Destroys the debug messenger before the instance is destroyed. + void DestroyDebugMessenger(); + // Validation layer callback; logs errors/warnings so synchronization issues + // are visible before they become GPU hangs. + static VKAPI_ATTR VkBool32 VKAPI_CALL + DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity, + VkDebugUtilsMessageTypeFlagsEXT messageType, + const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData, + void *pUserData); // Chooses a graphics-capable physical device and creates the logical device. bool CreateDevice(); // Creates a command pool for short-lived command buffers. @@ -212,7 +229,11 @@ private: bool clear_destination, int iteration); VkInstance instance_ = VK_NULL_HANDLE; + VkDebugUtilsMessengerEXT debug_messenger_ = VK_NULL_HANDLE; VkPhysicalDevice physical_device_ = VK_NULL_HANDLE; + // Set to true after the first VK_ERROR_DEVICE_LOST so we stop submitting + // work and don't flood the log with identical errors. + bool device_lost_ = false; uint32_t physical_device_count_ = 0; VkDevice device_ = VK_NULL_HANDLE; VkQueue graphics_queue_ = VK_NULL_HANDLE; @@ -223,9 +244,13 @@ private: VkSampler nearest_sampler_ = VK_NULL_HANDLE; QHash render_pass_cache_; + int descriptor_sets_since_reset_ = 0; VkBuffer vertex_buffer_ = VK_NULL_HANDLE; VkDeviceMemory vertex_buffer_memory_ = VK_NULL_HANDLE; + StagingBuffer *staging_buffer_ = nullptr; + VkCommandBuffer reusable_command_buffer_ = VK_NULL_HANDLE; + VkFence reusable_fence_ = VK_NULL_HANDLE; VkPhysicalDeviceMemoryProperties mem_properties_; VkPhysicalDeviceProperties device_properties_; diff --git a/app/render/worker/workermain.cpp b/app/render/worker/workermain.cpp index 68d173ef5..1e8ac70e9 100644 --- a/app/render/worker/workermain.cpp +++ b/app/render/worker/workermain.cpp @@ -33,9 +33,12 @@ #include "common/qtutils.h" #include "config/config.h" +#include "core.h" #include "node/factory.h" #include "node/input/multicam/multicamnode.h" #include "node/project/serializer/serializer.h" +#include "render/diskmanager.h" +#include "render/framemanager.h" #include "render/ipc/frameslotpool.h" #include "render/ipc/ipcmessage.h" #include "render/ipc/sharedmemoryregion.h" @@ -93,14 +96,26 @@ public: { project_.reset(); olive::ProjectSerializer::Destroy(); + olive::DiskManager::DestroyInstance(); + olive::FrameManager::DestroyInstance(); olive::NodeFactory::Destroy(); } bool InitializeRuntime() { + + // Create a minimal Core instance so that code paths calling Core::instance() + // (e.g. ViewerOutput::data for timecode display) do not dereference null. + // The worker is short-lived; leaking this on exit is harmless. + if (!olive::Core::instance()) { + new olive::Core(olive::Core::CoreParams()); + } + olive::Config::Load(); olive::NodeFactory::Initialize(); olive::ColorManager::SetUpDefaultConfig(); + olive::FrameManager::CreateInstance(); + olive::DiskManager::CreateInstance(); olive::ProjectSerializer::Initialize(); return true; } @@ -243,7 +258,9 @@ private: bool LoadGraph(const QString &path) { auto loaded = std::make_unique(); - loaded->Initialize(); + // Do not call Initialize() here: project serializers expect a blank + // project (root_ == nullptr) and will set root themselves. Calling + // Initialize() first triggers Q_ASSERT(!root_) in Project::Load. olive::ProjectSerializer::Result result = olive::ProjectSerializer::Load(loaded.get(), path, olive::ProjectSerializer::kProject); @@ -340,6 +357,11 @@ private: message.ticket_id)); } input_slots.append(int(consumed_slot)); + + const olive::ipc::FrameSlotMeta *meta = + input_pool_->Meta(consumed_slot); + if (meta) { + } } } @@ -408,10 +430,12 @@ private: return Write(ErrorMessage(QStringLiteral("no free output frame slot"), message.ticket_id)); } - const int data_size = frame->allocated_size(); + const int data_size = frame->linesize_bytes()*frame->height(); if (data_size > int(output_pool_->slot_data_bytes())) { output_pool_->Release(slot); - return Write(ErrorMessage(QStringLiteral("rendered frame does not fit output slot"), + LogError(QString("Output frame size")+QString::number(data_size)); + LogError(QString("Slot size")+QString::number(output_pool_->slot_data_bytes())); + return Write(ErrorMessage(QStringLiteral("rendered frame does not fit output slot "), message.ticket_id)); } @@ -432,7 +456,6 @@ private: return Write(ErrorMessage(QStringLiteral("failed to publish output frame slot"), message.ticket_id)); } - olive::ipc::FrameReadyMsg ready; ready.ticket_id = message.ticket_id; ready.output_slot = int(slot); @@ -463,6 +486,15 @@ int main(int argc, char *argv[]) QCoreApplication::setOrganizationName(QStringLiteral("oakvideoeditor.org")); QCoreApplication::setApplicationName(QStringLiteral("olive-render-worker")); + QString backend = QStringLiteral("opengl"); + const QStringList args = app.arguments(); + for (int i = 1; i < args.size(); ++i) { + if (args[i] == QStringLiteral("--backend") && i + 1 < args.size()) { + backend = args[i + 1].toLower(); + ++i; + } + } + QFile in; QFile out; if (!in.open(stdin, QIODevice::ReadOnly | QIODevice::Unbuffered) || @@ -473,13 +505,14 @@ int main(int argc, char *argv[]) olive::Renderer *renderer; #ifdef OAK_ENABLE_DYNAMIC_RENDER_BACKEND - auto *dynamic_renderer = new olive::DynamicRenderer(QStringLiteral("opengl")); + auto *dynamic_renderer = new olive::DynamicRenderer(backend); if (dynamic_renderer->Init()) { dynamic_renderer->PostInit(); renderer = dynamic_renderer; } else { delete dynamic_renderer; - qWarning() << "Failed to initialize dynamic OpenGL backend, falling back to direct OpenGL renderer"; + qWarning() << "Failed to initialize dynamic" << backend + << "backend, falling back to direct OpenGL renderer"; renderer = new olive::OpenGLRenderer(); if (!renderer->Init()) { LogError(QStringLiteral("failed to initialize OpenGL renderer")); @@ -498,16 +531,24 @@ int main(int argc, char *argv[]) renderer->PostInit(); #endif + // Validate the renderer. For OpenGL we check the GL context; for Vulkan we + // rely on Init()/PostInit() succeeding (there is no QOpenGLContext). + bool renderer_valid = true; QOpenGLContext *ctx = nullptr; + if (backend == QStringLiteral("opengl")) { #ifdef OAK_ENABLE_DYNAMIC_RENDER_BACKEND - if (auto *loaded_renderer = dynamic_cast(renderer)) { - ctx = loaded_renderer->OpenGLContext(); - } else + if (auto *loaded_renderer = dynamic_cast(renderer)) { + ctx = loaded_renderer->OpenGLContext(); + } else #endif - { - ctx = static_cast(renderer)->context(); + { + ctx = static_cast(renderer)->context(); + } + if (!ctx || !ctx->isValid()) { + renderer_valid = false; + } } - if (!ctx || !ctx->isValid()) { + if (!renderer_valid) { LogError(QStringLiteral("OpenGL context is not valid after init")); renderer->Destroy(); renderer->PostDestroy(); diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 177ef4834..ce174501c 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -97,6 +97,13 @@ ViewerWidget::ViewerWidget(ViewerDisplayWidget *display, QWidget *parent) &ViewerWidget::CursorColor); connect(display_widget_, &ViewerDisplayWidget::ColorProcessorChanged, this, &ViewerWidget::ColorProcessorChanged); + connect(display_widget_, &ViewerDisplayWidget::ColorProcessorChanged, this, + [](ColorProcessorPtr processor) { + RenderManager::instance()->GetCacher()->SetDisplayColorProcessor( + processor); + }); + RenderManager::instance()->GetCacher()->SetDisplayColorProcessor( + display_widget_->GetCurrentColorProcessor()); connect(display_widget_, &ViewerDisplayWidget::ColorManagerChanged, this, &ViewerWidget::ColorManagerChanged); connect(display_widget_, &ViewerDisplayWidget::DragEntered, this, @@ -205,9 +212,6 @@ void ViewerWidget::TimeChangedEvent(const rational &time) if (GetConnectedNode() && last_time_ != time) { if (!IsPlaying()) { - qDebug() << "[VIEWER] TimeChanged seeking to" << time.toDouble() - << "frame_exists=" << FrameExistsAtTime(time) - << "might_be_still=" << ViewerMightBeAStill(); UpdateTextureFromNode(); PushScrubbedAudio(); @@ -970,6 +974,17 @@ void ViewerWidget::QueueNoLongerStarved() } void ViewerWidget::ForceRequeueFromCurrentTime() +{ + // Defer the requeue to the next event-loop iteration. This function is often + // called from paintEvent paths (QueueStarved) where synchronously cancelling + // watchers can re-enter the same RenderTicket mutex and deadlock. + QMetaObject::invokeMethod( + this, + [this]() { ForceRequeueFromCurrentTimeInternal(); }, + Qt::QueuedConnection); +} + +void ViewerWidget::ForceRequeueFromCurrentTimeInternal() { // Allow half a second for requeue to complete static const rational kRequeueWaitTime(1); @@ -980,7 +995,7 @@ void ViewerWidget::ForceRequeueFromCurrentTime() playback_queue_next_frame_ = GetTimestamp() + playback_speed_ * Timecode::time_to_timestamp( - kRequeueWaitTime, timebase(), Timecode::kFloor); + kRequeueWaitTime, timebase(), Timecode::kFloor); ; first_requeue_watcher_ = nullptr; for (int i = 0; i < queue; i++) { @@ -1448,11 +1463,6 @@ void ViewerWidget::WindowAboutToClose() void ViewerWidget::RendererGeneratedFrame() { RenderTicketWatcher *ticket = static_cast(sender()); - rational t = ticket->property("time").value(); - bool has_result = ticket->HasResult(); - qDebug() << "[VIEWER] RendererGeneratedFrame time=" << t.toDouble() - << "has_result=" << has_result - << "nonqueue_size=" << nonqueue_watchers_.size(); if (nonqueue_watchers_.contains(ticket)) { while (!nonqueue_watchers_.isEmpty()) { @@ -1463,15 +1473,6 @@ void ViewerWidget::RendererGeneratedFrame() } if (ticket->HasResult()) { - QVariant v = ticket->Get(); - bool is_tex = v.canConvert(); - bool is_frame = v.canConvert(); - TexturePtr tex = v.value(); - qDebug() << "[VIEWER] SetDisplayImage time=" << t.toDouble() - << "is_texture=" << is_tex - << "is_frame=" << is_frame - << "tex_null=" << (tex == nullptr) - << "tex_dummy=" << (tex ? tex->IsDummy() : true); SetDisplayImage(ticket->GetTicket()); } } diff --git a/app/widget/viewer/viewer.h b/app/widget/viewer/viewer.h index 4075f121c..9e528d64c 100644 --- a/app/widget/viewer/viewer.h +++ b/app/widget/viewer/viewer.h @@ -407,6 +407,7 @@ private slots: void QueueNoLongerStarved(); void ForceRequeueFromCurrentTime(); + void ForceRequeueFromCurrentTimeInternal(); void UpdateAudioProcessor(); diff --git a/app/widget/viewer/viewerdisplay.cpp b/app/widget/viewer/viewerdisplay.cpp index cd657796b..e29fb5d9b 100644 --- a/app/widget/viewer/viewerdisplay.cpp +++ b/app/widget/viewer/viewerdisplay.cpp @@ -410,24 +410,24 @@ void ViewerDisplayWidget::OnPaint() DrawBlank(device_params); } } else if (color_service()) { + bool drew_backend_neutral_frame = false; if (FramePtr frame = load_frame_.value()) { - // This is a CPU frame, upload it now - if (!texture_ || + if (!drew_backend_neutral_frame && (!texture_ || texture_->renderer() != renderer() // Some implementations don't like it if we upload to a texture created in another (albeit shared) context || texture_->width() != frame->width() || texture_->height() != frame->height() || texture_->format() != frame->format() || - texture_->channel_count() != frame->channel_count()) { + texture_->channel_count() != frame->channel_count())) { texture_ = renderer()->CreateTexture( frame->video_params(), frame->data(), frame->linesize_pixels()); - } else { + } else if (!drew_backend_neutral_frame) { texture_->Upload(frame->data(), frame->linesize_pixels()); } } else if (TexturePtr texture = load_frame_.value()) { // This is a GPU texture, switch to it directly when possible. - if (texture && texture->renderer() && + if (!drew_backend_neutral_frame && texture && texture->renderer() && texture->renderer() != renderer()) { if (texture->renderer()->IsOpenGL() && renderer()->IsOpenGL()) { // Shared OpenGL contexts can display the producer texture @@ -450,69 +450,74 @@ void ViewerDisplayWidget::OnPaint() texture_ = texture; } } - } else { + } else if (!drew_backend_neutral_frame) { texture_ = texture; } } else { texture_ = LoadCustomTextureFromFrame(load_frame_); } + if (drew_backend_neutral_frame) { + texture_ = nullptr; + } + emit TextureChanged(texture_); push_mode_ = kPushUnnecessary; - TexturePtr texture_to_draw = texture_; + if (!drew_backend_neutral_frame) { + TexturePtr texture_to_draw = texture_; - if (!texture_to_draw || texture_to_draw->IsDummy()) { - if (!backend_neutral) { - DrawBlank(device_params); - } - } else { - if (deinterlace_) { - if (deinterlace_shader_.isNull()) { - deinterlace_shader_ = renderer()->CreateNativeShader( - ShaderCode(FileFunctions::ReadFileAsString( - QStringLiteral(":/shaders/deinterlace.frag")))); + if (!texture_to_draw || texture_to_draw->IsDummy()) { + if (!backend_neutral) { + DrawBlank(device_params); + } + } else { + if (deinterlace_) { + if (deinterlace_shader_.isNull()) { + deinterlace_shader_ = renderer()->CreateNativeShader( + ShaderCode(FileFunctions::ReadFileAsString( + QStringLiteral(":/shaders/deinterlace.frag")))); + } + + if (!deinterlace_texture_ || + deinterlace_texture_->params() != + texture_to_draw->params()) { + // (Re)create texture + deinterlace_texture_ = renderer()->CreateTexture( + texture_to_draw->params()); + } + + ShaderJob job; + job.Insert(QStringLiteral("resolution_in"), + NodeValue(NodeValue::kVec2, + QVector2D(texture_to_draw->width(), + texture_to_draw->height()))); + job.Insert(QStringLiteral("ove_maintex"), + NodeValue(NodeValue::kTexture, + QVariant::fromValue(texture_to_draw))); + + renderer()->BlitToTexture(deinterlace_shader_, job, + deinterlace_texture_.get()); + + texture_to_draw = deinterlace_texture_; } - if (!deinterlace_texture_ || - deinterlace_texture_->params() != - texture_to_draw->params()) { - // (Re)create texture - deinterlace_texture_ = renderer()->CreateTexture( - texture_to_draw->params()); - } + ctj.SetColorProcessor(color_service()); + ctj.SetInputTexture(texture_to_draw); + ctj.SetInputAlphaAssociation( + OLIVE_CONFIG("ReassocLinToNonLin").toBool() ? + kAlphaAssociated : + kAlphaNone); + ctj.SetClearDestinationEnabled(false); + ctj.SetTransformMatrix(combined_matrix_flipped_); + ctj.SetCropMatrix(crop_matrix_); + ctj.SetForceOpaque(true); - ShaderJob job; - job.Insert(QStringLiteral("resolution_in"), - NodeValue(NodeValue::kVec2, - QVector2D(texture_to_draw->width(), - texture_to_draw->height()))); - job.Insert(QStringLiteral("ove_maintex"), - NodeValue(NodeValue::kTexture, - QVariant::fromValue(texture_to_draw))); - - renderer()->BlitToTexture(deinterlace_shader_, job, - deinterlace_texture_.get()); - - texture_to_draw = deinterlace_texture_; + have_ctj = true; } - - ctj.SetColorProcessor(color_service()); - ctj.SetInputTexture(texture_to_draw); - ctj.SetInputAlphaAssociation( - OLIVE_CONFIG("ReassocLinToNonLin").toBool() ? - kAlphaAssociated : - kAlphaNone); - ctj.SetClearDestinationEnabled(false); - ctj.SetTransformMatrix(combined_matrix_flipped_); - ctj.SetCropMatrix(crop_matrix_); - ctj.SetForceOpaque(true); - - have_ctj = true; } } else { - qDebug() << "[VIEWER] OnPaint no color_service, skipping texture draw"; } } @@ -648,6 +653,13 @@ void ViewerDisplayWidget::OnPaint() p.setBrush(highlight); p.drawRect(QRect(add_band_start_, add_band_end_).normalized()); } + + // In backend-neutral mode there is no native buffer swap, so Qt will not + // emit frameSwapped automatically. Emit it ourselves so the playback queue + // keeps advancing (UpdateFromQueue is connected to it during Play()). + if (backend_neutral) { + emit frameSwapped(); + } } void ViewerDisplayWidget::OnDestroy() @@ -667,6 +679,11 @@ void ViewerDisplayWidget::OnDestroy() deinterlace_texture_ = nullptr; backend_neutral_texture_ = nullptr; backend_neutral_buffer_.clear(); + backend_neutral_cpu_image_ = QImage(); + backend_neutral_cpu_display_frame_.reset(); + backend_neutral_cpu_source_frame_.reset(); + backend_neutral_cpu_source_texture_.reset(); + backend_neutral_cpu_color_id_.clear(); if (load_frame_.isNull()) { push_mode_ = kPushNull; } else { @@ -717,9 +734,17 @@ void ViewerDisplayWidget::UpdateMatrix() { combined_matrix_ = scale_matrix_ * translate_matrix_; - combined_matrix_flipped_.setToIdentity(); - combined_matrix_flipped_.scale(1.0, -1.0, 1.0); - combined_matrix_flipped_ *= combined_matrix_; + combined_matrix_flipped_ = combined_matrix_; + // OpenGL's framebuffer origin is bottom-left and texture data is uploaded + // top-down, so the viewer matrix must flip Y to display images right-side + // up. Vulkan's framebuffer and texture coordinate origins are both top-left, + // so the same flip would invert the image. Default to the OpenGL flip when + // no renderer is available yet. + if (!renderer() || !renderer()->IsVulkan()) { + QMatrix4x4 flip; + flip.scale(1.0f, -1.0f, 1.0f); + combined_matrix_flipped_ = flip * combined_matrix_flipped_; + } update(); } @@ -1398,6 +1423,127 @@ void ViewerDisplayWidget::DrawBlank(const VideoParams &device_params) renderer()->Blit(blank_shader_, job, device_params, false); } +bool ViewerDisplayWidget::DrawBackendNeutralFrame(const FramePtr &frame, + QPainter *painter) +{ + if (!frame || !frame->is_allocated() || !painter || !painter->isActive() || + !color_service()) { + return false; + } + + const QString color_id = QString::fromUtf8(color_service()->id()); + if (backend_neutral_cpu_source_frame_.get() == frame.get() && + backend_neutral_cpu_color_id_ == color_id && + !backend_neutral_cpu_image_.isNull()) { + painter->save(); + painter->setRenderHint(QPainter::SmoothPixmapTransform, true); + painter->setWorldTransform(GenerateWorldTransform(), false); + painter->drawImage(rect(), backend_neutral_cpu_image_); + painter->restore(); + return true; + } + + // Do not run OCIO CPU conversion from paintEvent. Some OCIO processors are + // not safe to apply on this GUI path and a crash here kills preview. Worker + // frames tagged with display: have already been color managed; + // untagged frames are drawn directly as a safe fallback. + FramePtr display_frame = frame; + + QImage source_image; + if (display_frame->format() == PixelFormat::U8 && + display_frame->channel_count() == VideoParams::kRGBAChannelCount) { + backend_neutral_cpu_display_frame_ = display_frame; + backend_neutral_cpu_image_ = QImage( + reinterpret_cast(display_frame->const_data()), + display_frame->width(), display_frame->height(), + display_frame->linesize_bytes(), QImage::Format_RGBA8888); + source_image = backend_neutral_cpu_image_; + } else if (display_frame->format() == PixelFormat::U8 && + display_frame->channel_count() == VideoParams::kRGBChannelCount) { + backend_neutral_cpu_display_frame_ = display_frame; + backend_neutral_cpu_image_ = QImage( + reinterpret_cast(display_frame->const_data()), + display_frame->width(), display_frame->height(), + display_frame->linesize_bytes(), QImage::Format_RGB888); + source_image = backend_neutral_cpu_image_; + } else { + backend_neutral_cpu_display_frame_.reset(); + const int bytes_per_pixel = display_frame->video_params().GetBytesPerPixel(); + if (backend_neutral_cpu_image_.size() != + QSize(display_frame->width(), display_frame->height()) || + backend_neutral_cpu_image_.format() != QImage::Format_RGBA8888) { + backend_neutral_cpu_image_ = + QImage(display_frame->width(), display_frame->height(), + QImage::Format_RGBA8888); + } + + for (int y = 0; y < display_frame->height(); ++y) { + uchar *dst = backend_neutral_cpu_image_.scanLine(y); + const char *src = display_frame->const_data() + + y * display_frame->linesize_bytes(); + for (int x = 0; x < display_frame->width(); ++x) { + Color c(src + x * bytes_per_pixel, display_frame->format(), + display_frame->channel_count()); + dst[x * 4 + 0] = + static_cast(qBound(0, int(c.red() * 255.0), 255)); + dst[x * 4 + 1] = + static_cast(qBound(0, int(c.green() * 255.0), 255)); + dst[x * 4 + 2] = + static_cast(qBound(0, int(c.blue() * 255.0), 255)); + dst[x * 4 + 3] = 255; + } + } + source_image = backend_neutral_cpu_image_; + } + + backend_neutral_cpu_source_frame_ = frame; + backend_neutral_cpu_color_id_ = color_id; + + painter->save(); + painter->setRenderHint(QPainter::SmoothPixmapTransform, true); + painter->setWorldTransform(GenerateWorldTransform(), false); + painter->drawImage(rect(), source_image); + painter->restore(); + return true; +} + +bool ViewerDisplayWidget::DrawBackendNeutralTexture(const TexturePtr &texture, + QPainter *painter) +{ + if (!texture || texture->IsDummy() || !texture->renderer() || !painter || + !painter->isActive() || !color_service()) { + return false; + } + + const QString color_id = QString::fromUtf8(color_service()->id()); + if (backend_neutral_cpu_source_texture_.get() == texture.get() && + backend_neutral_cpu_color_id_ == color_id && + !backend_neutral_cpu_image_.isNull()) { + painter->save(); + painter->setRenderHint(QPainter::SmoothPixmapTransform, true); + painter->setWorldTransform(GenerateWorldTransform(), false); + painter->drawImage(rect(), backend_neutral_cpu_image_); + painter->restore(); + return true; + } + + FramePtr frame = Frame::Create(); + frame->set_video_params(texture->params()); + if (!frame->allocate()) { + return false; + } + + texture->Download(frame->data(), frame->linesize_pixels()); + + if (!DrawBackendNeutralFrame(frame, painter)) { + return false; + } + + backend_neutral_cpu_source_texture_ = texture; + backend_neutral_cpu_color_id_ = color_id; + return true; +} + // Renders a backend-neutral frame by drawing into an offscreen backend texture, // downloading it to CPU memory, then painting that image with QPainter. void ViewerDisplayWidget::DrawBackendNeutral(const ColorTransformJob &ctj, diff --git a/app/widget/viewer/viewerdisplay.h b/app/widget/viewer/viewerdisplay.h index d312580b5..56aa8c9a6 100644 --- a/app/widget/viewer/viewerdisplay.h +++ b/app/widget/viewer/viewerdisplay.h @@ -22,9 +22,11 @@ #ifndef VIEWERGLWIDGET_H #define VIEWERGLWIDGET_H +#include #include #include +#include "codec/frame.h" #include "node/color/colormanager/colormanager.h" #include "node/gizmo/text.h" #include "node/node.h" @@ -137,6 +139,11 @@ public: return texture_; } + ColorProcessorPtr GetCurrentColorProcessor() + { + return color_service(); + } + void Play(const int64_t &start_timestamp, const int &playback_speed, const rational &timebase, bool start_updating); @@ -328,6 +335,8 @@ private: void DrawBlank(const VideoParams &device_params); void DrawBackendNeutral(const ColorTransformJob &ctj, QPainter *painter); + bool DrawBackendNeutralFrame(const FramePtr &frame, QPainter *painter); + bool DrawBackendNeutralTexture(const TexturePtr &texture, QPainter *painter); /** * @brief Internal reference to the OpenGL texture to draw. Set in SetTexture() and used in paintGL(). @@ -351,6 +360,11 @@ private: * @brief CPU readback buffer for backend_neutral_texture_. */ QByteArray backend_neutral_buffer_; + QImage backend_neutral_cpu_image_; + FramePtr backend_neutral_cpu_display_frame_; + FramePtr backend_neutral_cpu_source_frame_; + TexturePtr backend_neutral_cpu_source_texture_; + QString backend_neutral_cpu_color_id_; /** * @brief Deinterlace shader diff --git a/docs/zh/render-process-isolation-plan.md b/docs/zh/render-process-isolation-plan.md index 445a4e0ce..6eddf634b 100644 --- a/docs/zh/render-process-isolation-plan.md +++ b/docs/zh/render-process-isolation-plan.md @@ -191,7 +191,7 @@ compact `QJsonObject`，`\n` 结尾。仅承载低频控制流量（握手、提 - 当前仅支持普通视频 `ReturnType::kFrame`；素材输入仍按阶段 4 处理，失败或不支持时回退旧路径。 - ✅ `RenderManager` 增加 `kMultiProcess` backend 分支（与 `kOpenGL` 并存），开关开启且 WorkerPool 接受任务时 `RenderFrame()` 走 `RenderWorkerPool`。 -- ✅ `Config` 增加 `RenderProcessIsolationEnabled`，默认 `false`，默认仍走进程内 `kOpenGL`。 +- ✅ 多进程渲染已设为唯一视频渲染路径，`RenderProcessIsolationEnabled` 配置项已移除。 - 待补：常驻 N worker、忙闲/负载派发、崩溃重启与重派、Viewer 开关实测。 **验证结果**： diff --git a/docs/zh/v04-color-audio-performance-manual-test-plan.md b/docs/zh/v04-color-audio-performance-manual-test-plan.md index bd21c1907..ace1b3e13 100644 --- a/docs/zh/v04-color-audio-performance-manual-test-plan.md +++ b/docs/zh/v04-color-audio-performance-manual-test-plan.md @@ -410,7 +410,18 @@ Vulkan 测试必须先区分两类环境： 通过标准：Viewer 通过 Vulkan backend-neutral readback 路径正常显示，播放和 seek 不崩溃；画面比例、裁切、缩放和 device pixel ratio 正常；没有长期黑屏、上一帧残留或 UI 死锁。 -### 10.5 Vulkan 调色/LUT 显示一致性 +### 10.5 Vulkan H.265 4:2:2 4K 播放 + +1. 准备一段 `h265_422_4k.mov`，使用 `ffprobe` 确认视频流为 `hevc`，`pix_fmt` 为 `yuv422p10le` 或 `yuv422p12le`。 +2. 选择 Vulkan 并重启。 +3. 导入 `h265_422_4k.mov`，放入时间线并播放 10 秒。 +4. 拖动时间线到多个位置，选择不同节点并重复刷新 Viewer。 +5. 观察日志中是否出现 `Failed to allocate Vulkan staging buffer memory`。 +6. 切换 OpenGL 后端重复同一素材播放，作为解码路径对照。 + +通过标准：Vulkan 下 Viewer 不黑屏、不闪烁且能稳定 seek；日志不应反复出现 Vulkan staging buffer 分配失败；若 Vulkan 环境确实内存不足，应给出明确失败或回退行为，不能持续显示一个非空但不可用的黑屏 texture。OpenGL 对照可播放时，Vulkan 失败应记录为 Vulkan 路径问题而不是素材不支持。 + +### 10.6 Vulkan 调色/LUT 显示一致性 1. 选择 Vulkan 并重启。 2. 将 `color_chart.mov` 放入时间线。 @@ -421,7 +432,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：Vulkan 与 OpenGL 预览颜色方向一致，LUT 和三向色轮均生效；不要求像素完全一致，但不能出现通道错乱、alpha 错误、明显 gamma 反转或 LUT 失效。 -### 10.6 Vulkan 代理媒体与重素材播放 +### 10.7 Vulkan 代理媒体与重素材播放 1. 选择 Vulkan 并重启。 2. 对 `8k_or_heavy_camera.mov` 生成代理并启用代理。 @@ -431,7 +442,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：启用代理后 Viewer 可播放且不崩溃；禁用代理后回到原片路径；保存重开后代理状态一致；Vulkan 路径不应把导出源降级为代理。 -### 10.7 Vulkan 软件导出 +### 10.8 Vulkan 软件导出 1. 选择 Vulkan 并重启。 2. 创建 10 秒 sequence，包含 `color_chart.mov`、LUT、三向调色、一个代理 clip 和一段音频。 @@ -441,7 +452,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：Vulkan 下导出成功，输出可播放，音画同步不超过 1 帧；颜色处理和 OpenGL 导出方向一致；启用代理时导出仍使用原片质量路径；失败时有明确错误，不生成损坏的完成文件。 -### 10.8 Vulkan Scope 行为 +### 10.9 Vulkan Scope 行为 1. 选择 Vulkan 并重启。 2. 打开 Waveform、Vectorscope、Histogram。 @@ -451,7 +462,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：当前 backend-neutral Scope 若仍是安全跳过，应明确记录为已知限制，且不能崩溃或卡死；OpenGL 下 Scope 必须正常更新。若 Vulkan Scope 已实现，则三类 Scope 必须随当前帧和调色变化更新。 -### 10.9 Vulkan OpenFX CPU 回退 +### 10.10 Vulkan OpenFX CPU 回退 1. 选择 Vulkan 并重启。 2. 在 clip 上添加一个已知可用的 OFX 插件，优先选择支持 CPU 渲染且效果明显的插件。 @@ -461,7 +472,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：Vulkan 下 OFX 插件不因缺少 OpenGL context 而被跳过或崩溃；CPU 回退输出可见且可导出；OpenGL 下原有 OFX OpenGL 路径不回退或失效。 -### 10.10 Vulkan 后端长时间稳定性 +### 10.11 Vulkan 后端长时间稳定性 1. 选择 Vulkan 并重启。 2. 打开包含 4K/8K、LUT、代理、音频和至少 10 个 clip 的项目。 @@ -471,7 +482,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：无崩溃、无持续不可控内存增长、无明显 Vulkan validation/driver error；停止播放后仍可保存项目和退出应用。 -### 10.11 Vulkan 驱动缺失或不可用 +### 10.12 Vulkan 驱动缺失或不可用 1. 在没有 Vulkan Runtime 或驱动不可用的机器上选择 Vulkan。 2. 重启 Oak。 @@ -481,7 +492,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：应用可以启动；日志应说明 Vulkan 请求不可完全满足或当前回退 OpenGL；`RenderManager::backend()` 必须与实际运行后端一致；用户能回到 Preferences 改回 OpenGL。 -### 10.12 从 Vulkan 切回 OpenGL +### 10.13 从 Vulkan 切回 OpenGL 1. 在 Vulkan 已选中状态下打开 Preferences。 2. 将 Graphics Backend 改为 OpenGL。 @@ -490,7 +501,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：重启后显示 OpenGL；播放和导出正常；不会保留错误的 Vulkan 状态。 -### 10.13 代理、Scope 与调色组合回归 +### 10.14 代理、Scope 与调色组合回归 1. 选择 Vulkan 并重启。 2. 对 `8k_or_heavy_camera.mov` 生成并启用代理。 @@ -500,7 +511,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：Vulkan 请求状态下代理、Scope、调色不崩溃；切回 OpenGL 后项目状态一致；两种选择下导出默认仍使用原片。 -### 10.14 动态 OpenGL 后端加载 +### 10.15 动态 OpenGL 后端加载 1. 使用开启 `OAK_ENABLE_DYNAMIC_RENDER_BACKEND` 的实验构建。 2. 确认应用目录存在 Oak 私有 OpenGL 后端库，例如 `liboakgl.so`、`liboakgl.dylib` 或 `oakgl.dll`。 @@ -510,7 +521,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：日志显示动态 OpenGL 后端加载成功；viewer、Scope、调色和播放行为与默认 OpenGL 路径一致；退出时执行 destroy/unload 无崩溃。 -### 10.15 动态后端缺失或损坏 +### 10.16 动态后端缺失或损坏 1. 使用开启 `OAK_ENABLE_DYNAMIC_RENDER_BACKEND` 的实验构建。 2. 临时移走或重命名 Oak 私有 OpenGL 后端库。 @@ -519,7 +530,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：应用不能静默崩溃；日志明确说明后端库加载失败；用户能够恢复库文件或切回默认构建继续打开项目。 -### 10.16 Vulkan 动态后端库缺失或不可加载 +### 10.17 Vulkan 动态后端库缺失或不可加载 1. 使用开启 `OAK_ENABLE_DYNAMIC_RENDER_BACKEND` 的实验构建。 2. 在 Preferences 中选择 Vulkan 并重启。 @@ -530,7 +541,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：Vulkan 后端库缺失、损坏或符号不完整时不崩溃；日志明确说明 Vulkan 后端加载失败并回退或拒绝初始化；切回 OpenGL 后项目可播放。 -### 10.17 Vulkan 与 OpenGL 结果记录 +### 10.18 Vulkan 与 OpenGL 结果记录 1. 对同一项目分别在 Vulkan 和 OpenGL 下执行 Viewer 播放、5 秒软件导出、代理启用导出。 2. 记录每个环境的实际 backend、GPU、driver、Vulkan API 版本和是否发生回退。 @@ -539,7 +550,7 @@ Vulkan 测试必须先区分两类环境： 通过标准：每次测试结果能明确区分“真实 Vulkan 后端通过”、“请求 Vulkan 但回退 OpenGL 通过”和“Vulkan 后端失败”；不能把回退 OpenGL 的结果记为 Vulkan 渲染通过。 -### 10.18 回退链路恢复 +### 10.19 回退链路恢复 1. 在可用 Vulkan 环境中选择 Vulkan 并确认实际使用 Vulkan。 2. 退出应用，临时破坏 Vulkan runtime 或移走 `liboakvulkan`。 diff --git a/tests/gtest/CMakeLists.txt b/tests/gtest/CMakeLists.txt index d0d0e4af1..45cdb1dd6 100644 --- a/tests/gtest/CMakeLists.txt +++ b/tests/gtest/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(olive-gtest render_sampleformat_test.cpp render_pixelformat_test.cpp render_ipc_test.cpp + render_worker_footage_test.cpp project_serializer_test.cpp proxy_manager_test.cpp timeline_marker_test.cpp diff --git a/tests/gtest/dynamic_render_backend_test.cpp b/tests/gtest/dynamic_render_backend_test.cpp index ad357c34d..4534cee50 100644 --- a/tests/gtest/dynamic_render_backend_test.cpp +++ b/tests/gtest/dynamic_render_backend_test.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include "node/value.h" #include "render/backend/dynamicrenderer.h" @@ -32,6 +34,49 @@ TEST(DynamicRenderBackend, LoadsExperimentalOpenGLBackend) #endif } +// Regression test: the backend renderer must follow DynamicRenderer when it is +// moved to a background thread. If it stays in the thread where Load() was +// called, GL operations are rejected as "wrong thread" and texture creation +// returns null, which manifests as a black screen. +TEST(DynamicRenderBackend, OpenGLBackendFollowsAdapterToRenderThread) +{ +#ifndef OAK_ENABLE_DYNAMIC_RENDER_BACKEND + GTEST_SKIP() << "Dynamic render backend is not enabled in this build"; +#else + olive::DynamicRenderer renderer(QStringLiteral("opengl")); + ASSERT_TRUE(renderer.Load()); + ASSERT_TRUE(renderer.Init()); + + QThread render_thread; + renderer.moveToThread(&render_thread); + render_thread.start(); + + QOpenGLContext *ctx = renderer.OpenGLContext(); + ASSERT_NE(ctx, nullptr); + EXPECT_EQ(ctx->thread(), &render_thread) + << "Backend OpenGL context did not follow DynamicRenderer to render thread"; + + // Exercise the actual GL path in the render thread: PostInit() creates the + // offscreen surface there, and CreateTexture() must not crash. + olive::TexturePtr texture; + QMetaObject::invokeMethod( + &renderer, + [&]() { + renderer.PostInit(); + texture = renderer.CreateTexture(olive::VideoParams( + 64, 64, olive::PixelFormat::U8, + olive::VideoParams::kRGBAChannelCount)); + }, + Qt::BlockingQueuedConnection); + + render_thread.quit(); + render_thread.wait(); + + ASSERT_NE(texture, nullptr); + EXPECT_FALSE(texture->IsDummy()); +#endif +} + // Verifies Vulkan backend discovery on systems with a working Vulkan ICD. The // test skips when the runtime correctly reports Vulkan as unavailable. TEST(DynamicRenderBackend, LoadsExperimentalVulkanBackendWhenAvailable) @@ -110,7 +155,7 @@ TEST(DynamicRenderBackend, VulkanUploadBlitDownload) src_data[i * 4 + 2] = static_cast(0); // B src_data[i * 4 + 3] = static_cast(255); // A } - src->Upload(src_data.data(), kSize * 4); + src->Upload(src_data.data(), kSize); olive::TexturePtr dst = renderer.CreateTexture(params); ASSERT_NE(dst, nullptr); @@ -145,7 +190,7 @@ TEST(DynamicRenderBackend, VulkanUploadBlitDownload) renderer.BlitToTexture(shader, job, dst.get(), true); QByteArray dst_data(kSize * kSize * 4, 0); - dst->Download(dst_data.data(), kSize * 4); + dst->Download(dst_data.data(), kSize); // The default pass-through shader should reproduce the red source pixel. EXPECT_EQ(static_cast(dst_data[0]), 255u); @@ -186,7 +231,7 @@ TEST(DynamicRenderBackend, VulkanNullDestinationBlitDoesNotCrash) src_data[i * 4 + 0] = static_cast(255); src_data[i * 4 + 3] = static_cast(255); } - src->Upload(src_data.data(), kSize * 4); + src->Upload(src_data.data(), kSize); const QString vert = QStringLiteral( "uniform mat4 ove_mvpmat;\n" @@ -252,7 +297,7 @@ TEST(DynamicRenderBackend, VulkanIterativeBlitPingPong) src_data[i * 4 + 0] = static_cast(255); src_data[i * 4 + 3] = static_cast(255); } - src->Upload(src_data.data(), kSize * 4); + src->Upload(src_data.data(), kSize); olive::TexturePtr dst = renderer.CreateTexture(params); ASSERT_NE(dst, nullptr); @@ -290,7 +335,7 @@ TEST(DynamicRenderBackend, VulkanIterativeBlitPingPong) renderer.BlitToTexture(shader, job, dst.get(), true); QByteArray dst_data(kSize * kSize * 4, 0); - dst->Download(dst_data.data(), kSize * 4); + dst->Download(dst_data.data(), kSize); // After two halving passes, red is 255 * 0.5 * 0.5. UNORM conversion floors // the intermediate value, so the result is 63 rather than 64. @@ -333,10 +378,10 @@ TEST(DynamicRenderBackend, VulkanUploadDownloadThreeChannel) src_data[i * 3 + 1] = static_cast(128); src_data[i * 3 + 2] = static_cast(64); } - tex->Upload(src_data.data(), kSize * 3); + tex->Upload(src_data.data(), kSize); QByteArray dst_data(kSize * kSize * 3, 0); - tex->Download(dst_data.data(), kSize * 3); + tex->Download(dst_data.data(), kSize); EXPECT_EQ(static_cast(dst_data[0]), 255u); EXPECT_EQ(static_cast(dst_data[1]), 128u); diff --git a/tests/gtest/render_worker_footage_test.cpp b/tests/gtest/render_worker_footage_test.cpp new file mode 100644 index 000000000..ba7eb1f87 --- /dev/null +++ b/tests/gtest/render_worker_footage_test.cpp @@ -0,0 +1,440 @@ +/* + * Oak Video Editor - Render Worker Footage Integration Test + * Copyright (C) 2026 Oak Team + * + * End-to-end test that spawns olive-render-worker, feeds it a real decoded + * frame from tests/demo.mp4 through the IPC shared-memory frame pool, and + * verifies that the worker returns a non-black output frame. + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "codec/decoder.h" +#include "codec/frame.h" +#include "common/filefunctions.h" +#include "node/color/colormanager/colormanager.h" +#include "node/project.h" +#include "node/project/footage/footage.h" +#include "node/project/serializer/serializer.h" +#include "render/diskmanager.h" +#include "render/ipc/frameslotpool.h" +#include "render/ipc/ipcmessage.h" +#include "render/ipc/sharedmemoryregion.h" +#include "render/videoparams.h" + +using namespace olive; +using namespace olive::core; + +namespace { + +constexpr int kInputSlots = 1; +constexpr int kOutputSlots = 1; +constexpr int kTimeoutMs = 30000; + +QString WorkerBinaryPath() +{ + // The test binary lives in cmake-build-debug/tests/gtest; the worker is in + // cmake-build-debug/app. + QDir dir(QCoreApplication::applicationDirPath()); + dir.cdUp(); // tests/gtest -> tests + dir.cdUp(); // tests -> build dir + dir.cd(QStringLiteral("app")); + return dir.filePath(QStringLiteral("olive-render-worker")); +} + +QString DemoVideoPath() +{ + return QDir(QStringLiteral(OAK_TEST_SOURCE_DIR)) + .filePath(QStringLiteral("tests/demo.mp4")); +} + +double SampleBrightnessF32(const void *data, int width, int height, int stride) +{ + const auto *base = reinterpret_cast(data); + double avg = 0.0; + int samples = 0; + for (int y = 0; y < height && y < 1080; y += 120) { + for (int x = 0; x < width && x < 1920; x += 240) { + const auto *p = reinterpret_cast( + base + y * stride + x * 4 * sizeof(float)); + for (int c = 0; c < 3; ++c) { + avg += p[c]; + } + samples += 3; + } + } + return samples > 0 ? avg / samples : 0.0; +} + +void SaveFrameAsPng(const void *data, int width, int height, + const QString &path) +{ + QImage img(width, height, QImage::Format_RGBA8888); + const auto *src = reinterpret_cast(data); + for (int y = 0; y < height; ++y) { + uchar *dst = img.scanLine(y); + for (int x = 0; x < width; ++x) { + for (int c = 0; c < 4; ++c) { + float v = src[(y * width + x) * 4 + c]; + if (v < 0.0f) v = 0.0f; + if (v > 1.0f) v = 1.0f; + dst[(x * 4) + c] = static_cast(v * 255.0f); + } + } + } + if (!img.save(path)) { + std::cerr << "Failed to save " << path.toStdString() << std::endl; + } else { + std::cerr << "Saved " << path.toStdString() << std::endl; + } +} + +} // namespace + +class RenderWorkerFootageTest : public ::testing::Test { +protected: + void SetUp() override + { + ColorManager::SetUpDefaultConfig(); + ProjectSerializer::Initialize(); + DiskManager::CreateInstance(); + + demo_path_ = DemoVideoPath(); + ASSERT_TRUE(QFileInfo::exists(demo_path_)) + << "demo.mp4 not found at " << demo_path_.toStdString(); + + worker_path_ = WorkerBinaryPath(); + ASSERT_TRUE(QFileInfo::exists(worker_path_)) + << "worker binary not found at " << worker_path_.toStdString(); + + ASSERT_TRUE(temp_dir_.isValid()); + + // Create a minimal project containing the demo footage. + CreateProjectFile(); + } + + void TearDown() override + { + input_region_.Close(); + output_region_.Close(); + if (worker_.state() != QProcess::NotRunning) { + worker_.terminate(); + worker_.waitForFinished(5000); + if (worker_.state() != QProcess::NotRunning) { + worker_.kill(); + worker_.waitForFinished(5000); + } + } + DiskManager::DestroyInstance(); + ProjectSerializer::Destroy(); + } + + void CreateProjectFile() + { + project_ = std::make_unique(); + project_->Initialize(); + + footage_ = new Footage(demo_path_); + footage_->setParent(project_.get()); + footage_->SetLabel(QStringLiteral("demo")); + ASSERT_TRUE(footage_->IsValid()) + << "Footage failed to probe " << demo_path_.toStdString(); + + footage_id_ = QString::number(reinterpret_cast(footage_)); + + project_file_ = FileFunctions::GetSafeTemporaryFilename( + temp_dir_.filePath(QStringLiteral("worker_graph.ove"))); + + ProjectSerializer::Result r = ProjectSerializer::Save( + ProjectSerializer::SaveData(ProjectSerializer::kProject, project_.get(), + project_file_), + false); + ASSERT_EQ(r.code(), ProjectSerializer::kSuccess) + << "Failed to save project file: " << r.GetDetails().toStdString(); + ASSERT_TRUE(QFileInfo::exists(project_file_)); + } + + bool StartWorker(const QString &backend) + { + // ---- decode a frame so we know the dimensions and slot sizes ---- + DecoderPtr decoder = Decoder::CreateFromID(QStringLiteral("ffmpeg")); + if (!decoder || !decoder->Open(Decoder::CodecStream(demo_path_, 0, nullptr))) { + return false; + } + Decoder::RetrieveVideoParams retrieve; + retrieve.time = rational(0); + retrieve.maximum_format = PixelFormat::U16; + FramePtr frame = decoder->RetrieveVideoFrame(retrieve); + if (!frame || !frame->is_allocated()) { + return false; + } + + input_width_ = frame->width(); + input_height_ = frame->height(); + input_stride_ = frame->linesize_bytes(); + input_bpc_ = VideoParams::GetBytesPerChannel(frame->format()); + input_data_bytes_ = frame->allocated_size(); + decoded_frame_ = frame; + + // Output at 1920x1080 float RGBA, like the real viewer path. + output_width_ = 1920; + output_height_ = 1080; + output_data_bytes_ = size_t(output_width_) * output_height_ * 4 * + VideoParams::GetBytesPerChannel(PixelFormat::F32); + + // ---- create shared memory pools ---- + const qint64 owner_pid = QCoreApplication::applicationPid(); + output_shm_key_ = ipc::SharedMemoryRegion::MakeKey(owner_pid, 0); + input_shm_key_ = ipc::SharedMemoryRegion::MakeKey(owner_pid, 1); + + const size_t output_bytes = ipc::FrameSlotPool::BytesNeeded( + kOutputSlots, output_data_bytes_); + const size_t input_bytes = ipc::FrameSlotPool::BytesNeeded( + kInputSlots, input_data_bytes_); + + if (!output_region_.Open(output_shm_key_, output_bytes, + ipc::SharedMemoryRegion::kCreate)) { + return false; + } + if (!input_region_.Open(input_shm_key_, input_bytes, + ipc::SharedMemoryRegion::kCreate)) { + return false; + } + + output_pool_ = std::make_unique( + ipc::FrameSlotPool::Create(output_region_.data(), kOutputSlots, + output_data_bytes_)); + input_pool_ = std::make_unique( + ipc::FrameSlotPool::Create(input_region_.data(), kInputSlots, + input_data_bytes_)); + + if (!output_pool_->IsValid() || !input_pool_->IsValid()) { + return false; + } + + // ---- spawn worker ---- + worker_.setProcessChannelMode(QProcess::ForwardedErrorChannel); + worker_.start(worker_path_, QStringList{QStringLiteral("--backend"), backend}); + if (!worker_.waitForStarted(kTimeoutMs)) { + return false; + } + + // ---- wait for worker handshake ---- + if (!WaitForMessage(&worker_handshake_)) { + return false; + } + if (worker_handshake_[QStringLiteral("type")].toString() != + QLatin1String(ipc::msgtype::kHandshake)) { + return false; + } + + // ---- respond with our shm keys ---- + ipc::HandshakeMsg response; + response.protocol_version = 1; + response.shm_key = output_shm_key_; + response.input_shm_key = input_shm_key_; + response.input_slots = kInputSlots; + response.output_slots = kOutputSlots; + response.slot_data_bytes = qint64(output_data_bytes_); + response.input_slot_data_bytes = qint64(input_data_bytes_); + if (!ipc::WriteMessage(&worker_, response.ToJson())) { + return false; + } + + // ---- publish the decoded frame to the input pool ---- + uint32_t input_slot = 0; + if (!input_pool_->Acquire(&input_slot)) { + return false; + } + EXPECT_EQ(input_slot, 0u); + std::memcpy(input_pool_->SlotData(input_slot), frame->const_data(), + input_data_bytes_); + ipc::FrameSlotMeta *meta = input_pool_->Meta(input_slot); + meta->id = 0; + meta->time_num = 0; + meta->time_den = 1; + meta->width = input_width_; + meta->height = input_height_; + meta->format = int32_t(frame->format()); + meta->channel_count = frame->channel_count(); + meta->linesize = input_stride_; + meta->data_size = int32_t(input_data_bytes_); + std::strncpy(meta->colorspace, + frame->video_params().colorspace().toUtf8().constData(), + sizeof(meta->colorspace) - 1); + meta->colorspace[sizeof(meta->colorspace) - 1] = '\0'; + input_pool_->Publish(input_slot); + + // ---- load graph ---- + ipc::LoadGraphMsg load; + load.path = project_file_; + if (!ipc::WriteMessage(&worker_, load.ToJson())) { + return false; + } + + // ---- wait for graph_loaded ---- + QJsonObject loaded; + if (!WaitForMessage(&loaded)) { + return false; + } + if (loaded[QStringLiteral("type")].toString() != + QLatin1String("graph_loaded")) { + return false; + } + + return true; + } + + bool RenderFrameAndWait(int *output_slot) + { + ipc::RenderFrameMsg req; + req.ticket_id = 1; + req.node_uuid = footage_id_; + req.time_num = 0; + req.time_den = 1; + req.width = output_width_; + req.height = output_height_; + req.format = int(PixelFormat::F32); + req.channel_count = VideoParams::kRGBAChannelCount; + req.mode = int(RenderMode::kOnline); + req.input_slot = 0; + if (!ipc::WriteMessage(&worker_, req.ToJson())) { + return false; + } + + QJsonObject ready; + if (!WaitForMessage(&ready)) { + return false; + } + if (ready[QStringLiteral("type")].toString() != + QLatin1String(ipc::msgtype::kFrameReady)) { + return false; + } + *output_slot = ready[QStringLiteral("slot")].toInt(); + return true; + } + + bool WaitForMessage(QJsonObject *out) + { + QElapsedTimer timer; + timer.start(); + while (!timer.hasExpired(kTimeoutMs)) { + if (worker_.waitForReadyRead(100)) { + read_buffer_.append(worker_.readAllStandardOutput()); + } + bool ok = true; + if (ipc::ReadMessage(&read_buffer_, out, &ok)) { + return true; + } + if (!ok) { + return false; + } + if (worker_.state() == QProcess::NotRunning) { + return false; + } + } + return false; + } + + QString demo_path_; + QString worker_path_; + QString project_file_; + QString footage_id_; + QString output_shm_key_; + QString input_shm_key_; + QTemporaryDir temp_dir_; + + std::unique_ptr project_; + Footage *footage_ = nullptr; + + QProcess worker_; + QJsonObject worker_handshake_; + QByteArray read_buffer_; + + ipc::SharedMemoryRegion output_region_; + ipc::SharedMemoryRegion input_region_; + std::unique_ptr output_pool_; + std::unique_ptr input_pool_; + + FramePtr decoded_frame_; + + int input_width_ = 0; + int input_height_ = 0; + int input_stride_ = 0; + int input_bpc_ = 0; + size_t input_data_bytes_ = 0; + + int output_width_ = 0; + int output_height_ = 0; + size_t output_data_bytes_ = 0; +}; + +TEST_F(RenderWorkerFootageTest, VulkanFootageIsNotBlack) +{ + ASSERT_TRUE(StartWorker(QStringLiteral("vulkan"))); + + int output_slot = -1; + ASSERT_TRUE(RenderFrameAndWait(&output_slot)); + ASSERT_GE(output_slot, 0); + ASSERT_LT(output_slot, kOutputSlots); + + const void *output_data = output_pool_->SlotData(uint32_t(output_slot)); + const double brightness = SampleBrightnessF32( + output_data, output_width_, output_height_, + output_width_ * 4 * int(sizeof(float))); + + EXPECT_GT(brightness, 0.01) + << "Worker output frame is black (brightness=" << brightness << ")"; + + SaveFrameAsPng(output_data, output_width_, output_height_, + temp_dir_.filePath(QStringLiteral("worker_output_vulkan.png"))); + QFile::remove(QStringLiteral("/tmp/worker_output_vulkan.png")); + QFile::copy(temp_dir_.filePath(QStringLiteral("worker_output_vulkan.png")), + QStringLiteral("/tmp/worker_output_vulkan.png")); + std::cerr << "Vulkan output copied to /tmp/worker_output_vulkan.png" << std::endl; +} + +TEST_F(RenderWorkerFootageTest, OpenGLFootageIsNotBlack) +{ + ASSERT_TRUE(StartWorker(QStringLiteral("opengl"))); + + int output_slot = -1; + ASSERT_TRUE(RenderFrameAndWait(&output_slot)); + ASSERT_GE(output_slot, 0); + ASSERT_LT(output_slot, kOutputSlots); + + const void *output_data = output_pool_->SlotData(uint32_t(output_slot)); + const double brightness = SampleBrightnessF32( + output_data, output_width_, output_height_, + output_width_ * 4 * int(sizeof(float))); + + EXPECT_GT(brightness, 0.01) + << "Worker output frame is black (brightness=" << brightness << ")"; + + SaveFrameAsPng(output_data, output_width_, output_height_, + temp_dir_.filePath(QStringLiteral("worker_output_opengl.png"))); + QFile::remove(QStringLiteral("/tmp/worker_output_opengl.png")); + QFile::copy(temp_dir_.filePath(QStringLiteral("worker_output_opengl.png")), + QStringLiteral("/tmp/worker_output_opengl.png")); + std::cerr << "OpenGL output copied to /tmp/worker_output_opengl.png" << std::endl; +} +