diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index 88de05435..4e8c3988b 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -124,6 +124,29 @@ TexturePtr Decoder::RetrieveVideo(const RetrieveVideoParams &p) return cached_texture_; } +FramePtr Decoder::RetrieveVideoFrame(const RetrieveVideoParams &p) +{ + QMutexLocker locker(&mutex_); + + UpdateLastAccessed(); + + if (!stream_.IsValid()) { + qCritical() << "Can't retrieve video frame on a closed decoder"; + return nullptr; + } + + if (!SupportsVideo()) { + qCritical() << "Decoder doesn't support video"; + return nullptr; + } + + if (p.cancelled && p.cancelled->IsCancelled()) { + return nullptr; + } + + return RetrieveVideoFrameInternal(p); +} + Decoder::RetrieveAudioStatus Decoder::RetrieveAudio(SampleBuffer &dest, const TimeRange &range, const AudioParams ¶ms, const QString &cache_path, @@ -291,6 +314,12 @@ TexturePtr Decoder::RetrieveVideoInternal(const RetrieveVideoParams &p) return nullptr; } +FramePtr Decoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p) +{ + Q_UNUSED(p) + return nullptr; +} + bool Decoder::ConformAudioInternal(const QVector &filenames, const AudioParams ¶ms, CancelAtom *cancelled) diff --git a/app/codec/decoder.h b/app/codec/decoder.h index 0308edd80..ee1614be7 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -32,6 +32,7 @@ extern "C" { #include #include + #include "codec/frame.h" #include "node/block/block.h" #include "node/project/footage/footagedescription.h" #include "render/cancelatom.h" @@ -181,6 +182,14 @@ public: */ TexturePtr RetrieveVideo(const RetrieveVideoParams &p); + /** + * @brief Retrieves a decoded video frame in CPU memory. + * + * Used by render-process isolation to decode media in the main process and pass packed pixel + * data to workers through shared memory. + */ + FramePtr RetrieveVideoFrame(const RetrieveVideoParams &p); + enum RetrieveAudioStatus { kInvalid = -1, kOK, @@ -283,6 +292,8 @@ protected: */ virtual TexturePtr RetrieveVideoInternal(const RetrieveVideoParams &p); + virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p); + virtual bool ConformAudioInternal(const QVector &filenames, const AudioParams ¶ms, CancelAtom *cancelled); diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 40ecdac16..08df26241 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -23,6 +23,34 @@ namespace olive { +static FramePtr CopyPackedAVFrameToFrame(const AVFramePtr &src, + PixelFormat format, + int channel_count, + const rational ×tamp) +{ + if (!src || !src->data[0]) { + return nullptr; + } + + VideoParams params(src->width, src->height, format, channel_count); + FramePtr frame = Frame::Create(); + frame->set_video_params(params); + frame->set_timestamp(timestamp); + if (!frame->allocate()) { + return nullptr; + } + + const int row_bytes = params.effective_width() * + VideoParams::GetBytesPerPixel(format, channel_count); + for (int y = 0; y < frame->height(); y++) { + memcpy(frame->data() + y * frame->linesize_bytes(), + src->data[0] + y * src->linesize[0], + size_t(row_bytes)); + } + + return frame; +} + static VideoParams::Interlacing FFmpegFieldOrderToOlive(AVFieldOrder fo) { switch (fo) { @@ -375,6 +403,74 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(const RetrieveVideoParams &p) return nullptr; } +FramePtr FFmpegDecoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p) +{ + if (AVFramePtr f = RetrieveFrame(p.time, p.cancelled)) { + if (p.cancelled && p.cancelled->IsCancelled()) { + return nullptr; + } + + f->format = FFmpegUtils::ConvertJPEGSpaceToRegularSpace( + static_cast(f->format)); + f->color_range = p.force_range == VideoParams::kColorRangeFull ? + AVCOL_RANGE_JPEG : + AVCOL_RANGE_MPEG; + + AVFramePtr dest = CreateAVFramePtr(); + dest->width = f->width; + dest->height = f->height; + dest->format = p.maximum_format == PixelFormat::U8 + ? AV_PIX_FMT_RGBA + : AV_PIX_FMT_RGBA64; + dest->color_range = f->color_range; + dest->colorspace = f->colorspace; + if (p.divider > 1) { + dest->width = VideoParams::GetScaledDimension(dest->width, p.divider); + dest->height = VideoParams::GetScaledDimension(dest->height, p.divider); + } + + int r = av_frame_get_buffer(dest.get(), 0); + if (r < 0) { + FFmpegError(r); + return nullptr; + } + + SwsContext *cpu_sws = sws_getContext( + f->width, f->height, static_cast(f->format), + dest->width, dest->height, static_cast(dest->format), + SWS_POINT, nullptr, nullptr, nullptr); + if (!cpu_sws) { + qCritical() << "Failed to create CPU frame conversion context"; + return nullptr; + } + + sws_setColorspaceDetails( + cpu_sws, + sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace( + dest->colorspace)), + dest->color_range == AVCOL_RANGE_JPEG ? 1 : 0, + sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace( + dest->colorspace)), + dest->color_range == AVCOL_RANGE_JPEG ? 1 : 0, 0, 0x10000, 0x10000); + + r = sws_scale_frame(cpu_sws, dest.get(), f.get()); + sws_freeContext(cpu_sws); + if (r < 0) { + FFmpegError(r); + return nullptr; + } + + return CopyPackedAVFrameToFrame(dest, + dest->format == AV_PIX_FMT_RGBA + ? PixelFormat::U8 + : PixelFormat::U16, + VideoParams::kRGBAChannelCount, + p.time); + } + + return nullptr; +} + void FFmpegDecoder::CloseInternal() { if (working_packet_) { diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 98d432124..f187fd2d4 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -73,6 +73,7 @@ protected: virtual bool OpenInternal() override; virtual TexturePtr RetrieveVideoInternal(const RetrieveVideoParams &p) override; + virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p) override; virtual bool ConformAudioInternal(const QVector &filenames, const AudioParams ¶ms, CancelAtom *cancelled) override; diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index b60cae8a9..60b2e9c98 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -122,6 +122,17 @@ bool OIIODecoder::OpenInternal() } TexturePtr OIIODecoder::RetrieveVideoInternal(const RetrieveVideoParams &p) +{ + FramePtr frame = RetrieveVideoFrameInternal(p); + if (!frame) { + return nullptr; + } + + return p.renderer->CreateTexture(frame->video_params(), frame->data(), + frame->linesize_pixels()); +} + +FramePtr OIIODecoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p) { VideoParams vp = GetVideoParamsFromImageSpec(image_->spec()); vp.set_divider(p.divider); @@ -163,15 +174,19 @@ TexturePtr OIIODecoder::RetrieveVideoInternal(const RetrieveVideoParams &p) if (vp.format() != PixelFormat::F32) { FramePtr f32_frame = buffer_.convert(PixelFormat::F32); if (f32_frame) { - VideoParams f32_vp = vp; - f32_vp.set_format(PixelFormat::F32); - return p.renderer->CreateTexture(f32_vp, f32_frame->data(), - f32_frame->linesize_pixels()); + f32_frame->set_timestamp(p.time); + return f32_frame; } } - return p.renderer->CreateTexture(vp, buffer_.data(), - buffer_.linesize_pixels()); + FramePtr frame = Frame::Create(); + frame->set_video_params(buffer_.video_params()); + frame->set_timestamp(p.time); + if (!frame->allocate()) { + return nullptr; + } + memcpy(frame->data(), buffer_.const_data(), size_t(buffer_.allocated_size())); + return frame; } void OIIODecoder::CloseInternal() diff --git a/app/codec/oiio/oiiodecoder.h b/app/codec/oiio/oiiodecoder.h index e25172dfc..4029e173c 100644 --- a/app/codec/oiio/oiiodecoder.h +++ b/app/codec/oiio/oiiodecoder.h @@ -51,6 +51,7 @@ protected: virtual bool OpenInternal() override; virtual TexturePtr RetrieveVideoInternal(const RetrieveVideoParams &p) override; + virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p) override; virtual void CloseInternal() override; private: diff --git a/app/render/ipc/ipcmessage.cpp b/app/render/ipc/ipcmessage.cpp index 8b8822b8b..899620da7 100644 --- a/app/render/ipc/ipcmessage.cpp +++ b/app/render/ipc/ipcmessage.cpp @@ -20,6 +20,7 @@ #include "ipcmessage.h" +#include #include #include @@ -78,9 +79,11 @@ QJsonObject HandshakeMsg::ToJson() const o["type"] = msgtype::kHandshake; o["protocol_version"] = protocol_version; o["shm_key"] = shm_key; + o["input_shm_key"] = input_shm_key; o["input_slots"] = input_slots; o["output_slots"] = output_slots; o["slot_data_bytes"] = double(slot_data_bytes); + o["input_slot_data_bytes"] = double(input_slot_data_bytes); return o; } @@ -91,9 +94,11 @@ bool HandshakeMsg::FromJson(const QJsonObject &o, HandshakeMsg *out) } out->protocol_version = o["protocol_version"].toInt(); out->shm_key = o["shm_key"].toString(); + out->input_shm_key = o["input_shm_key"].toString(); out->input_slots = o["input_slots"].toInt(); out->output_slots = o["output_slots"].toInt(); out->slot_data_bytes = qint64(o["slot_data_bytes"].toDouble()); + out->input_slot_data_bytes = qint64(o["input_slot_data_bytes"].toDouble()); return true; } @@ -112,6 +117,12 @@ QJsonObject RenderFrameMsg::ToJson() const o["format"] = format; o["channels"] = channel_count; o["mode"] = mode; + o["input_slot"] = input_slot; + QJsonArray input_slot_array; + for (int slot : input_slots) { + input_slot_array.append(slot); + } + o["input_slots"] = input_slot_array; return o; } @@ -129,6 +140,15 @@ bool RenderFrameMsg::FromJson(const QJsonObject &o, RenderFrameMsg *out) out->format = o["format"].toInt(-1); out->channel_count = o["channels"].toInt(); out->mode = o["mode"].toInt(); + out->input_slot = o["input_slot"].toInt(-1); + out->input_slots.clear(); + const QJsonArray input_slot_array = o["input_slots"].toArray(); + for (const QJsonValue &slot : input_slot_array) { + out->input_slots.append(slot.toInt(-1)); + } + if (out->input_slots.isEmpty() && out->input_slot >= 0) { + out->input_slots.append(out->input_slot); + } return true; } @@ -192,4 +212,4 @@ bool LoadGraphMsg::FromJson(const QJsonObject &o, LoadGraphMsg *out) } } // namespace ipc -} // namespace olive \ No newline at end of file +} // namespace olive diff --git a/app/render/ipc/ipcmessage.h b/app/render/ipc/ipcmessage.h index 56cb2cb83..6a581460f 100644 --- a/app/render/ipc/ipcmessage.h +++ b/app/render/ipc/ipcmessage.h @@ -25,6 +25,7 @@ #include #include #include +#include class QIODevice; @@ -91,10 +92,12 @@ bool ReadMessage(QByteArray *buffer, QJsonObject *out, bool *ok = nullptr); struct HandshakeMsg { int protocol_version = 0; - QString shm_key; ///< Shared-memory segment key for this worker. + QString shm_key; ///< Worker->main output shared-memory segment key. + QString input_shm_key; ///< Main->worker input shared-memory segment key (optional). int input_slots = 0; ///< Number of main->worker input frame slots. int output_slots = 0; ///< Number of worker->main output frame slots. - qint64 slot_data_bytes = 0; ///< Per-slot pixel block size (max frame size). + qint64 slot_data_bytes = 0; ///< Per-output-slot pixel block size. + qint64 input_slot_data_bytes = 0; ///< Per-input-slot pixel block size. QJsonObject ToJson() const; static bool FromJson(const QJsonObject &o, HandshakeMsg *out); @@ -110,6 +113,8 @@ struct RenderFrameMsg { int format = -1; ///< Forced PixelFormat::Format (-1 = default/INVALID). int channel_count = 0; ///< 0 = default. int mode = 0; ///< RenderMode::Mode. + int input_slot = -1; ///< Optional main->worker decoded input slot for footage nodes. + QVector input_slots; ///< Optional ordered decoded input slots for footage nodes. QJsonObject ToJson() const; static bool FromJson(const QJsonObject &o, RenderFrameMsg *out); @@ -140,4 +145,4 @@ struct LoadGraphMsg { } // namespace ipc } // namespace olive -#endif // IPC_IPCMESSAGE_H \ No newline at end of file +#endif // IPC_IPCMESSAGE_H diff --git a/app/render/rendermanager.cpp b/app/render/rendermanager.cpp index 137733846..0471bf245 100644 --- a/app/render/rendermanager.cpp +++ b/app/render/rendermanager.cpp @@ -68,7 +68,7 @@ RenderManager::RenderManager(QObject *parent) auto_cacher_ = new PreviewAutoCacher(this); if (OLIVE_CONFIG("RenderProcessIsolationEnabled").toBool()) { - worker_pool_ = new RenderWorkerPool(this); + worker_pool_ = new RenderWorkerPool(decoder_cache_, this); worker_pool_->start(QThread::NormalPriority); backend_ = kMultiProcess; } diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index 8c0c2694e..a7513616a 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -35,6 +35,7 @@ #include "render/plugin/pluginrenderer.h" #include "pluginSupport/OliveClip.h" #include "pluginSupport/OliveHost.h" +#include "render/ipc/frameslotpool.h" namespace olive { @@ -419,6 +420,81 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination, qWarning() << "HAVEN'T GOTTEN DEFAULT INPUT COLORSPACE"; } + auto blit_color_managed = [&](const TexturePtr &unmanaged_texture, + const VideoParams &texture_params) { + if (!render_ctx_ || !unmanaged_texture || IsCancelled()) { + return; + } + + // We convert to our rendering pixel format, since that will always be float-based which + // is necessary for correct color conversion + ColorProcessorPtr processor = ColorProcessor::Create( + color_manager, using_colorspace, + color_manager->GetReferenceColorSpace()); + + ColorTransformJob job; + job.SetColorProcessor(processor); + job.SetInputTexture(unmanaged_texture); + + if (texture_params.channel_count() != VideoParams::kRGBAChannelCount || + texture_params.colorspace() == color_manager->GetReferenceColorSpace()) { + job.SetInputAlphaAssociation(kAlphaNone); + } else if (texture_params.premultiplied_alpha()) { + job.SetInputAlphaAssociation(kAlphaAssociated); + } else { + job.SetInputAlphaAssociation(kAlphaUnassociated); + } + + render_ctx_->BlitColorManaged(job, destination.get()); + // macOS TBDR: ensure tile writeback completes before the texture + // is read back in a potentially different shared OpenGL context. + render_ctx_->Flush(); + }; + + auto *input_pool = + QtUtils::ValueToPtr(ticket_->property("ipc_input_pool")); + int input_slot = -1; + const QVariantList input_slots = ticket_->property("ipc_input_slots").toList(); + if (!input_slots.isEmpty()) { + const QVariant cursor_value = ticket_->property("ipc_input_slot_cursor"); + const int cursor = cursor_value.isValid() ? cursor_value.toInt() : 0; + if (cursor >= 0 && cursor < input_slots.size()) { + input_slot = input_slots.at(cursor).toInt(); + ticket_->setProperty("ipc_input_slot_cursor", cursor + 1); + } + } else { + const QVariant input_slot_value = ticket_->property("ipc_input_slot"); + input_slot = input_slot_value.isValid() ? input_slot_value.toInt() : -1; + } + if (render_ctx_ && input_pool && input_slot >= 0) { + const ipc::FrameSlotMeta *meta = input_pool->Meta(uint32_t(input_slot)); + if (meta && meta->width > 0 && meta->height > 0 && meta->data_size > 0 && + meta->data_size <= int(input_pool->slot_data_bytes())) { + VideoParams input_params = stream_data; + input_params.set_width(meta->width); + input_params.set_height(meta->height); + input_params.set_format(PixelFormat::Format(meta->format)); + input_params.set_channel_count(meta->channel_count); + + const int bytes_per_pixel = input_params.GetBytesPerPixel(); + const int linesize_pixels = bytes_per_pixel > 0 + ? meta->linesize / bytes_per_pixel + : input_params.effective_width(); + TexturePtr unmanaged_texture = render_ctx_->CreateTexture( + input_params, input_pool->SlotData(uint32_t(input_slot)), linesize_pixels); + blit_color_managed(unmanaged_texture, input_params); + return; + } + qWarning() << "RenderProcessor received invalid IPC input frame slot" << input_slot; + return; + } + + if (!decoder_cache_) { + qWarning() << "RenderProcessor has no decoder cache or IPC input frame for" + << stream->filename(); + return; + } + Decoder::CodecStream default_codec_stream( stream->filename(), stream_data.stream_index(), GetCurrentBlock()); @@ -474,32 +550,7 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination, unmanaged_texture = decoder->RetrieveVideo(p); if (!IsCancelled() && unmanaged_texture) { - // We convert to our rendering pixel format, since that will always be float-based which - // is necessary for correct color conversion - ColorProcessorPtr processor = ColorProcessor::Create( - color_manager, using_colorspace, - color_manager->GetReferenceColorSpace()); - - ColorTransformJob job; - - job.SetColorProcessor(processor); - job.SetInputTexture(unmanaged_texture); - - if (stream_data.channel_count() != - VideoParams::kRGBAChannelCount || - stream_data.colorspace() == - color_manager->GetReferenceColorSpace()) { - job.SetInputAlphaAssociation(kAlphaNone); - } else if (stream_data.premultiplied_alpha()) { - job.SetInputAlphaAssociation(kAlphaAssociated); - } else { - job.SetInputAlphaAssociation(kAlphaUnassociated); - } - - render_ctx_->BlitColorManaged(job, destination.get()); - // macOS TBDR: ensure tile writeback completes before the texture - // is read back in a potentially different shared OpenGL context. - render_ctx_->Flush(); + blit_color_managed(unmanaged_texture, stream_data); } } } diff --git a/app/render/renderworkerpool.cpp b/app/render/renderworkerpool.cpp index 94788dd93..8ee316f17 100644 --- a/app/render/renderworkerpool.cpp +++ b/app/render/renderworkerpool.cpp @@ -23,14 +23,18 @@ #include #include #include +#include #include #include #include #include #include +#include #include "codec/frame.h" #include "common/qtutils.h" +#include "node/project/footage/footage.h" +#include "node/traverser.h" namespace olive { @@ -40,6 +44,167 @@ namespace constexpr int kProtocolVersion = 1; +struct FootageInput { + FootageJob job; + rational time; +}; + +class FootageInputCollector : public NodeTraverser { +public: + QVector Collect(const RenderManager::RenderVideoParams ¶ms, + CancelAtom *cancel) + { + SetCancelPointer(cancel); + VideoParams cache_params = params.video_params; + cache_params.set_format(PixelFormat::F32); + SetCacheVideoParams(cache_params); + SetCacheAudioParams(params.audio_params); + + rational frame_length = cache_params.frame_rate_as_time_base(); + if (cache_params.interlacing() != VideoParams::kInterlaceNone) { + frame_length /= 2; + } + NodeValueTable table = GenerateTable(params.node, + TimeRange(params.time, + params.time + frame_length)); + NodeValue texture = table.Get(NodeValue::kTexture); + ResolveJobs(texture); + + if (cache_params.interlacing() != VideoParams::kInterlaceNone) { + NodeValueTable second_table = + GenerateTable(params.node, + TimeRange(params.time + frame_length, + params.time + frame_length * 2)); + NodeValue second_texture = second_table.Get(NodeValue::kTexture); + ResolveJobs(second_texture); + } + + return inputs_; + } + +protected: + void ProcessVideoFootage(TexturePtr destination, + const FootageJob *stream, + const rational &input_time) override + { + Q_UNUSED(destination) + if (stream) { + inputs_.append({*stream, input_time}); + } + } + +private: + QVector inputs_; +}; + +DecoderPtr ResolveDecoderFromCache(DecoderCache *decoder_cache, + const QString &decoder_id, + const Decoder::CodecStream &stream) +{ + if (!decoder_cache || !stream.IsValid()) { + return nullptr; + } + + QMutexLocker locker(decoder_cache->mutex()); + DecoderPair decoder = decoder_cache->value(stream); + const qint64 file_last_modified = + QFileInfo(stream.filename()).lastModified().toMSecsSinceEpoch(); + + if (decoder.decoder && decoder.last_modified == file_last_modified) { + return decoder.decoder; + } + + decoder.decoder = Decoder::CreateFromID(decoder_id); + decoder.last_modified = file_last_modified; + decoder_cache->insert(stream, decoder); + locker.unlock(); + + if (!decoder.decoder || !decoder.decoder->Open(stream)) { + qWarning() << "RenderWorkerPool failed to open decoder for" + << stream.filename() << "::" << stream.stream(); + return nullptr; + } + + return decoder.decoder; +} + +FramePtr DecodeInputFrame(DecoderCache *decoder_cache, + const FootageInput &input, + CancelAtom *cancel) +{ + VideoParams stream_data = input.job.video_params(); + QString filename = input.job.filename(); + DecoderPtr decoder; + + switch (stream_data.video_type()) { + case VideoParams::kVideoTypeVideo: + case VideoParams::kVideoTypeStill: + decoder = ResolveDecoderFromCache( + decoder_cache, + input.job.decoder(), + Decoder::CodecStream(filename, stream_data.stream_index(), nullptr)); + break; + case VideoParams::kVideoTypeImageSequence: { + const int64_t frame_number = + stream_data.get_time_in_timebase_units(input.time); + filename = Decoder::TransformImageSequenceFileName(filename, frame_number); + decoder = Decoder::CreateFromID(input.job.decoder()); + if (decoder && + !decoder->Open(Decoder::CodecStream(filename, + stream_data.stream_index(), + nullptr))) { + decoder = nullptr; + } + break; + } + } + + if (!decoder) { + return nullptr; + } + + Decoder::RetrieveVideoParams retrieve; + retrieve.divider = stream_data.divider(); + retrieve.maximum_format = PixelFormat::U16; + retrieve.time = stream_data.video_type() == VideoParams::kVideoTypeVideo + ? input.time + : Decoder::kAnyTimecode; + retrieve.cancelled = cancel; + retrieve.force_range = stream_data.color_range(); + retrieve.src_interlacing = stream_data.interlacing(); + FramePtr frame = decoder->RetrieveVideoFrame(retrieve); + if (frame) { + frame->set_timestamp(input.time); + } + return frame; +} + +bool DecodeInputFrames(DecoderCache *decoder_cache, + const RenderManager::RenderVideoParams ¶ms, + CancelAtom *cancel, + QVector *frames) +{ + frames->clear(); + + FootageInputCollector collector; + const QVector inputs = collector.Collect(params, cancel); + frames->reserve(inputs.size()); + for (const FootageInput &input : inputs) { + if (cancel && cancel->IsCancelled()) { + return false; + } + + FramePtr frame = DecodeInputFrame(decoder_cache, input, cancel); + if (!frame || !frame->is_allocated()) { + frames->clear(); + return false; + } + frames->append(frame); + } + + return true; +} + QString WorkerProgramPath() { const QString dir = QCoreApplication::applicationDirPath(); @@ -100,8 +265,10 @@ bool ReadControlMessage(QProcess *process, QJsonObject *out, QString *error, } // namespace -RenderWorkerPool::RenderWorkerPool(QObject *parent) +RenderWorkerPool::RenderWorkerPool(DecoderCache *decoder_cache, + QObject *parent) : QThread(parent) + , decoder_cache_(decoder_cache) { } @@ -174,6 +341,14 @@ bool RenderWorkerPool::PrepareJob(RenderTicketPtr ticket, return false; } + QVector input_frames; + if (!DecodeInputFrames(decoder_cache_, params, ticket->GetCancelAtom(), + &input_frames)) { + qWarning() << "RenderWorkerPool could not predecode footage inputs;" + << "falling back to in-process render"; + return false; + } + QString graph_path; if (!WriteGraphSnapshot(project, &graph_path)) { return false; @@ -183,6 +358,7 @@ bool RenderWorkerPool::PrepareJob(RenderTicketPtr ticket, job->params = params; job->graph_path = graph_path; job->node_token = QString::number(reinterpret_cast(params.node)); + job->input_frames = input_frames; return true; } @@ -244,6 +420,72 @@ void RenderWorkerPool::ProcessJob(const Job &job) ipc::FrameSlotPool output_pool = ipc::FrameSlotPool::Create(region.data(), kOutputSlots, slot_bytes); + const QString input_shm_key = + job.input_frames.isEmpty() + ? QString() + : ipc::SharedMemoryRegion::MakeKey( + QCoreApplication::applicationPid(), + int((reinterpret_cast(job.ticket.get()) + 1) & 0xFFFF)); + ipc::SharedMemoryRegion input_region; + std::optional input_pool; + QVector input_slots; + if (!job.input_frames.isEmpty()) { + const uint32_t input_slot_count = uint32_t(job.input_frames.size()); + const size_t input_region_bytes = + ipc::FrameSlotPool::BytesNeeded(input_slot_count, slot_bytes); + if (!input_region.Open(input_shm_key, input_region_bytes, + ipc::SharedMemoryRegion::kCreate)) { + qWarning() << "RenderWorkerPool failed to create input shared memory" + << input_region.error(); + job.ticket->Finish(); + return; + } else { + input_pool = ipc::FrameSlotPool::Create(input_region.data(), + input_slot_count, + slot_bytes); + for (const FramePtr &frame : job.input_frames) { + if (frame->allocated_size() > int(slot_bytes)) { + qWarning() << "RenderWorkerPool decoded input frame exceeds slot size"; + job.ticket->Finish(); + return; + } + + uint32_t slot = 0; + if (!input_pool->Acquire(&slot)) { + qWarning() << "RenderWorkerPool input pool had no free slot"; + job.ticket->Finish(); + return; + } + + memcpy(input_pool->SlotData(slot), frame->const_data(), + size_t(frame->allocated_size())); + ipc::FrameSlotMeta *meta = input_pool->Meta(slot); + meta->id = qint64(input_slots.size()); + meta->time_num = frame->timestamp().numerator(); + meta->time_den = frame->timestamp().denominator(); + meta->width = frame->width(); + meta->height = frame->height(); + meta->format = int32_t(frame->format()); + meta->channel_count = frame->channel_count(); + meta->linesize = frame->linesize_bytes(); + meta->data_size = frame->allocated_size(); + if (!input_pool->Publish(slot)) { + qWarning() << "RenderWorkerPool failed to publish input slot"; + job.ticket->Finish(); + return; + } + input_slots.append(int(slot)); + } + + if (input_slots.size() != job.input_frames.size()) { + qWarning() << "RenderWorkerPool failed to publish all input frames;" + << "aborting worker render"; + job.ticket->Finish(); + return; + } + } + } + QProcess worker; worker.setProgram(WorkerProgramPath()); worker.start(); @@ -268,9 +510,11 @@ void RenderWorkerPool::ProcessJob(const Job &job) ipc::HandshakeMsg handshake; handshake.protocol_version = kProtocolVersion; handshake.shm_key = shm_key; - handshake.input_slots = 0; + handshake.input_shm_key = input_slots.isEmpty() ? QString() : input_shm_key; + handshake.input_slots = input_slots.size(); handshake.output_slots = int(kOutputSlots); handshake.slot_data_bytes = qint64(slot_bytes); + handshake.input_slot_data_bytes = input_slots.isEmpty() ? 0 : qint64(slot_bytes); if (!WriteControlMessage(&worker, handshake.ToJson())) { qWarning() << "RenderWorkerPool failed to send shared-memory handshake"; worker.kill(); @@ -301,6 +545,8 @@ void RenderWorkerPool::ProcessJob(const Job &job) render.format = int(job.params.force_format); render.channel_count = job.params.force_channel_count; render.mode = int(job.params.mode); + render.input_slot = input_slots.isEmpty() ? -1 : input_slots.front(); + render.input_slots = input_slots; if (!WriteControlMessage(&worker, render.ToJson())) { qWarning() << "RenderWorkerPool failed to send render_frame"; diff --git a/app/render/renderworkerpool.h b/app/render/renderworkerpool.h index 6e1d864b5..b19ca1f5c 100644 --- a/app/render/renderworkerpool.h +++ b/app/render/renderworkerpool.h @@ -23,9 +23,11 @@ #include #include +#include #include #include +#include "codec/frame.h" #include "node/project/serializer/serializer.h" #include "render/ipc/frameslotpool.h" #include "render/ipc/ipcmessage.h" @@ -38,7 +40,8 @@ namespace olive class RenderWorkerPool : public QThread { Q_OBJECT public: - explicit RenderWorkerPool(QObject *parent = nullptr); + explicit RenderWorkerPool(DecoderCache *decoder_cache, + QObject *parent = nullptr); ~RenderWorkerPool() override; bool SubmitFrame(RenderTicketPtr ticket, @@ -61,6 +64,7 @@ private: RenderManager::RenderVideoParams params; QString graph_path; QString node_token; + QVector input_frames; }; bool PrepareJob(RenderTicketPtr ticket, @@ -74,6 +78,7 @@ private: uint32_t slot); void CleanupGraphFile(const QString &path); + DecoderCache *decoder_cache_; QMutex mutex_; QWaitCondition wait_; std::deque queue_; diff --git a/app/render/worker/workermain.cpp b/app/render/worker/workermain.cpp index 66039abaf..494c10543 100644 --- a/app/render/worker/workermain.cpp +++ b/app/render/worker/workermain.cpp @@ -107,9 +107,11 @@ public: olive::ipc::HandshakeMsg hs; hs.protocol_version = kProtocolVersion; hs.shm_key = QString(); + hs.input_shm_key = QString(); hs.input_slots = 0; hs.output_slots = 0; hs.slot_data_bytes = 0; + hs.input_slot_data_bytes = 0; QJsonObject handshake = hs.ToJson(); if (QOpenGLContext *ctx = renderer_->context()) { @@ -200,6 +202,29 @@ private: return Write(ErrorMessage(QStringLiteral("shared memory does not contain a frame slot pool"))); } + input_pool_.reset(); + input_region_.Close(); + if (hs.input_slots > 0) { + if (hs.input_shm_key.isEmpty() || hs.input_slot_data_bytes <= 0) { + return Write(ErrorMessage(QStringLiteral("handshake missing input shared-memory geometry"))); + } + + const size_t input_bytes = olive::ipc::FrameSlotPool::BytesNeeded( + uint32_t(hs.input_slots), size_t(hs.input_slot_data_bytes)); + if (!input_region_.Open(hs.input_shm_key, input_bytes, + olive::ipc::SharedMemoryRegion::kAttach)) { + return Write(ErrorMessage(QStringLiteral("failed to attach input shared memory: %1") + .arg(input_region_.error()))); + } + + input_pool_ = olive::ipc::FrameSlotPool::Attach(input_region_.data()); + if (!input_pool_->IsValid()) { + input_region_.Close(); + input_pool_.reset(); + return Write(ErrorMessage(QStringLiteral("input shared memory does not contain a frame slot pool"))); + } + } + return true; } @@ -265,6 +290,38 @@ private: message.ticket_id)); } + QVector input_slots; + const QVector requested_input_slots = + message.input_slots.isEmpty() && message.input_slot >= 0 + ? QVector{message.input_slot} + : message.input_slots; + if (!requested_input_slots.isEmpty()) { + if (!input_pool_ || !input_pool_->IsValid()) { + return Write(ErrorMessage(QStringLiteral("render_frame referenced input slot without input pool"), + message.ticket_id)); + } + + for (int requested_slot : requested_input_slots) { + uint32_t consumed_slot = 0; + if (!input_pool_->Consume(&consumed_slot)) { + for (int slot : input_slots) { + input_pool_->Release(uint32_t(slot)); + } + return Write(ErrorMessage(QStringLiteral("input slot was not ready"), + message.ticket_id)); + } + if (int(consumed_slot) != requested_slot) { + input_pool_->Release(consumed_slot); + for (int slot : input_slots) { + input_pool_->Release(uint32_t(slot)); + } + return Write(ErrorMessage(QStringLiteral("input slot order mismatch"), + message.ticket_id)); + } + input_slots.append(int(consumed_slot)); + } + } + olive::VideoParams vparams(message.width > 0 ? message.width : kDefaultWidth, message.height > 0 ? message.height : kDefaultHeight, olive::rational(1, kDefaultFrameRate), @@ -298,9 +355,24 @@ private: ticket->setProperty("cachetimebase", QVariant::fromValue(olive::rational(1))); ticket->setProperty("cacheid", QVariant::fromValue(QUuid())); ticket->setProperty("multicam", olive::QtUtils::PtrToValue(static_cast(nullptr))); + ticket->setProperty("ipc_input_pool", + olive::QtUtils::PtrToValue( + input_pool_ ? static_cast(&*input_pool_) + : static_cast(nullptr))); + QVariantList input_slot_values; + for (int slot : input_slots) { + input_slot_values.append(slot); + } + ticket->setProperty("ipc_input_slots", input_slot_values); + ticket->setProperty("ipc_input_slot_cursor", 0); + ticket->setProperty("ipc_input_slot", + input_slots.isEmpty() ? -1 : input_slots.front()); ticket->Start(); olive::RenderProcessor::Process(ticket, renderer_, nullptr, &shader_cache_); + for (int slot : input_slots) { + input_pool_->Release(uint32_t(slot)); + } if (!ticket->HasResult()) { return Write(ErrorMessage(QStringLiteral("render produced no frame"), message.ticket_id)); } @@ -353,6 +425,8 @@ private: QHash node_by_token_; olive::ipc::SharedMemoryRegion output_region_; std::optional output_pool_; + olive::ipc::SharedMemoryRegion input_region_; + std::optional input_pool_; olive::ShaderCache shader_cache_; }; diff --git a/docs/zh/render-process-isolation-plan.md b/docs/zh/render-process-isolation-plan.md index df5bef0a6..2ce58c9ad 100644 --- a/docs/zh/render-process-isolation-plan.md +++ b/docs/zh/render-process-isolation-plan.md @@ -203,10 +203,11 @@ compact `QJsonObject`,`\n` 结尾。仅承载低频控制流量(握手、提 ### 阶段 4:素材输入解耦(关键重构) -- `RenderProcessor::ProcessVideoFootage()`(`renderprocessor.cpp:397`)当前经 `ResolveDecoderFromInput` + `DecoderCache` 解码。worker 不链接 FFmpeg,需改为从输入 slot 取已解码帧上传纹理。 -- 主进程侧 `RenderWorkerPool` 派发前用 `DecoderCache` 解出所需原始帧写入输入 slot,索引随 `render_frame` 一起发。 -- 先支持单素材片段,再扩展到多层/转场。 -- 验证:渲染含真实素材的时间线帧,与单进程结果逐像素一致。 +- ✅ `Decoder` 增加 CPU 帧接口 `RetrieveVideoFrame()`;FFmpeg 路径输出 packed RGBA CPU frame,OIIO 路径返回 still frame CPU buffer。 +- ✅ `RenderWorkerPool` 派发前 dry-run 遍历当前帧素材输入,使用主进程 `DecoderCache` 预解码,成功后写入 main→worker 输入 `FrameSlotPool`。 +- ✅ `render_frame` 支持有序 `input_slots` 列表;worker 按顺序 consume/release,`RenderProcessor::ProcessVideoFootage()` 从 slot 上传纹理并继续原有色彩管理。 +- ✅ 没有输入 slot 且 worker 无 `DecoderCache` 时,素材节点安全跳过,不再空指针崩溃。 +- 待补:真实素材项目端到端像素一致性验证;复杂多层/转场/重复素材场景下输入 slot 顺序回归;CPU 预解码失败时的更细粒度回退策略。 ### 阶段 5:多 worker、取消、健壮性 @@ -246,7 +247,9 @@ compact `QJsonObject`,`\n` 结尾。仅承载低频控制流量(握手、提 | `app/CMakeLists.txt`(新增 `olive-render-worker` target) | 1 | ✅ | | `app/node/project/serializer/serializer*.{h,cpp}`(暴露加载映射供 worker 查节点) | 2 | ✅ | | `app/render/rendermanager.{h,cpp}`(`kMultiProcess` 分支 + WorkerPool 接线) | 3 | ✅ 单 worker MVP | -| `app/render/renderprocessor.cpp`(`ProcessVideoFootage` 改取输入 slot) | 4 | 待办 | +| `app/codec/decoder.{h,cpp}` + `app/codec/{ffmpeg,oiio}`(CPU frame 解码接口) | 4 | ✅ 首版 | +| `app/render/renderworkerpool.{h,cpp}`(主进程预解码并填 input slot) | 4 | ✅ 首版 | +| `app/render/renderprocessor.cpp`(`ProcessVideoFootage` 改取输入 slot) | 4 | ✅ 首版 | | `app/config/config.cpp`(多进程开关默认值) | 3 | ✅ 默认关闭 | --- diff --git a/tests/demo.mp4 b/tests/demo.mp4 new file mode 100644 index 000000000..894ccbffd Binary files /dev/null and b/tests/demo.mp4 differ diff --git a/tests/gtest/CMakeLists.txt b/tests/gtest/CMakeLists.txt index 00fa941b8..864795c56 100644 --- a/tests/gtest/CMakeLists.txt +++ b/tests/gtest/CMakeLists.txt @@ -30,6 +30,7 @@ add_executable(olive-gtest plugin_renderer_readback_test.cpp plugin_ofx_integration_test.cpp codec_frame_test.cpp + codec_decoder_test.cpp codec_exportcodec_test.cpp codec_exportformat_test.cpp codec_encoder_test.cpp diff --git a/tests/gtest/codec_decoder_test.cpp b/tests/gtest/codec_decoder_test.cpp new file mode 100644 index 000000000..c484a377c --- /dev/null +++ b/tests/gtest/codec_decoder_test.cpp @@ -0,0 +1,40 @@ +#include + +#include + +#include "codec/decoder.h" + +TEST(CodecDecoder, RetrieveVideoFrameFromDemoMp4) +{ + const QString path = QStringLiteral("tests/demo.mp4"); + ASSERT_TRUE(QFileInfo::exists(path)); + + olive::DecoderPtr decoder = olive::Decoder::CreateFromID(QStringLiteral("ffmpeg")); + ASSERT_TRUE(decoder); + + ASSERT_TRUE(decoder->Open(olive::Decoder::CodecStream(path, 0, nullptr))); + + olive::Decoder::RetrieveVideoParams params; + params.time = olive::rational(0); + params.maximum_format = olive::core::PixelFormat::U16; + + olive::FramePtr frame = decoder->RetrieveVideoFrame(params); + ASSERT_TRUE(frame); + ASSERT_TRUE(frame->is_allocated()); + EXPECT_EQ(frame->width(), 1920); + EXPECT_EQ(frame->height(), 1080); + EXPECT_EQ(frame->format(), olive::core::PixelFormat::U16); + EXPECT_EQ(frame->channel_count(), 4); + EXPECT_GT(frame->allocated_size(), 0); + EXPECT_GT(frame->linesize_bytes(), 0); + + bool has_nonzero_byte = false; + const char *data = frame->const_data(); + for (int i = 0; i < frame->allocated_size(); i++) { + if (data[i] != 0) { + has_nonzero_byte = true; + break; + } + } + EXPECT_TRUE(has_nonzero_byte); +} diff --git a/tests/gtest/render_ipc_test.cpp b/tests/gtest/render_ipc_test.cpp index 75a36ba14..469b9cfc9 100644 --- a/tests/gtest/render_ipc_test.cpp +++ b/tests/gtest/render_ipc_test.cpp @@ -274,9 +274,11 @@ TEST(IpcMessage, TypedRoundTrip) HandshakeMsg hs; hs.protocol_version = 1; hs.shm_key = QStringLiteral("olive-rw-1234-0"); + hs.input_shm_key = QStringLiteral("olive-in-1234-0"); hs.input_slots = 4; hs.output_slots = 6; hs.slot_data_bytes = 256ll * 1024 * 1024; + hs.input_slot_data_bytes = 128ll * 1024 * 1024; ASSERT_TRUE(WriteMessage(&dev, hs.ToJson())); RenderFrameMsg rf; @@ -289,6 +291,8 @@ TEST(IpcMessage, TypedRoundTrip) rf.format = 3; rf.channel_count = 4; rf.mode = 1; + rf.input_slot = 2; + rf.input_slots = {2, 3}; ASSERT_TRUE(WriteMessage(&dev, rf.ToJson())); FrameReadyMsg fr; @@ -308,9 +312,11 @@ TEST(IpcMessage, TypedRoundTrip) ASSERT_TRUE(HandshakeMsg::FromJson(obj, &hs2)); EXPECT_EQ(hs2.protocol_version, 1); EXPECT_EQ(hs2.shm_key, hs.shm_key); + EXPECT_EQ(hs2.input_shm_key, hs.input_shm_key); EXPECT_EQ(hs2.input_slots, 4); EXPECT_EQ(hs2.output_slots, 6); EXPECT_EQ(hs2.slot_data_bytes, hs.slot_data_bytes); + EXPECT_EQ(hs2.input_slot_data_bytes, hs.input_slot_data_bytes); ASSERT_TRUE(ReadMessage(&reader, &obj, &ok)); ASSERT_TRUE(ok); @@ -322,6 +328,10 @@ TEST(IpcMessage, TypedRoundTrip) EXPECT_EQ(rf2.time_den, 30000); EXPECT_EQ(rf2.width, 1920); EXPECT_EQ(rf2.format, 3); + EXPECT_EQ(rf2.input_slot, 2); + ASSERT_EQ(rf2.input_slots.size(), 2); + EXPECT_EQ(rf2.input_slots[0], 2); + EXPECT_EQ(rf2.input_slots[1], 3); ASSERT_TRUE(ReadMessage(&reader, &obj, &ok)); ASSERT_TRUE(ok); @@ -378,4 +388,4 @@ TEST(IpcMessage, WrongTypeRejected) RenderFrameMsg rf; EXPECT_FALSE(RenderFrameMsg::FromJson(obj, &rf)); -} \ No newline at end of file +}