Add decoded input slots for render workers

app/codec/decoder.cpp

@@ -124,6 +124,29 @@ TexturePtr Decoder::RetrieveVideo(const RetrieveVideoParams &p)
 	return cached_texture_;
 }
 
+FramePtr Decoder::RetrieveVideoFrame(const RetrieveVideoParams &p)
+{
+	QMutexLocker locker(&mutex_);
+
+	UpdateLastAccessed();
+
+	if (!stream_.IsValid()) {
+		qCritical() << "Can't retrieve video frame on a closed decoder";
+		return nullptr;
+	}
+
+	if (!SupportsVideo()) {
+		qCritical() << "Decoder doesn't support video";
+		return nullptr;
+	}
+
+	if (p.cancelled && p.cancelled->IsCancelled()) {
+		return nullptr;
+	}
+
+	return RetrieveVideoFrameInternal(p);
+}
+
 Decoder::RetrieveAudioStatus
 Decoder::RetrieveAudio(SampleBuffer &dest, const TimeRange &range,
 					   const AudioParams &params, const QString &cache_path,
@@ -291,6 +314,12 @@ TexturePtr Decoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
 	return nullptr;
 }
 
+FramePtr Decoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p)
+{
+	Q_UNUSED(p)
+	return nullptr;
+}
+
 bool Decoder::ConformAudioInternal(const QVector<QString> &filenames,
 								   const AudioParams &params,
 								   CancelAtom *cancelled)

app/codec/decoder.h

@@ -32,6 +32,7 @@ extern "C" {
 #include <QWaitCondition>
 #include <stdint.h>
 
+ #include "codec/frame.h"
 #include "node/block/block.h"
 #include "node/project/footage/footagedescription.h"
 #include "render/cancelatom.h"
@@ -181,6 +182,14 @@ public:
    */
 	TexturePtr RetrieveVideo(const RetrieveVideoParams &p);
 
+	/**
+   * @brief Retrieves a decoded video frame in CPU memory.
+   *
+   * Used by render-process isolation to decode media in the main process and pass packed pixel
+   * data to workers through shared memory.
+   */
+	FramePtr RetrieveVideoFrame(const RetrieveVideoParams &p);
+
 	enum RetrieveAudioStatus {
 		kInvalid = -1,
 		kOK,
@@ -283,6 +292,8 @@ protected:
    */
 	virtual TexturePtr RetrieveVideoInternal(const RetrieveVideoParams &p);
 
+	virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p);
+
 	virtual bool ConformAudioInternal(const QVector<QString> &filenames,
 									  const AudioParams &params,
 									  CancelAtom *cancelled);

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -23,6 +23,34 @@
 
 namespace olive {
 
+static FramePtr CopyPackedAVFrameToFrame(const AVFramePtr &src,
+										 PixelFormat format,
+										 int channel_count,
+										 const rational &timestamp)
+{
+	if (!src || !src->data[0]) {
+		return nullptr;
+	}
+
+	VideoParams params(src->width, src->height, format, channel_count);
+	FramePtr frame = Frame::Create();
+	frame->set_video_params(params);
+	frame->set_timestamp(timestamp);
+	if (!frame->allocate()) {
+		return nullptr;
+	}
+
+	const int row_bytes = params.effective_width() *
+						  VideoParams::GetBytesPerPixel(format, channel_count);
+	for (int y = 0; y < frame->height(); y++) {
+		memcpy(frame->data() + y * frame->linesize_bytes(),
+			   src->data[0] + y * src->linesize[0],
+			   size_t(row_bytes));
+	}
+
+	return frame;
+}
+
 static VideoParams::Interlacing FFmpegFieldOrderToOlive(AVFieldOrder fo)
 {
 	switch (fo) {
@@ -375,6 +403,74 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
 	return nullptr;
 }
 
+FramePtr FFmpegDecoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p)
+{
+	if (AVFramePtr f = RetrieveFrame(p.time, p.cancelled)) {
+		if (p.cancelled && p.cancelled->IsCancelled()) {
+			return nullptr;
+		}
+
+		f->format = FFmpegUtils::ConvertJPEGSpaceToRegularSpace(
+			static_cast<AVPixelFormat>(f->format));
+		f->color_range = p.force_range == VideoParams::kColorRangeFull ?
+							 AVCOL_RANGE_JPEG :
+							 AVCOL_RANGE_MPEG;
+
+		AVFramePtr dest = CreateAVFramePtr();
+		dest->width = f->width;
+		dest->height = f->height;
+		dest->format = p.maximum_format == PixelFormat::U8
+						   ? AV_PIX_FMT_RGBA
+						   : AV_PIX_FMT_RGBA64;
+		dest->color_range = f->color_range;
+		dest->colorspace = f->colorspace;
+		if (p.divider > 1) {
+			dest->width = VideoParams::GetScaledDimension(dest->width, p.divider);
+			dest->height = VideoParams::GetScaledDimension(dest->height, p.divider);
+		}
+
+		int r = av_frame_get_buffer(dest.get(), 0);
+		if (r < 0) {
+			FFmpegError(r);
+			return nullptr;
+		}
+
+		SwsContext *cpu_sws = sws_getContext(
+			f->width, f->height, static_cast<AVPixelFormat>(f->format),
+			dest->width, dest->height, static_cast<AVPixelFormat>(dest->format),
+			SWS_POINT, nullptr, nullptr, nullptr);
+		if (!cpu_sws) {
+			qCritical() << "Failed to create CPU frame conversion context";
+			return nullptr;
+		}
+
+		sws_setColorspaceDetails(
+			cpu_sws,
+			sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace(
+				dest->colorspace)),
+			dest->color_range == AVCOL_RANGE_JPEG ? 1 : 0,
+			sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace(
+				dest->colorspace)),
+			dest->color_range == AVCOL_RANGE_JPEG ? 1 : 0, 0, 0x10000, 0x10000);
+
+		r = sws_scale_frame(cpu_sws, dest.get(), f.get());
+		sws_freeContext(cpu_sws);
+		if (r < 0) {
+			FFmpegError(r);
+			return nullptr;
+		}
+
+		return CopyPackedAVFrameToFrame(dest,
+										dest->format == AV_PIX_FMT_RGBA
+											? PixelFormat::U8
+											: PixelFormat::U16,
+										VideoParams::kRGBAChannelCount,
+										p.time);
+	}
+
+	return nullptr;
+}
+
 void FFmpegDecoder::CloseInternal()
 {
 	if (working_packet_) {

app/codec/ffmpeg/ffmpegdecoder.h

@@ -73,6 +73,7 @@ protected:
 	virtual bool OpenInternal() override;
 	virtual TexturePtr
 	RetrieveVideoInternal(const RetrieveVideoParams &p) override;
+	virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p) override;
 	virtual bool ConformAudioInternal(const QVector<QString> &filenames,
 									  const AudioParams &params,
 									  CancelAtom *cancelled) override;

app/codec/oiio/oiiodecoder.cpp

@@ -122,6 +122,17 @@ bool OIIODecoder::OpenInternal()
 }
 
 TexturePtr OIIODecoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
+{
+	FramePtr frame = RetrieveVideoFrameInternal(p);
+	if (!frame) {
+		return nullptr;
+	}
+
+	return p.renderer->CreateTexture(frame->video_params(), frame->data(),
+									 frame->linesize_pixels());
+}
+
+FramePtr OIIODecoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p)
 {
 	VideoParams vp = GetVideoParamsFromImageSpec(image_->spec());
 	vp.set_divider(p.divider);
@@ -163,15 +174,19 @@ TexturePtr OIIODecoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
 	if (vp.format() != PixelFormat::F32) {
 		FramePtr f32_frame = buffer_.convert(PixelFormat::F32);
 		if (f32_frame) {
-			VideoParams f32_vp = vp;
+			f32_frame->set_timestamp(p.time);
-			f32_vp.set_format(PixelFormat::F32);
+			return f32_frame;
-			return p.renderer->CreateTexture(f32_vp, f32_frame->data(),
-											 f32_frame->linesize_pixels());
 		}
 	}
 
-	return p.renderer->CreateTexture(vp, buffer_.data(),
+	FramePtr frame = Frame::Create();
-									 buffer_.linesize_pixels());
+	frame->set_video_params(buffer_.video_params());
+	frame->set_timestamp(p.time);
+	if (!frame->allocate()) {
+		return nullptr;
+	}
+	memcpy(frame->data(), buffer_.const_data(), size_t(buffer_.allocated_size()));
+	return frame;
 }
 
 void OIIODecoder::CloseInternal()

app/codec/oiio/oiiodecoder.h

@@ -51,6 +51,7 @@ protected:
 	virtual bool OpenInternal() override;
 	virtual TexturePtr
 	RetrieveVideoInternal(const RetrieveVideoParams &p) override;
+	virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p) override;
 	virtual void CloseInternal() override;
 
 private:

app/render/ipc/ipcmessage.cpp

@@ -20,6 +20,7 @@
 
 #include "ipcmessage.h"
 
+#include <QJsonArray>
 #include <QJsonDocument>
 #include <QIODevice>
 
@@ -78,9 +79,11 @@ QJsonObject HandshakeMsg::ToJson() const
 	o["type"] = msgtype::kHandshake;
 	o["protocol_version"] = protocol_version;
 	o["shm_key"] = shm_key;
+	o["input_shm_key"] = input_shm_key;
 	o["input_slots"] = input_slots;
 	o["output_slots"] = output_slots;
 	o["slot_data_bytes"] = double(slot_data_bytes);
+	o["input_slot_data_bytes"] = double(input_slot_data_bytes);
 	return o;
 }
 
@@ -91,9 +94,11 @@ bool HandshakeMsg::FromJson(const QJsonObject &o, HandshakeMsg *out)
 	}
 	out->protocol_version = o["protocol_version"].toInt();
 	out->shm_key = o["shm_key"].toString();
+	out->input_shm_key = o["input_shm_key"].toString();
 	out->input_slots = o["input_slots"].toInt();
 	out->output_slots = o["output_slots"].toInt();
 	out->slot_data_bytes = qint64(o["slot_data_bytes"].toDouble());
+	out->input_slot_data_bytes = qint64(o["input_slot_data_bytes"].toDouble());
 	return true;
 }
 
@@ -112,6 +117,12 @@ QJsonObject RenderFrameMsg::ToJson() const
 	o["format"] = format;
 	o["channels"] = channel_count;
 	o["mode"] = mode;
+	o["input_slot"] = input_slot;
+	QJsonArray input_slot_array;
+	for (int slot : input_slots) {
+		input_slot_array.append(slot);
+	}
+	o["input_slots"] = input_slot_array;
 	return o;
 }
 
@@ -129,6 +140,15 @@ bool RenderFrameMsg::FromJson(const QJsonObject &o, RenderFrameMsg *out)
 	out->format = o["format"].toInt(-1);
 	out->channel_count = o["channels"].toInt();
 	out->mode = o["mode"].toInt();
+	out->input_slot = o["input_slot"].toInt(-1);
+	out->input_slots.clear();
+	const QJsonArray input_slot_array = o["input_slots"].toArray();
+	for (const QJsonValue &slot : input_slot_array) {
+		out->input_slots.append(slot.toInt(-1));
+	}
+	if (out->input_slots.isEmpty() && out->input_slot >= 0) {
+		out->input_slots.append(out->input_slot);
+	}
 	return true;
 }
 

app/render/ipc/ipcmessage.h

@@ -25,6 +25,7 @@
 #include <QByteArray>
 #include <QJsonObject>
 #include <QString>
+#include <QVector>
 
 class QIODevice;
 
@@ -91,10 +92,12 @@ bool ReadMessage(QByteArray *buffer, QJsonObject *out, bool *ok = nullptr);
 
 struct HandshakeMsg {
 	int protocol_version = 0;
-	QString shm_key;          ///< Shared-memory segment key for this worker.
+	QString shm_key;          ///< Worker->main output shared-memory segment key.
+	QString input_shm_key;    ///< Main->worker input shared-memory segment key (optional).
 	int input_slots = 0;      ///< Number of main->worker input frame slots.
 	int output_slots = 0;     ///< Number of worker->main output frame slots.
-	qint64 slot_data_bytes = 0;  ///< Per-slot pixel block size (max frame size).
+	qint64 slot_data_bytes = 0;        ///< Per-output-slot pixel block size.
+	qint64 input_slot_data_bytes = 0;  ///< Per-input-slot pixel block size.
 
 	QJsonObject ToJson() const;
 	static bool FromJson(const QJsonObject &o, HandshakeMsg *out);
@@ -110,6 +113,8 @@ struct RenderFrameMsg {
 	int format = -1;          ///< Forced PixelFormat::Format (-1 = default/INVALID).
 	int channel_count = 0;    ///< 0 = default.
 	int mode = 0;             ///< RenderMode::Mode.
+	int input_slot = -1;      ///< Optional main->worker decoded input slot for footage nodes.
+	QVector<int> input_slots; ///< Optional ordered decoded input slots for footage nodes.
 
 	QJsonObject ToJson() const;
 	static bool FromJson(const QJsonObject &o, RenderFrameMsg *out);

app/render/rendermanager.cpp

@@ -68,7 +68,7 @@ RenderManager::RenderManager(QObject *parent)
 		auto_cacher_ = new PreviewAutoCacher(this);
 
 		if (OLIVE_CONFIG("RenderProcessIsolationEnabled").toBool()) {
-			worker_pool_ = new RenderWorkerPool(this);
+			worker_pool_ = new RenderWorkerPool(decoder_cache_, this);
 			worker_pool_->start(QThread::NormalPriority);
 			backend_ = kMultiProcess;
 		}

app/render/renderprocessor.cpp

@@ -35,6 +35,7 @@
 #include "render/plugin/pluginrenderer.h"
 #include "pluginSupport/OliveClip.h"
 #include "pluginSupport/OliveHost.h"
+#include "render/ipc/frameslotpool.h"
 
 namespace olive
 {
@@ -419,6 +420,81 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination,
 		qWarning() << "HAVEN'T GOTTEN DEFAULT INPUT COLORSPACE";
 	}
 
+	auto blit_color_managed = [&](const TexturePtr &unmanaged_texture,
+								  const VideoParams &texture_params) {
+		if (!render_ctx_ || !unmanaged_texture || IsCancelled()) {
+			return;
+		}
+
+		// We convert to our rendering pixel format, since that will always be float-based which
+		// is necessary for correct color conversion
+		ColorProcessorPtr processor = ColorProcessor::Create(
+			color_manager, using_colorspace,
+			color_manager->GetReferenceColorSpace());
+
+		ColorTransformJob job;
+		job.SetColorProcessor(processor);
+		job.SetInputTexture(unmanaged_texture);
+
+		if (texture_params.channel_count() != VideoParams::kRGBAChannelCount ||
+			texture_params.colorspace() == color_manager->GetReferenceColorSpace()) {
+			job.SetInputAlphaAssociation(kAlphaNone);
+		} else if (texture_params.premultiplied_alpha()) {
+			job.SetInputAlphaAssociation(kAlphaAssociated);
+		} else {
+			job.SetInputAlphaAssociation(kAlphaUnassociated);
+		}
+
+		render_ctx_->BlitColorManaged(job, destination.get());
+		// macOS TBDR: ensure tile writeback completes before the texture
+		// is read back in a potentially different shared OpenGL context.
+		render_ctx_->Flush();
+	};
+
+	auto *input_pool =
+		QtUtils::ValueToPtr<ipc::FrameSlotPool>(ticket_->property("ipc_input_pool"));
+	int input_slot = -1;
+	const QVariantList input_slots = ticket_->property("ipc_input_slots").toList();
+	if (!input_slots.isEmpty()) {
+		const QVariant cursor_value = ticket_->property("ipc_input_slot_cursor");
+		const int cursor = cursor_value.isValid() ? cursor_value.toInt() : 0;
+		if (cursor >= 0 && cursor < input_slots.size()) {
+			input_slot = input_slots.at(cursor).toInt();
+			ticket_->setProperty("ipc_input_slot_cursor", cursor + 1);
+		}
+	} else {
+		const QVariant input_slot_value = ticket_->property("ipc_input_slot");
+		input_slot = input_slot_value.isValid() ? input_slot_value.toInt() : -1;
+	}
+	if (render_ctx_ && input_pool && input_slot >= 0) {
+		const ipc::FrameSlotMeta *meta = input_pool->Meta(uint32_t(input_slot));
+		if (meta && meta->width > 0 && meta->height > 0 && meta->data_size > 0 &&
+			meta->data_size <= int(input_pool->slot_data_bytes())) {
+			VideoParams input_params = stream_data;
+			input_params.set_width(meta->width);
+			input_params.set_height(meta->height);
+			input_params.set_format(PixelFormat::Format(meta->format));
+			input_params.set_channel_count(meta->channel_count);
+
+			const int bytes_per_pixel = input_params.GetBytesPerPixel();
+			const int linesize_pixels = bytes_per_pixel > 0
+											? meta->linesize / bytes_per_pixel
+											: input_params.effective_width();
+			TexturePtr unmanaged_texture = render_ctx_->CreateTexture(
+				input_params, input_pool->SlotData(uint32_t(input_slot)), linesize_pixels);
+			blit_color_managed(unmanaged_texture, input_params);
+			return;
+		}
+		qWarning() << "RenderProcessor received invalid IPC input frame slot" << input_slot;
+		return;
+	}
+
+	if (!decoder_cache_) {
+		qWarning() << "RenderProcessor has no decoder cache or IPC input frame for"
+				   << stream->filename();
+		return;
+	}
+
 	Decoder::CodecStream default_codec_stream(
 		stream->filename(), stream_data.stream_index(), GetCurrentBlock());
 
@@ -474,32 +550,7 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination,
 				unmanaged_texture = decoder->RetrieveVideo(p);
 
 				if (!IsCancelled() && unmanaged_texture) {
-					// We convert to our rendering pixel format, since that will always be float-based which
+					blit_color_managed(unmanaged_texture, stream_data);
-					// is necessary for correct color conversion
-					ColorProcessorPtr processor = ColorProcessor::Create(
-						color_manager, using_colorspace,
-						color_manager->GetReferenceColorSpace());
-
-					ColorTransformJob job;
-
-					job.SetColorProcessor(processor);
-					job.SetInputTexture(unmanaged_texture);
-
-					if (stream_data.channel_count() !=
-							VideoParams::kRGBAChannelCount ||
-						stream_data.colorspace() ==
-							color_manager->GetReferenceColorSpace()) {
-						job.SetInputAlphaAssociation(kAlphaNone);
-					} else if (stream_data.premultiplied_alpha()) {
-						job.SetInputAlphaAssociation(kAlphaAssociated);
-					} else {
-						job.SetInputAlphaAssociation(kAlphaUnassociated);
-					}
-
-					render_ctx_->BlitColorManaged(job, destination.get());
-					// macOS TBDR: ensure tile writeback completes before the texture
-					// is read back in a potentially different shared OpenGL context.
-					render_ctx_->Flush();
 				}
 			}
 		}

app/render/renderworkerpool.cpp

@@ -23,14 +23,18 @@
 #include <QCoreApplication>
 #include <QDir>
 #include <QFile>
+#include <QFileInfo>
 #include <QJsonDocument>
 #include <QJsonObject>
 #include <QProcess>
 #include <QTemporaryFile>
 #include <QXmlStreamWriter>
+#include <optional>
 
 #include "codec/frame.h"
 #include "common/qtutils.h"
+#include "node/project/footage/footage.h"
+#include "node/traverser.h"
 
 namespace olive
 {
@@ -40,6 +44,167 @@ namespace
 
 constexpr int kProtocolVersion = 1;
 
+struct FootageInput {
+	FootageJob job;
+	rational time;
+};
+
+class FootageInputCollector : public NodeTraverser {
+public:
+	QVector<FootageInput> Collect(const RenderManager::RenderVideoParams &params,
+								  CancelAtom *cancel)
+	{
+		SetCancelPointer(cancel);
+		VideoParams cache_params = params.video_params;
+		cache_params.set_format(PixelFormat::F32);
+		SetCacheVideoParams(cache_params);
+		SetCacheAudioParams(params.audio_params);
+
+		rational frame_length = cache_params.frame_rate_as_time_base();
+		if (cache_params.interlacing() != VideoParams::kInterlaceNone) {
+			frame_length /= 2;
+		}
+		NodeValueTable table = GenerateTable(params.node,
+											 TimeRange(params.time,
+													   params.time + frame_length));
+		NodeValue texture = table.Get(NodeValue::kTexture);
+		ResolveJobs(texture);
+
+		if (cache_params.interlacing() != VideoParams::kInterlaceNone) {
+			NodeValueTable second_table =
+				GenerateTable(params.node,
+							  TimeRange(params.time + frame_length,
+										params.time + frame_length * 2));
+			NodeValue second_texture = second_table.Get(NodeValue::kTexture);
+			ResolveJobs(second_texture);
+		}
+
+		return inputs_;
+	}
+
+protected:
+	void ProcessVideoFootage(TexturePtr destination,
+							 const FootageJob *stream,
+							 const rational &input_time) override
+	{
+		Q_UNUSED(destination)
+		if (stream) {
+			inputs_.append({*stream, input_time});
+		}
+	}
+
+private:
+	QVector<FootageInput> inputs_;
+};
+
+DecoderPtr ResolveDecoderFromCache(DecoderCache *decoder_cache,
+								   const QString &decoder_id,
+								   const Decoder::CodecStream &stream)
+{
+	if (!decoder_cache || !stream.IsValid()) {
+		return nullptr;
+	}
+
+	QMutexLocker locker(decoder_cache->mutex());
+	DecoderPair decoder = decoder_cache->value(stream);
+	const qint64 file_last_modified =
+		QFileInfo(stream.filename()).lastModified().toMSecsSinceEpoch();
+
+	if (decoder.decoder && decoder.last_modified == file_last_modified) {
+		return decoder.decoder;
+	}
+
+	decoder.decoder = Decoder::CreateFromID(decoder_id);
+	decoder.last_modified = file_last_modified;
+	decoder_cache->insert(stream, decoder);
+	locker.unlock();
+
+	if (!decoder.decoder || !decoder.decoder->Open(stream)) {
+		qWarning() << "RenderWorkerPool failed to open decoder for"
+				   << stream.filename() << "::" << stream.stream();
+		return nullptr;
+	}
+
+	return decoder.decoder;
+}
+
+FramePtr DecodeInputFrame(DecoderCache *decoder_cache,
+						  const FootageInput &input,
+						  CancelAtom *cancel)
+{
+	VideoParams stream_data = input.job.video_params();
+	QString filename = input.job.filename();
+	DecoderPtr decoder;
+
+	switch (stream_data.video_type()) {
+	case VideoParams::kVideoTypeVideo:
+	case VideoParams::kVideoTypeStill:
+		decoder = ResolveDecoderFromCache(
+			decoder_cache,
+			input.job.decoder(),
+			Decoder::CodecStream(filename, stream_data.stream_index(), nullptr));
+		break;
+	case VideoParams::kVideoTypeImageSequence: {
+		const int64_t frame_number =
+			stream_data.get_time_in_timebase_units(input.time);
+		filename = Decoder::TransformImageSequenceFileName(filename, frame_number);
+		decoder = Decoder::CreateFromID(input.job.decoder());
+		if (decoder &&
+			!decoder->Open(Decoder::CodecStream(filename,
+												stream_data.stream_index(),
+												nullptr))) {
+			decoder = nullptr;
+		}
+		break;
+	}
+	}
+
+	if (!decoder) {
+		return nullptr;
+	}
+
+	Decoder::RetrieveVideoParams retrieve;
+	retrieve.divider = stream_data.divider();
+	retrieve.maximum_format = PixelFormat::U16;
+	retrieve.time = stream_data.video_type() == VideoParams::kVideoTypeVideo
+						? input.time
+						: Decoder::kAnyTimecode;
+	retrieve.cancelled = cancel;
+	retrieve.force_range = stream_data.color_range();
+	retrieve.src_interlacing = stream_data.interlacing();
+	FramePtr frame = decoder->RetrieveVideoFrame(retrieve);
+	if (frame) {
+		frame->set_timestamp(input.time);
+	}
+	return frame;
+}
+
+bool DecodeInputFrames(DecoderCache *decoder_cache,
+					   const RenderManager::RenderVideoParams &params,
+					   CancelAtom *cancel,
+					   QVector<FramePtr> *frames)
+{
+	frames->clear();
+
+	FootageInputCollector collector;
+	const QVector<FootageInput> inputs = collector.Collect(params, cancel);
+	frames->reserve(inputs.size());
+	for (const FootageInput &input : inputs) {
+		if (cancel && cancel->IsCancelled()) {
+			return false;
+		}
+
+		FramePtr frame = DecodeInputFrame(decoder_cache, input, cancel);
+		if (!frame || !frame->is_allocated()) {
+			frames->clear();
+			return false;
+		}
+		frames->append(frame);
+	}
+
+	return true;
+}
+
 QString WorkerProgramPath()
 {
 	const QString dir = QCoreApplication::applicationDirPath();
@@ -100,8 +265,10 @@ bool ReadControlMessage(QProcess *process, QJsonObject *out, QString *error,
 
 }  // namespace
 
-RenderWorkerPool::RenderWorkerPool(QObject *parent)
+RenderWorkerPool::RenderWorkerPool(DecoderCache *decoder_cache,
+								   QObject *parent)
 	: QThread(parent)
+	, decoder_cache_(decoder_cache)
 {
 }
 
@@ -174,6 +341,14 @@ bool RenderWorkerPool::PrepareJob(RenderTicketPtr ticket,
 		return false;
 	}
 
+	QVector<FramePtr> input_frames;
+	if (!DecodeInputFrames(decoder_cache_, params, ticket->GetCancelAtom(),
+						   &input_frames)) {
+		qWarning() << "RenderWorkerPool could not predecode footage inputs;"
+				   << "falling back to in-process render";
+		return false;
+	}
+
 	QString graph_path;
 	if (!WriteGraphSnapshot(project, &graph_path)) {
 		return false;
@@ -183,6 +358,7 @@ bool RenderWorkerPool::PrepareJob(RenderTicketPtr ticket,
 	job->params = params;
 	job->graph_path = graph_path;
 	job->node_token = QString::number(reinterpret_cast<quintptr>(params.node));
+	job->input_frames = input_frames;
 	return true;
 }
 
@@ -244,6 +420,72 @@ void RenderWorkerPool::ProcessJob(const Job &job)
 	ipc::FrameSlotPool output_pool =
 		ipc::FrameSlotPool::Create(region.data(), kOutputSlots, slot_bytes);
 
+	const QString input_shm_key =
+		job.input_frames.isEmpty()
+			? QString()
+			: ipc::SharedMemoryRegion::MakeKey(
+				  QCoreApplication::applicationPid(),
+				  int((reinterpret_cast<quintptr>(job.ticket.get()) + 1) & 0xFFFF));
+	ipc::SharedMemoryRegion input_region;
+	std::optional<ipc::FrameSlotPool> input_pool;
+	QVector<int> input_slots;
+	if (!job.input_frames.isEmpty()) {
+		const uint32_t input_slot_count = uint32_t(job.input_frames.size());
+		const size_t input_region_bytes =
+			ipc::FrameSlotPool::BytesNeeded(input_slot_count, slot_bytes);
+		if (!input_region.Open(input_shm_key, input_region_bytes,
+							   ipc::SharedMemoryRegion::kCreate)) {
+			qWarning() << "RenderWorkerPool failed to create input shared memory"
+					   << input_region.error();
+			job.ticket->Finish();
+			return;
+		} else {
+			input_pool = ipc::FrameSlotPool::Create(input_region.data(),
+													input_slot_count,
+													slot_bytes);
+			for (const FramePtr &frame : job.input_frames) {
+				if (frame->allocated_size() > int(slot_bytes)) {
+					qWarning() << "RenderWorkerPool decoded input frame exceeds slot size";
+					job.ticket->Finish();
+					return;
+				}
+
+				uint32_t slot = 0;
+				if (!input_pool->Acquire(&slot)) {
+					qWarning() << "RenderWorkerPool input pool had no free slot";
+					job.ticket->Finish();
+					return;
+				}
+
+				memcpy(input_pool->SlotData(slot), frame->const_data(),
+					   size_t(frame->allocated_size()));
+				ipc::FrameSlotMeta *meta = input_pool->Meta(slot);
+				meta->id = qint64(input_slots.size());
+				meta->time_num = frame->timestamp().numerator();
+				meta->time_den = frame->timestamp().denominator();
+				meta->width = frame->width();
+				meta->height = frame->height();
+				meta->format = int32_t(frame->format());
+				meta->channel_count = frame->channel_count();
+				meta->linesize = frame->linesize_bytes();
+				meta->data_size = frame->allocated_size();
+				if (!input_pool->Publish(slot)) {
+					qWarning() << "RenderWorkerPool failed to publish input slot";
+					job.ticket->Finish();
+					return;
+				}
+				input_slots.append(int(slot));
+			}
+
+			if (input_slots.size() != job.input_frames.size()) {
+				qWarning() << "RenderWorkerPool failed to publish all input frames;"
+						   << "aborting worker render";
+				job.ticket->Finish();
+				return;
+			}
+		}
+	}
+
 	QProcess worker;
 	worker.setProgram(WorkerProgramPath());
 	worker.start();
@@ -268,9 +510,11 @@ void RenderWorkerPool::ProcessJob(const Job &job)
 	ipc::HandshakeMsg handshake;
 	handshake.protocol_version = kProtocolVersion;
 	handshake.shm_key = shm_key;
-	handshake.input_slots = 0;
+	handshake.input_shm_key = input_slots.isEmpty() ? QString() : input_shm_key;
+	handshake.input_slots = input_slots.size();
 	handshake.output_slots = int(kOutputSlots);
 	handshake.slot_data_bytes = qint64(slot_bytes);
+	handshake.input_slot_data_bytes = input_slots.isEmpty() ? 0 : qint64(slot_bytes);
 	if (!WriteControlMessage(&worker, handshake.ToJson())) {
 		qWarning() << "RenderWorkerPool failed to send shared-memory handshake";
 		worker.kill();
@@ -301,6 +545,8 @@ void RenderWorkerPool::ProcessJob(const Job &job)
 	render.format = int(job.params.force_format);
 	render.channel_count = job.params.force_channel_count;
 	render.mode = int(job.params.mode);
+	render.input_slot = input_slots.isEmpty() ? -1 : input_slots.front();
+	render.input_slots = input_slots;
 
 	if (!WriteControlMessage(&worker, render.ToJson())) {
 		qWarning() << "RenderWorkerPool failed to send render_frame";

app/render/renderworkerpool.h

@@ -23,9 +23,11 @@
 
 #include <QMutex>
 #include <QThread>
+#include <QVector>
 #include <QWaitCondition>
 #include <deque>
 
+#include "codec/frame.h"
 #include "node/project/serializer/serializer.h"
 #include "render/ipc/frameslotpool.h"
 #include "render/ipc/ipcmessage.h"
@@ -38,7 +40,8 @@ namespace olive
 class RenderWorkerPool : public QThread {
 	Q_OBJECT
 public:
-	explicit RenderWorkerPool(QObject *parent = nullptr);
+	explicit RenderWorkerPool(DecoderCache *decoder_cache,
+							  QObject *parent = nullptr);
 	~RenderWorkerPool() override;
 
 	bool SubmitFrame(RenderTicketPtr ticket,
@@ -61,6 +64,7 @@ private:
 		RenderManager::RenderVideoParams params;
 		QString graph_path;
 		QString node_token;
+		QVector<FramePtr> input_frames;
 	};
 
 	bool PrepareJob(RenderTicketPtr ticket,
@@ -74,6 +78,7 @@ private:
 						 uint32_t slot);
 	void CleanupGraphFile(const QString &path);
 
+	DecoderCache *decoder_cache_;
 	QMutex mutex_;
 	QWaitCondition wait_;
 	std::deque<Job> queue_;

app/render/worker/workermain.cpp

@@ -107,9 +107,11 @@ public:
 		olive::ipc::HandshakeMsg hs;
 		hs.protocol_version = kProtocolVersion;
 		hs.shm_key = QString();
+		hs.input_shm_key = QString();
 		hs.input_slots = 0;
 		hs.output_slots = 0;
 		hs.slot_data_bytes = 0;
+		hs.input_slot_data_bytes = 0;
 
 		QJsonObject handshake = hs.ToJson();
 		if (QOpenGLContext *ctx = renderer_->context()) {
@@ -200,6 +202,29 @@ private:
 			return Write(ErrorMessage(QStringLiteral("shared memory does not contain a frame slot pool")));
 		}
 
+		input_pool_.reset();
+		input_region_.Close();
+		if (hs.input_slots > 0) {
+			if (hs.input_shm_key.isEmpty() || hs.input_slot_data_bytes <= 0) {
+				return Write(ErrorMessage(QStringLiteral("handshake missing input shared-memory geometry")));
+			}
+
+			const size_t input_bytes = olive::ipc::FrameSlotPool::BytesNeeded(
+				uint32_t(hs.input_slots), size_t(hs.input_slot_data_bytes));
+			if (!input_region_.Open(hs.input_shm_key, input_bytes,
+									olive::ipc::SharedMemoryRegion::kAttach)) {
+				return Write(ErrorMessage(QStringLiteral("failed to attach input shared memory: %1")
+											  .arg(input_region_.error())));
+			}
+
+			input_pool_ = olive::ipc::FrameSlotPool::Attach(input_region_.data());
+			if (!input_pool_->IsValid()) {
+				input_region_.Close();
+				input_pool_.reset();
+				return Write(ErrorMessage(QStringLiteral("input shared memory does not contain a frame slot pool")));
+			}
+		}
+
 		return true;
 	}
 
@@ -265,6 +290,38 @@ private:
 									  message.ticket_id));
 		}
 
+		QVector<int> input_slots;
+		const QVector<int> requested_input_slots =
+			message.input_slots.isEmpty() && message.input_slot >= 0
+				? QVector<int>{message.input_slot}
+				: message.input_slots;
+		if (!requested_input_slots.isEmpty()) {
+			if (!input_pool_ || !input_pool_->IsValid()) {
+				return Write(ErrorMessage(QStringLiteral("render_frame referenced input slot without input pool"),
+										  message.ticket_id));
+			}
+
+			for (int requested_slot : requested_input_slots) {
+				uint32_t consumed_slot = 0;
+				if (!input_pool_->Consume(&consumed_slot)) {
+					for (int slot : input_slots) {
+						input_pool_->Release(uint32_t(slot));
+					}
+					return Write(ErrorMessage(QStringLiteral("input slot was not ready"),
+											  message.ticket_id));
+				}
+				if (int(consumed_slot) != requested_slot) {
+					input_pool_->Release(consumed_slot);
+					for (int slot : input_slots) {
+						input_pool_->Release(uint32_t(slot));
+					}
+					return Write(ErrorMessage(QStringLiteral("input slot order mismatch"),
+											  message.ticket_id));
+				}
+				input_slots.append(int(consumed_slot));
+			}
+		}
+
 		olive::VideoParams vparams(message.width > 0 ? message.width : kDefaultWidth,
 								   message.height > 0 ? message.height : kDefaultHeight,
 								   olive::rational(1, kDefaultFrameRate),
@@ -298,9 +355,24 @@ private:
 		ticket->setProperty("cachetimebase", QVariant::fromValue(olive::rational(1)));
 		ticket->setProperty("cacheid", QVariant::fromValue(QUuid()));
 		ticket->setProperty("multicam", olive::QtUtils::PtrToValue(static_cast<void *>(nullptr)));
+		ticket->setProperty("ipc_input_pool",
+							olive::QtUtils::PtrToValue(
+								input_pool_ ? static_cast<void *>(&*input_pool_)
+											: static_cast<void *>(nullptr)));
+		QVariantList input_slot_values;
+		for (int slot : input_slots) {
+			input_slot_values.append(slot);
+		}
+		ticket->setProperty("ipc_input_slots", input_slot_values);
+		ticket->setProperty("ipc_input_slot_cursor", 0);
+		ticket->setProperty("ipc_input_slot",
+							input_slots.isEmpty() ? -1 : input_slots.front());
 
 		ticket->Start();
 		olive::RenderProcessor::Process(ticket, renderer_, nullptr, &shader_cache_);
+		for (int slot : input_slots) {
+			input_pool_->Release(uint32_t(slot));
+		}
 		if (!ticket->HasResult()) {
 			return Write(ErrorMessage(QStringLiteral("render produced no frame"), message.ticket_id));
 		}
@@ -353,6 +425,8 @@ private:
 	QHash<QString, olive::Node *> node_by_token_;
 	olive::ipc::SharedMemoryRegion output_region_;
 	std::optional<olive::ipc::FrameSlotPool> output_pool_;
+	olive::ipc::SharedMemoryRegion input_region_;
+	std::optional<olive::ipc::FrameSlotPool> input_pool_;
 	olive::ShaderCache shader_cache_;
 };
 

docs/zh/render-process-isolation-plan.md

@@ -203,10 +203,11 @@ compact `QJsonObject`，`\n` 结尾。仅承载低频控制流量（握手、提
 
 ### 阶段 4：素材输入解耦（关键重构）
 
-- `RenderProcessor::ProcessVideoFootage()`（`renderprocessor.cpp:397`）当前经 `ResolveDecoderFromInput` + `DecoderCache` 解码。worker 不链接 FFmpeg，需改为从输入 slot 取已解码帧上传纹理。
+- ✅ `Decoder` 增加 CPU 帧接口 `RetrieveVideoFrame()`；FFmpeg 路径输出 packed RGBA CPU frame，OIIO 路径返回 still frame CPU buffer。
-- 主进程侧 `RenderWorkerPool` 派发前用 `DecoderCache` 解出所需原始帧写入输入 slot，索引随 `render_frame` 一起发。
+- ✅ `RenderWorkerPool` 派发前 dry-run 遍历当前帧素材输入，使用主进程 `DecoderCache` 预解码，成功后写入 main→worker 输入 `FrameSlotPool`。
-- 先支持单素材片段，再扩展到多层/转场。
+- ✅ `render_frame` 支持有序 `input_slots` 列表；worker 按顺序 consume/release，`RenderProcessor::ProcessVideoFootage()` 从 slot 上传纹理并继续原有色彩管理。
-- 验证：渲染含真实素材的时间线帧，与单进程结果逐像素一致。
+- ✅ 没有输入 slot 且 worker 无 `DecoderCache` 时，素材节点安全跳过，不再空指针崩溃。
+- 待补：真实素材项目端到端像素一致性验证；复杂多层/转场/重复素材场景下输入 slot 顺序回归；CPU 预解码失败时的更细粒度回退策略。
 
 ### 阶段 5：多 worker、取消、健壮性
 
@@ -246,7 +247,9 @@ compact `QJsonObject`，`\n` 结尾。仅承载低频控制流量（握手、提
 | `app/CMakeLists.txt`（新增 `olive-render-worker` target） | 1 | ✅ |
 | `app/node/project/serializer/serializer*.{h,cpp}`（暴露加载映射供 worker 查节点） | 2 | ✅ |
 | `app/render/rendermanager.{h,cpp}`（`kMultiProcess` 分支 + WorkerPool 接线） | 3 | ✅ 单 worker MVP |
-| `app/render/renderprocessor.cpp`（`ProcessVideoFootage` 改取输入 slot） | 4 | 待办 |
+| `app/codec/decoder.{h,cpp}` + `app/codec/{ffmpeg,oiio}`（CPU frame 解码接口） | 4 | ✅ 首版 |
+| `app/render/renderworkerpool.{h,cpp}`（主进程预解码并填 input slot） | 4 | ✅ 首版 |
+| `app/render/renderprocessor.cpp`（`ProcessVideoFootage` 改取输入 slot） | 4 | ✅ 首版 |
 | `app/config/config.cpp`（多进程开关默认值） | 3 | ✅ 默认关闭 |
 
 ---

tests/gtest/CMakeLists.txt

@@ -30,6 +30,7 @@ add_executable(olive-gtest
   plugin_renderer_readback_test.cpp
   plugin_ofx_integration_test.cpp
   codec_frame_test.cpp
+  codec_decoder_test.cpp
   codec_exportcodec_test.cpp
   codec_exportformat_test.cpp
   codec_encoder_test.cpp

tests/gtest/codec_decoder_test.cpp

@@ -0,0 +1,40 @@
+#include <gtest/gtest.h>
+
+#include <QFileInfo>
+
+#include "codec/decoder.h"
+
+TEST(CodecDecoder, RetrieveVideoFrameFromDemoMp4)
+{
+	const QString path = QStringLiteral("tests/demo.mp4");
+	ASSERT_TRUE(QFileInfo::exists(path));
+
+	olive::DecoderPtr decoder = olive::Decoder::CreateFromID(QStringLiteral("ffmpeg"));
+	ASSERT_TRUE(decoder);
+
+	ASSERT_TRUE(decoder->Open(olive::Decoder::CodecStream(path, 0, nullptr)));
+
+	olive::Decoder::RetrieveVideoParams params;
+	params.time = olive::rational(0);
+	params.maximum_format = olive::core::PixelFormat::U16;
+
+	olive::FramePtr frame = decoder->RetrieveVideoFrame(params);
+	ASSERT_TRUE(frame);
+	ASSERT_TRUE(frame->is_allocated());
+	EXPECT_EQ(frame->width(), 1920);
+	EXPECT_EQ(frame->height(), 1080);
+	EXPECT_EQ(frame->format(), olive::core::PixelFormat::U16);
+	EXPECT_EQ(frame->channel_count(), 4);
+	EXPECT_GT(frame->allocated_size(), 0);
+	EXPECT_GT(frame->linesize_bytes(), 0);
+
+	bool has_nonzero_byte = false;
+	const char *data = frame->const_data();
+	for (int i = 0; i < frame->allocated_size(); i++) {
+		if (data[i] != 0) {
+			has_nonzero_byte = true;
+			break;
+		}
+	}
+	EXPECT_TRUE(has_nonzero_byte);
+}

tests/gtest/render_ipc_test.cpp

@@ -274,9 +274,11 @@ TEST(IpcMessage, TypedRoundTrip)
 	HandshakeMsg hs;
 	hs.protocol_version = 1;
 	hs.shm_key = QStringLiteral("olive-rw-1234-0");
+	hs.input_shm_key = QStringLiteral("olive-in-1234-0");
 	hs.input_slots = 4;
 	hs.output_slots = 6;
 	hs.slot_data_bytes = 256ll * 1024 * 1024;
+	hs.input_slot_data_bytes = 128ll * 1024 * 1024;
 	ASSERT_TRUE(WriteMessage(&dev, hs.ToJson()));
 
 	RenderFrameMsg rf;
@@ -289,6 +291,8 @@ TEST(IpcMessage, TypedRoundTrip)
 	rf.format = 3;
 	rf.channel_count = 4;
 	rf.mode = 1;
+	rf.input_slot = 2;
+	rf.input_slots = {2, 3};
 	ASSERT_TRUE(WriteMessage(&dev, rf.ToJson()));
 
 	FrameReadyMsg fr;
@@ -308,9 +312,11 @@ TEST(IpcMessage, TypedRoundTrip)
 	ASSERT_TRUE(HandshakeMsg::FromJson(obj, &hs2));
 	EXPECT_EQ(hs2.protocol_version, 1);
 	EXPECT_EQ(hs2.shm_key, hs.shm_key);
+	EXPECT_EQ(hs2.input_shm_key, hs.input_shm_key);
 	EXPECT_EQ(hs2.input_slots, 4);
 	EXPECT_EQ(hs2.output_slots, 6);
 	EXPECT_EQ(hs2.slot_data_bytes, hs.slot_data_bytes);
+	EXPECT_EQ(hs2.input_slot_data_bytes, hs.input_slot_data_bytes);
 
 	ASSERT_TRUE(ReadMessage(&reader, &obj, &ok));
 	ASSERT_TRUE(ok);
@@ -322,6 +328,10 @@ TEST(IpcMessage, TypedRoundTrip)
 	EXPECT_EQ(rf2.time_den, 30000);
 	EXPECT_EQ(rf2.width, 1920);
 	EXPECT_EQ(rf2.format, 3);
+	EXPECT_EQ(rf2.input_slot, 2);
+	ASSERT_EQ(rf2.input_slots.size(), 2);
+	EXPECT_EQ(rf2.input_slots[0], 2);
+	EXPECT_EQ(rf2.input_slots[1], 3);
 
 	ASSERT_TRUE(ReadMessage(&reader, &obj, &ok));
 	ASSERT_TRUE(ok);