Add decoded input slots for render workers

app/codec/decoder.cpp

@@ -124,6 +124,29 @@ TexturePtr Decoder::RetrieveVideo(const RetrieveVideoParams &p)
 	return cached_texture_;
 }
 
+FramePtr Decoder::RetrieveVideoFrame(const RetrieveVideoParams &p)
+{
+	QMutexLocker locker(&mutex_);
+
+	UpdateLastAccessed();
+
+	if (!stream_.IsValid()) {
+		qCritical() << "Can't retrieve video frame on a closed decoder";
+		return nullptr;
+	}
+
+	if (!SupportsVideo()) {
+		qCritical() << "Decoder doesn't support video";
+		return nullptr;
+	}
+
+	if (p.cancelled && p.cancelled->IsCancelled()) {
+		return nullptr;
+	}
+
+	return RetrieveVideoFrameInternal(p);
+}
+
 Decoder::RetrieveAudioStatus
 Decoder::RetrieveAudio(SampleBuffer &dest, const TimeRange &range,
 					   const AudioParams &params, const QString &cache_path,
@@ -291,6 +314,12 @@ TexturePtr Decoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
 	return nullptr;
 }
 
+FramePtr Decoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p)
+{
+	Q_UNUSED(p)
+	return nullptr;
+}
+
 bool Decoder::ConformAudioInternal(const QVector<QString> &filenames,
 								   const AudioParams &params,
 								   CancelAtom *cancelled)

app/codec/decoder.h

@@ -32,6 +32,7 @@ extern "C" {
 #include <QWaitCondition>
 #include <stdint.h>
 
+ #include "codec/frame.h"
 #include "node/block/block.h"
 #include "node/project/footage/footagedescription.h"
 #include "render/cancelatom.h"
@@ -181,6 +182,14 @@ public:
    */
 	TexturePtr RetrieveVideo(const RetrieveVideoParams &p);
 
+	/**
+   * @brief Retrieves a decoded video frame in CPU memory.
+   *
+   * Used by render-process isolation to decode media in the main process and pass packed pixel
+   * data to workers through shared memory.
+   */
+	FramePtr RetrieveVideoFrame(const RetrieveVideoParams &p);
+
 	enum RetrieveAudioStatus {
 		kInvalid = -1,
 		kOK,
@@ -283,6 +292,8 @@ protected:
    */
 	virtual TexturePtr RetrieveVideoInternal(const RetrieveVideoParams &p);
 
+	virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p);
+
 	virtual bool ConformAudioInternal(const QVector<QString> &filenames,
 									  const AudioParams &params,
 									  CancelAtom *cancelled);

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -23,6 +23,34 @@
 
 namespace olive {
 
+static FramePtr CopyPackedAVFrameToFrame(const AVFramePtr &src,
+										 PixelFormat format,
+										 int channel_count,
+										 const rational &timestamp)
+{
+	if (!src || !src->data[0]) {
+		return nullptr;
+	}
+
+	VideoParams params(src->width, src->height, format, channel_count);
+	FramePtr frame = Frame::Create();
+	frame->set_video_params(params);
+	frame->set_timestamp(timestamp);
+	if (!frame->allocate()) {
+		return nullptr;
+	}
+
+	const int row_bytes = params.effective_width() *
+						  VideoParams::GetBytesPerPixel(format, channel_count);
+	for (int y = 0; y < frame->height(); y++) {
+		memcpy(frame->data() + y * frame->linesize_bytes(),
+			   src->data[0] + y * src->linesize[0],
+			   size_t(row_bytes));
+	}
+
+	return frame;
+}
+
 static VideoParams::Interlacing FFmpegFieldOrderToOlive(AVFieldOrder fo)
 {
 	switch (fo) {
@@ -375,6 +403,74 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
 	return nullptr;
 }
 
+FramePtr FFmpegDecoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p)
+{
+	if (AVFramePtr f = RetrieveFrame(p.time, p.cancelled)) {
+		if (p.cancelled && p.cancelled->IsCancelled()) {
+			return nullptr;
+		}
+
+		f->format = FFmpegUtils::ConvertJPEGSpaceToRegularSpace(
+			static_cast<AVPixelFormat>(f->format));
+		f->color_range = p.force_range == VideoParams::kColorRangeFull ?
+							 AVCOL_RANGE_JPEG :
+							 AVCOL_RANGE_MPEG;
+
+		AVFramePtr dest = CreateAVFramePtr();
+		dest->width = f->width;
+		dest->height = f->height;
+		dest->format = p.maximum_format == PixelFormat::U8
+						   ? AV_PIX_FMT_RGBA
+						   : AV_PIX_FMT_RGBA64;
+		dest->color_range = f->color_range;
+		dest->colorspace = f->colorspace;
+		if (p.divider > 1) {
+			dest->width = VideoParams::GetScaledDimension(dest->width, p.divider);
+			dest->height = VideoParams::GetScaledDimension(dest->height, p.divider);
+		}
+
+		int r = av_frame_get_buffer(dest.get(), 0);
+		if (r < 0) {
+			FFmpegError(r);
+			return nullptr;
+		}
+
+		SwsContext *cpu_sws = sws_getContext(
+			f->width, f->height, static_cast<AVPixelFormat>(f->format),
+			dest->width, dest->height, static_cast<AVPixelFormat>(dest->format),
+			SWS_POINT, nullptr, nullptr, nullptr);
+		if (!cpu_sws) {
+			qCritical() << "Failed to create CPU frame conversion context";
+			return nullptr;
+		}
+
+		sws_setColorspaceDetails(
+			cpu_sws,
+			sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace(
+				dest->colorspace)),
+			dest->color_range == AVCOL_RANGE_JPEG ? 1 : 0,
+			sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace(
+				dest->colorspace)),
+			dest->color_range == AVCOL_RANGE_JPEG ? 1 : 0, 0, 0x10000, 0x10000);
+
+		r = sws_scale_frame(cpu_sws, dest.get(), f.get());
+		sws_freeContext(cpu_sws);
+		if (r < 0) {
+			FFmpegError(r);
+			return nullptr;
+		}
+
+		return CopyPackedAVFrameToFrame(dest,
+										dest->format == AV_PIX_FMT_RGBA
+											? PixelFormat::U8
+											: PixelFormat::U16,
+										VideoParams::kRGBAChannelCount,
+										p.time);
+	}
+
+	return nullptr;
+}
+
 void FFmpegDecoder::CloseInternal()
 {
 	if (working_packet_) {

app/codec/ffmpeg/ffmpegdecoder.h

@@ -73,6 +73,7 @@ protected:
 	virtual bool OpenInternal() override;
 	virtual TexturePtr
 	RetrieveVideoInternal(const RetrieveVideoParams &p) override;
+	virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p) override;
 	virtual bool ConformAudioInternal(const QVector<QString> &filenames,
 									  const AudioParams &params,
 									  CancelAtom *cancelled) override;

app/codec/oiio/oiiodecoder.cpp

@@ -122,6 +122,17 @@ bool OIIODecoder::OpenInternal()
 }
 
 TexturePtr OIIODecoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
+{
+	FramePtr frame = RetrieveVideoFrameInternal(p);
+	if (!frame) {
+		return nullptr;
+	}
+
+	return p.renderer->CreateTexture(frame->video_params(), frame->data(),
+									 frame->linesize_pixels());
+}
+
+FramePtr OIIODecoder::RetrieveVideoFrameInternal(const RetrieveVideoParams &p)
 {
 	VideoParams vp = GetVideoParamsFromImageSpec(image_->spec());
 	vp.set_divider(p.divider);
@@ -163,15 +174,19 @@ TexturePtr OIIODecoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
 	if (vp.format() != PixelFormat::F32) {
 		FramePtr f32_frame = buffer_.convert(PixelFormat::F32);
 		if (f32_frame) {
-			VideoParams f32_vp = vp;
-			f32_vp.set_format(PixelFormat::F32);
-			return p.renderer->CreateTexture(f32_vp, f32_frame->data(),
-											 f32_frame->linesize_pixels());
+			f32_frame->set_timestamp(p.time);
+			return f32_frame;
 		}
 	}
 
-	return p.renderer->CreateTexture(vp, buffer_.data(),
-									 buffer_.linesize_pixels());
+	FramePtr frame = Frame::Create();
+	frame->set_video_params(buffer_.video_params());
+	frame->set_timestamp(p.time);
+	if (!frame->allocate()) {
+		return nullptr;
+	}
+	memcpy(frame->data(), buffer_.const_data(), size_t(buffer_.allocated_size()));
+	return frame;
 }
 
 void OIIODecoder::CloseInternal()

app/codec/oiio/oiiodecoder.h

@@ -51,6 +51,7 @@ protected:
 	virtual bool OpenInternal() override;
 	virtual TexturePtr
 	RetrieveVideoInternal(const RetrieveVideoParams &p) override;
+	virtual FramePtr RetrieveVideoFrameInternal(const RetrieveVideoParams &p) override;
 	virtual void CloseInternal() override;
 
 private: