From d28202a2dbbf86d4eceef536898cf114f6551a41 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Mon, 30 May 2022 15:34:38 -0700 Subject: [PATCH] render: convert yuv to rgb on gpu Huge performance improvement --- app/codec/ffmpeg/ffmpegdecoder.cpp | 112 +++++++++++++++++++++++++---- app/shaders/yuv2rgb.frag | 35 +++++++++ 2 files changed, 135 insertions(+), 12 deletions(-) create mode 100644 app/shaders/yuv2rgb.frag diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 81ef1a306..12d669d5c 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -52,6 +52,8 @@ extern "C" { namespace olive { +QVariant Yuv2RgbShader; + FFmpegDecoder::FFmpegDecoder() : filter_graph_(nullptr), buffersrc_ctx_(nullptr), @@ -143,16 +145,6 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(Renderer *renderer, const ration { if (AVFramePtr f = RetrieveFrame(timecode, cancelled)) { if (InitScaler(f.get(), params)) { - int r; - r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF); - if (r < 0) { - return nullptr; - } - r = av_buffersink_get_frame(buffersink_ctx_, working_frame_); - if (r < 0) { - return nullptr; - } - VideoParams vp(instance_.avstream()->codecpar->width, instance_.avstream()->codecpar->height, native_pix_fmt_, @@ -161,9 +153,105 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(Renderer *renderer, const ration VideoParams::kInterlaceNone, params.divider); - TexturePtr tex = renderer->CreateTexture(vp, working_frame_->data[0], working_frame_->linesize[0] / vp.GetBytesPerPixel()); + QElapsedTimer t; + t.start(); - av_frame_unref(working_frame_); + TexturePtr tex = nullptr; + const bool hwscale = true; + + // Attempt to use GLSL shader for faster YUV to RGB conversion + if (hwscale) { + AVPixelFormat src_fmt = AVPixelFormat(f.get()->format); + if (src_fmt == AV_PIX_FMT_YUV420P + || src_fmt == AV_PIX_FMT_YUV422P + || src_fmt == AV_PIX_FMT_YUV444P + || src_fmt == AV_PIX_FMT_YUV420P10LE + || src_fmt == AV_PIX_FMT_YUV422P10LE + || src_fmt == AV_PIX_FMT_YUV444P10LE + || src_fmt == AV_PIX_FMT_YUV420P12LE + || src_fmt == AV_PIX_FMT_YUV422P12LE + || src_fmt == AV_PIX_FMT_YUV444P12LE) { + if (Yuv2RgbShader.isNull()) { + // Compile shader + Yuv2RgbShader = renderer->CreateNativeShader(ShaderCode(FileFunctions::ReadFileAsString(QStringLiteral(":/shaders/yuv2rgb.frag")))); + } + + if (!Yuv2RgbShader.isNull()) { + int px_size; + int bits_per_pixel; + switch (src_fmt) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUV444P: + default: + px_size = 1; + bits_per_pixel = 8; + break; + case AV_PIX_FMT_YUV420P10LE: + case AV_PIX_FMT_YUV422P10LE: + case AV_PIX_FMT_YUV444P10LE: + px_size = 2; + bits_per_pixel = 10; + break; + case AV_PIX_FMT_YUV420P12LE: + case AV_PIX_FMT_YUV422P12LE: + case AV_PIX_FMT_YUV444P12LE: + px_size = 2; + bits_per_pixel = 12; + break; + } + + VideoParams plane_params = vp; + plane_params.set_channel_count(1); + plane_params.set_divider(1); + TexturePtr y_plane = renderer->CreateTexture(plane_params, f->data[0], f->linesize[0] / px_size); + + if (src_fmt == AV_PIX_FMT_YUV420P + || src_fmt == AV_PIX_FMT_YUV422P + || src_fmt == AV_PIX_FMT_YUV420P10LE + || src_fmt == AV_PIX_FMT_YUV422P10LE + || src_fmt == AV_PIX_FMT_YUV420P12LE + || src_fmt == AV_PIX_FMT_YUV422P12LE) { + plane_params.set_width(plane_params.width()/2); + } + + if (src_fmt == AV_PIX_FMT_YUV420P + || src_fmt == AV_PIX_FMT_YUV420P10LE + || src_fmt == AV_PIX_FMT_YUV420P12LE) { + plane_params.set_height(plane_params.height()/2); + } + + TexturePtr u_plane = renderer->CreateTexture(plane_params, f->data[1], f->linesize[1] / px_size); + TexturePtr v_plane = renderer->CreateTexture(plane_params, f->data[2], f->linesize[2] / px_size); + + ShaderJob job; + job.Insert(QStringLiteral("y_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(y_plane))); + job.Insert(QStringLiteral("u_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(u_plane))); + job.Insert(QStringLiteral("v_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(v_plane))); + job.Insert(QStringLiteral("bits_per_pixel"), NodeValue(NodeValue::kInt, bits_per_pixel)); + + tex = renderer->CreateTexture(vp); + renderer->BlitToTexture(Yuv2RgbShader, job, tex.get(), false); + } + } + } + + if (!tex) { + // Fallback to software pixel format conversion + int r; + r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + return nullptr; + } + r = av_buffersink_get_frame(buffersink_ctx_, working_frame_); + if (r < 0) { + return nullptr; + } + + tex = renderer->CreateTexture(vp, working_frame_->data[0], working_frame_->linesize[0] / vp.GetBytesPerPixel()); + + av_frame_unref(working_frame_); + } return tex; } diff --git a/app/shaders/yuv2rgb.frag b/app/shaders/yuv2rgb.frag new file mode 100644 index 000000000..62c73e149 --- /dev/null +++ b/app/shaders/yuv2rgb.frag @@ -0,0 +1,35 @@ +uniform sampler2D y_channel; +uniform sampler2D u_channel; +uniform sampler2D v_channel; + +uniform int bits_per_pixel; + +in vec2 ove_texcoord; +out vec4 frag_color; + +void main() { + vec4 rgba; + + vec3 yuv; + + yuv.r = texture(y_channel, ove_texcoord).r; + yuv.g = texture(u_channel, ove_texcoord).r; + yuv.b = texture(v_channel, ove_texcoord).r; + + if (bits_per_pixel == 10) { + yuv *= 64.0; + } else if (bits_per_pixel == 12) { + yuv *= 16.0; + } + + yuv.r = 1.1643 * (yuv.r - 0.0625); + yuv.g = yuv.g - 0.5; + yuv.b = yuv.b - 0.5; + + rgba.r = yuv.r + 1.5958 * yuv.b; + rgba.g = yuv.r - 0.39173 * yuv.g - 0.81290 * yuv.b; + rgba.b = yuv.r + 2.017 * yuv.g; + rgba.a = 1.0; + + frag_color = rgba; +}