ffmpegdecoder: use real colorspace coefficients in shader

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -208,7 +208,7 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(Renderer *renderer, const ration
               break;
             }
 
-            bool jpeg_range = src_fmt == AV_PIX_FMT_YUVJ420P
+            bool full_range = src_fmt == AV_PIX_FMT_YUVJ420P
                 || src_fmt == AV_PIX_FMT_YUVJ422P
                 || src_fmt == AV_PIX_FMT_YUVJ444P;
 
@@ -244,7 +244,13 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(Renderer *renderer, const ration
             job.Insert(QStringLiteral("u_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(u_plane)));
             job.Insert(QStringLiteral("v_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(v_plane)));
             job.Insert(QStringLiteral("bits_per_pixel"), NodeValue(NodeValue::kInt, bits_per_pixel));
-            job.Insert(QStringLiteral("jpeg_range"), NodeValue(NodeValue::kBoolean, jpeg_range));
+            job.Insert(QStringLiteral("full_range"), NodeValue(NodeValue::kBoolean, full_range));
+
+            const int *yuv_coeffs = sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace(f.get()->colorspace));
+            job.Insert(QStringLiteral("yuv_crv"), NodeValue(NodeValue::kInt, yuv_coeffs[0]));
+            job.Insert(QStringLiteral("yuv_cgu"), NodeValue(NodeValue::kInt, yuv_coeffs[2]));
+            job.Insert(QStringLiteral("yuv_cgv"), NodeValue(NodeValue::kInt, yuv_coeffs[3]));
+            job.Insert(QStringLiteral("yuv_cbu"), NodeValue(NodeValue::kInt, yuv_coeffs[1]));
 
             tex = renderer->CreateTexture(vp);
             renderer->BlitToTexture(Yuv2RgbShader, job, tex.get(), false);

app/common/ffmpegutils.cpp

@@ -111,6 +111,28 @@ AVSampleFormat FFmpegUtils::GetFFmpegSampleFormat(const AudioParams::Format &smp
   return AV_SAMPLE_FMT_NONE;
 }
 
+int FFmpegUtils::GetSwsColorspaceFromAVColorSpace(AVColorSpace cs)
+{
+  switch (cs) {
+  case AVCOL_SPC_BT709:
+    return SWS_CS_ITU709;
+  case AVCOL_SPC_FCC:
+    return SWS_CS_FCC;
+  case AVCOL_SPC_BT470BG:
+    return SWS_CS_ITU624;
+  case AVCOL_SPC_SMPTE170M:
+    return SWS_CS_SMPTE170M;
+  case AVCOL_SPC_SMPTE240M:
+    return SWS_CS_SMPTE240M;
+  case AVCOL_SPC_BT2020_NCL:
+    return SWS_CS_BT2020;
+  default:
+    break;
+  }
+
+  return SWS_CS_DEFAULT;
+}
+
 AVPixelFormat FFmpegUtils::GetFFmpegPixelFormat(const VideoParams::Format &pix_fmt, int channel_layout)
 {
   if (channel_layout == VideoParams::kRGBChannelCount) {

app/common/ffmpegutils.h

@@ -24,6 +24,7 @@
 extern "C" {
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
+#include <libswscale/swscale.h>
 }
 
 #include "render/audioparams.h"
@@ -57,6 +58,14 @@ public:
    * @brief Returns an FFmpeg sample format type for a given native type
    */
   static AVSampleFormat GetFFmpegSampleFormat(const AudioParams::Format &smp_fmt);
+
+  /**
+   * @brief Returns an SWS_CS_* macro from an AVColorSpace enum member
+   *
+   * Why aren't these the same thing anyway? And for that matter, why doesn't FFmpeg provide a
+   * convenience function to do this conversion for us? Who knows, but here we are.
+   */
+  static int GetSwsColorspaceFromAVColorSpace(AVColorSpace cs);
 };
 
 using AVFramePtr = std::shared_ptr<AVFrame>;

app/shaders/yuv2rgb.frag

@@ -3,39 +3,59 @@ uniform sampler2D u_channel;
 uniform sampler2D v_channel;
 
 uniform int bits_per_pixel;
-uniform bool jpeg_range;
+uniform bool full_range;
+
+uniform int yuv_crv;
+uniform int yuv_cgu;
+uniform int yuv_cgv;
+uniform int yuv_cbu;
 
 in vec2 ove_texcoord;
 out vec4 frag_color;
 
-void main() {
-  vec4 rgba;
-
+void main()
+{
+  // Sample YUV planes
   vec3 yuv;
-
   yuv.r = texture(y_channel, ove_texcoord).r;
   yuv.g = texture(u_channel, ove_texcoord).r;
   yuv.b = texture(v_channel, ove_texcoord).r;
 
+  // Pixels will have come in aligned to 16-bit regardless of their actual bit depth, so they must
+  // be scaled as if they were actually 16-bit
   if (bits_per_pixel == 10) {
     yuv *= 64.0;
   } else if (bits_per_pixel == 12) {
     yuv *= 16.0;
   }
 
-  yuv.r = 1.1643 * (yuv.r - 0.0625);
+  // Convert YUV limited range from 16-235 to 0-255
+  yuv.r -= 0.0625; // 16/256
+  yuv.r *= 1.1643; // 255/219
+
+  // Convert 0.0-1.0 to -0.5-0.5
   yuv.g = yuv.g - 0.5;
   yuv.b = yuv.b - 0.5;
 
-  rgba.r = yuv.r + 1.5958 * yuv.b;
-  rgba.g = yuv.r - 0.39173 * yuv.g - 0.81290 * yuv.b;
-  rgba.b = yuv.r + 2.017 * yuv.g;
-  rgba.a = 1.0;
+  // Use coefficients to weigh YUV into RGB
+  float crv = float(yuv_crv) / 65536.0;
+  float cgu = float(yuv_cgu) / 65536.0;
+  float cgv = float(yuv_cgv) / 65536.0;
+  float cbu = float(yuv_cbu) / 65536.0;
 
-  if (jpeg_range) {
-    rgba.rgb *= 219.0 / 255.0;
-    rgba.rgb += 16.0 / 255.0;
+  vec4 rgba;
+  rgba.r = yuv.r + crv * yuv.b;
+  rgba.g = yuv.r - cgu * yuv.g - cgv * yuv.b;
+  rgba.b = yuv.r + cbu * yuv.g;
+
+  // If the expected value is full range, transform to full range here
+  if (full_range) {
+    rgba.rgb /= 1.1643;
+    rgba.rgb += 0.0625;
   }
 
+  // Currently this shader is only used for RGB textures, so just set alpha to 1
+  rgba.a = 1.0;
+
   frag_color = rgba;
 }