diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index bed947cbc..aebeb1a9a 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -125,6 +125,24 @@ TexturePtr FFmpegDecoder::ProcessFrameIntoTexture(AVFramePtr f, instance_.avstream(), nullptr), VideoParams::kInterlaceNone, p.divider); + // For YUV formats, force the output texture to F32 RGBA for maximum precision + switch (f->format) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUV444P: + case AV_PIX_FMT_YUV420P10LE: + case AV_PIX_FMT_YUV422P10LE: + case AV_PIX_FMT_YUV444P10LE: + case AV_PIX_FMT_YUV420P12LE: + case AV_PIX_FMT_YUV422P12LE: + case AV_PIX_FMT_YUV444P12LE: + vp.set_format(PixelFormat::F32); + vp.set_channel_count(VideoParams::kRGBAChannelCount); + break; + default: + break; + } + // Create texture TexturePtr tex = p.renderer->CreateTexture(vp); @@ -246,6 +264,11 @@ TexturePtr FFmpegDecoder::ProcessFrameIntoTexture(AVFramePtr f, tex->handleFrame(f); tex->Upload(f->data[0], f->linesize[0] / vp.GetBytesPerPixel()); break; + case AV_PIX_FMT_RGBAF32: + // RGBA F32 can be uploaded directly to the texture + tex->handleFrame(f); + tex->Upload(f->data[0], f->linesize[0] / vp.GetBytesPerPixel()); + break; } // Deinterlace if necessary @@ -752,6 +775,9 @@ PixelFormat FFmpegDecoder::GetNativePixelFormat(AVPixelFormat pix_fmt) case AV_PIX_FMT_RGB48: case AV_PIX_FMT_RGBA64: return PixelFormat::U16; + case AV_PIX_FMT_RGBF32: + case AV_PIX_FMT_RGBAF32: + return PixelFormat::F32; default: return PixelFormat::INVALID; } @@ -762,9 +788,11 @@ int FFmpegDecoder::GetNativeChannelCount(AVPixelFormat pix_fmt) switch (pix_fmt) { case AV_PIX_FMT_RGB24: case AV_PIX_FMT_RGB48: + case AV_PIX_FMT_RGBF32: return VideoParams::kRGBChannelCount; case AV_PIX_FMT_RGBA: case AV_PIX_FMT_RGBA64: + case AV_PIX_FMT_RGBAF32: return VideoParams::kRGBAChannelCount; default: return 0; @@ -807,6 +835,7 @@ bool FFmpegDecoder::IsPixelFormatGLSLCompatible(AVPixelFormat f) case AV_PIX_FMT_YUV444P12LE: case AV_PIX_FMT_RGBA: case AV_PIX_FMT_RGBA64LE: + case AV_PIX_FMT_RGBAF32: return true; default: return false; @@ -856,6 +885,11 @@ AVFramePtr FFmpegDecoder::PreProcessFrame(AVFramePtr f, static_cast(dest->format), p.maximum_format); } + // swscale does not support RGBAF32 as output, fallback to RGBA64 + if (dest->format == AV_PIX_FMT_RGBAF32) { + dest->format = AV_PIX_FMT_RGBA64; + } + int r = av_frame_get_buffer(dest.get(), 0); if (r < 0) { FFmpegError(r); diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index 94d3f3624..b60cae8a9 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -159,6 +159,17 @@ TexturePtr OIIODecoder::RetrieveVideoInternal(const RetrieveVideoParams &p) } } + // Force F32 output for all still images + if (vp.format() != PixelFormat::F32) { + FramePtr f32_frame = buffer_.convert(PixelFormat::F32); + if (f32_frame) { + VideoParams f32_vp = vp; + f32_vp.set_format(PixelFormat::F32); + return p.renderer->CreateTexture(f32_vp, f32_frame->data(), + f32_frame->linesize_pixels()); + } + } + return p.renderer->CreateTexture(vp, buffer_.data(), buffer_.linesize_pixels()); } diff --git a/app/common/ffmpegutils.cpp b/app/common/ffmpegutils.cpp index 2096a347b..c693ee712 100644 --- a/app/common/ffmpegutils.cpp +++ b/app/common/ffmpegutils.cpp @@ -28,7 +28,7 @@ AVPixelFormat FFmpegUtils::GetCompatiblePixelFormat(const AVPixelFormat &pix_fmt, PixelFormat maximum) { - AVPixelFormat possible_pix_fmts[3]; + AVPixelFormat possible_pix_fmts[4]; possible_pix_fmts[0] = AV_PIX_FMT_RGBA; @@ -36,7 +36,12 @@ FFmpegUtils::GetCompatiblePixelFormat(const AVPixelFormat &pix_fmt, possible_pix_fmts[1] = AV_PIX_FMT_NONE; } else { possible_pix_fmts[1] = AV_PIX_FMT_RGBA64; - possible_pix_fmts[2] = AV_PIX_FMT_NONE; + if (maximum == PixelFormat::F32) { + possible_pix_fmts[2] = AV_PIX_FMT_RGBAF32; + possible_pix_fmts[3] = AV_PIX_FMT_NONE; + } else { + possible_pix_fmts[2] = AV_PIX_FMT_NONE; + } } return avcodec_find_best_pix_fmt_of_list(possible_pix_fmts, pix_fmt, 1, diff --git a/app/config/config.cpp b/app/config/config.cpp index bbb9a6780..d9ac4d016 100644 --- a/app/config/config.cpp +++ b/app/config/config.cpp @@ -249,7 +249,7 @@ void Config::SetDefaults() SetEntryInternal(QStringLiteral("OnlinePixelFormat"), NodeValue::kInt, PixelFormat::F32); SetEntryInternal(QStringLiteral("OfflinePixelFormat"), NodeValue::kInt, - PixelFormat::F16); + PixelFormat::F32); SetEntryInternal(QStringLiteral("MarkerColor"), NodeValue::kInt, ColorCoding::kLime); diff --git a/app/node/traverser.cpp b/app/node/traverser.cpp index 63efb9c22..d8b0fc6ff 100644 --- a/app/node/traverser.cpp +++ b/app/node/traverser.cpp @@ -503,6 +503,10 @@ void NodeTraverser::ResolveJobs(NodeValue &val) } else if (plugin::PluginJob* plugin_job=dynamic_cast(base_job)) { VideoParams tex_params = job_tex->params(); + // Force internal working format (F32) for plugin processing, + // matching FootageJob/GenerateJob behavior. + tex_params.set_format(GetCacheVideoParams().format()); + tex_params.set_channel_count(VideoParams::kRGBAChannelCount); TexturePtr tex = CreateTexture(tex_params); diff --git a/app/pluginSupport/OliveClip.cpp b/app/pluginSupport/OliveClip.cpp index 08042cb10..491299a4b 100644 --- a/app/pluginSupport/OliveClip.cpp +++ b/app/pluginSupport/OliveClip.cpp @@ -443,14 +443,6 @@ olive::plugin::OliveClipInstance::getImage(OfxTime time, return image; } - // If this input clip was never populated (no setInputTexture call), - // return nullptr so the plugin knows no image is available. - // This prevents EXC_BAD_ACCESS when plugins (e.g. ofxsMaskMixPix) - // try to read pixel data from an empty/invalid image buffer. - if (!getConnected()) { - return nullptr; - } - // Fetch on demand for the input clip. // Use plugin-preferred params to ensure the image format matches // what the plugin expects (may differ from input texture format) diff --git a/app/render/plugin/pluginrenderer.cpp b/app/render/plugin/pluginrenderer.cpp index 90a36c416..ed0cdf193 100644 --- a/app/render/plugin/pluginrenderer.cpp +++ b/app/render/plugin/pluginrenderer.cpp @@ -489,9 +489,12 @@ static void ChooseSupportedOutputParams( } } -static olive::TexturePtr ConvertTextureForParams( - olive::TexturePtr src, - const olive::VideoParams &dst_params); +// Forward declarations for functions defined later in this file. +static olive::AVFramePtr ConvertFrameIfNeeded(olive::AVFramePtr src, + const olive::VideoParams &dst_params, + olive::Renderer *renderer); +static olive::TexturePtr ConvertTextureForParams(olive::TexturePtr src, + const olive::VideoParams &dst_params); // 作用：根据插件能力与偏好选择输入格式并执行转换。 // Purpose: Select a supported input format and convert texture for the clip. @@ -705,14 +708,13 @@ static olive::AVFramePtr create_avframe_from_ofx_image(OFX::Host::ImageEffect::I // 前置声明：必要时将 AVFrame 转换为目标 VideoParams 对应格式。 // Forward declaration: Convert AVFrame to match destination VideoParams when needed. -static olive::AVFramePtr ConvertFrameIfNeeded(olive::AVFramePtr src, - const olive::VideoParams &dst_params); // 作用：按指定 VideoParams 复制 OFX Image 到 AVFrame，必要时做格式转换。 // Purpose: Copy OFX Image data into an AVFrame using target VideoParams with format conversion. static olive::AVFramePtr create_avframe_from_ofx_image_with_params( OFX::Host::ImageEffect::Image &image, - const olive::VideoParams ¶ms) + const olive::VideoParams ¶ms, + olive::Renderer *renderer = nullptr) { void *data_ptr = image.getPointerProperty(kOfxImagePropData); if (!data_ptr) { @@ -730,19 +732,21 @@ static olive::AVFramePtr create_avframe_from_ofx_image_with_params( // Get ACTUAL source format from image properties std::string image_depth = image.getStringProperty(kOfxImageEffectPropPixelDepth); std::string image_comp = image.getStringProperty(kOfxImageEffectPropComponents); - + int src_channel_count = 4; if (image_comp == kOfxImageComponentRGB) src_channel_count = 3; else if (image_comp == kOfxImageComponentAlpha) src_channel_count = 1; - + + // NOTE: FP16 (Half) format handling has been removed. + // FP16 data is now treated as U16 (2 bytes per component) and converted via FFmpeg. int src_bytes_per_component = 1; if (image_depth == kOfxBitDepthShort) src_bytes_per_component = 2; - else if (image_depth == kOfxBitDepthHalf) src_bytes_per_component = 2; + else if (image_depth == kOfxBitDepthHalf) src_bytes_per_component = 2; // Treat as U16 else if (image_depth == kOfxBitDepthFloat) src_bytes_per_component = 4; - + const int src_bytes_per_pixel = src_channel_count * src_bytes_per_component; const int dst_bytes_per_pixel = params.channel_count() * params.format().byte_count(); - + int row_bytes = image.getIntProperty(kOfxImagePropRowBytes); if (row_bytes <= 0) { row_bytes = width * src_bytes_per_pixel; @@ -752,12 +756,13 @@ static olive::AVFramePtr create_avframe_from_ofx_image_with_params( src += bounds[1] * row_bytes + bounds[0] * src_bytes_per_pixel; // Check if format conversion is needed - bool needs_conversion = (src_bytes_per_pixel != dst_bytes_per_pixel) || + bool needs_conversion = (src_bytes_per_pixel != dst_bytes_per_pixel) || (src_channel_count != params.channel_count()); - + if (needs_conversion) { // Create source frame with actual format AVPixelFormat src_fmt = AV_PIX_FMT_NONE; + if (src_channel_count == 4) { if (src_bytes_per_component == 1) src_fmt = AV_PIX_FMT_RGBA; else if (src_bytes_per_component == 2) src_fmt = AV_PIX_FMT_RGBA64LE; @@ -771,21 +776,29 @@ static olive::AVFramePtr create_avframe_from_ofx_image_with_params( else if (src_bytes_per_component == 2) src_fmt = AV_PIX_FMT_GRAY16LE; else if (src_bytes_per_component == 4) src_fmt = AV_PIX_FMT_GRAYF32LE; } - + + + if (src_fmt != AV_PIX_FMT_NONE) { olive::AVFramePtr src_frame = olive::CreateAVFramePtr(); src_frame->width = width; src_frame->height = height; src_frame->format = src_fmt; if (av_frame_get_buffer(src_frame.get(), 0) >= 0) { + // Copy source data row by row for (int y = 0; y < height; ++y) { memcpy(src_frame->data[0] + y * src_frame->linesize[0], src + y * row_bytes, width * src_bytes_per_pixel); } // Convert to destination format - return ConvertFrameIfNeeded(src_frame, params); + return ConvertFrameIfNeeded(src_frame, params, renderer); + } else { + qWarning().noquote() << "[WARN] av_frame_get_buffer failed for src_fmt=" << src_fmt; } + } else { + qWarning().noquote() + << "[WARN] src_fmt is NONE for depth=" << QString::fromStdString(image_depth); } } @@ -945,10 +958,45 @@ int olive::plugin::detail::BytesToPixels(int byte_linesize, return byte_linesize / bytes_per_pixel; } +// 作用：将 AVPixelFormat 映射为 Olive 的 PixelFormat 和通道数（仅常见 packed 格式）。 +static void GetOliveFormatFromAV(AVPixelFormat fmt, olive::core::PixelFormat *out_fmt, int *out_ch) +{ + switch (fmt) { + case AV_PIX_FMT_GRAY8: + *out_fmt = olive::core::PixelFormat::U8; *out_ch = 1; return; + case AV_PIX_FMT_RGB24: + *out_fmt = olive::core::PixelFormat::U8; *out_ch = 3; return; + case AV_PIX_FMT_RGBA: + *out_fmt = olive::core::PixelFormat::U8; *out_ch = 4; return; + case AV_PIX_FMT_GRAY16LE: + case AV_PIX_FMT_GRAY16BE: + *out_fmt = olive::core::PixelFormat::U16; *out_ch = 1; return; + case AV_PIX_FMT_RGB48LE: + case AV_PIX_FMT_RGB48BE: + *out_fmt = olive::core::PixelFormat::U16; *out_ch = 3; return; + case AV_PIX_FMT_RGBA64LE: + case AV_PIX_FMT_RGBA64BE: + *out_fmt = olive::core::PixelFormat::U16; *out_ch = 4; return; + case AV_PIX_FMT_GRAYF32LE: + case AV_PIX_FMT_GRAYF32BE: + *out_fmt = olive::core::PixelFormat::F32; *out_ch = 1; return; + case AV_PIX_FMT_RGBF32LE: + case AV_PIX_FMT_RGBF32BE: + *out_fmt = olive::core::PixelFormat::F32; *out_ch = 3; return; + case AV_PIX_FMT_RGBAF32LE: + case AV_PIX_FMT_RGBAF32BE: + *out_fmt = olive::core::PixelFormat::F32; *out_ch = 4; return; + default: + *out_fmt = olive::core::PixelFormat::INVALID; *out_ch = 0; return; + } +} + // 作用：必要时将 AVFrame 转换为目标 VideoParams 对应格式。 -// Purpose: Convert AVFrame to match destination VideoParams when needed. +// 优先使用 FFmpeg sws_scale；若不支持且 renderer 可用，则走 GPU 路径。 +// 删除所有手写 CPU 像素循环，避免精度损失与性能瓶颈。 static olive::AVFramePtr ConvertFrameIfNeeded(olive::AVFramePtr src, - const olive::VideoParams &dst_params) + const olive::VideoParams &dst_params, + olive::Renderer *renderer = nullptr) { if (!src) { return nullptr; @@ -959,6 +1007,7 @@ static olive::AVFramePtr ConvertFrameIfNeeded(olive::AVFramePtr src, return src; } + // Same format & size, no conversion needed if (src->format == dst_fmt && src->width == dst_params.width() && src->height == dst_params.height()) { @@ -970,256 +1019,61 @@ static olive::AVFramePtr ConvertFrameIfNeeded(olive::AVFramePtr src, dst->width = dst_params.width(); dst->height = dst_params.height(); if (av_frame_get_buffer(dst.get(), 0) < 0) { + qWarning().noquote() << "[WARN] av_frame_get_buffer failed for dst_fmt=" << dst_fmt; return src; } - auto float_channels = [](AVPixelFormat fmt) -> int { - switch (fmt) { - case AV_PIX_FMT_GRAYF32LE: - case AV_PIX_FMT_GRAYF32BE: - return 1; - case AV_PIX_FMT_RGBF32LE: - case AV_PIX_FMT_RGBF32BE: - return 3; - case AV_PIX_FMT_RGBAF32LE: - case AV_PIX_FMT_RGBAF32BE: - return 4; - default: - return 0; - } - }; - - auto dst_packed_info = [](AVPixelFormat fmt, int *channels, - int *bytes_per_component) -> bool { - switch (fmt) { - case AV_PIX_FMT_GRAY8: - *channels = 1; - *bytes_per_component = 1; - return true; - case AV_PIX_FMT_RGB24: - *channels = 3; - *bytes_per_component = 1; - return true; - case AV_PIX_FMT_RGBA: - *channels = 4; - *bytes_per_component = 1; - return true; - case AV_PIX_FMT_GRAY16LE: - *channels = 1; - *bytes_per_component = 2; - return true; - case AV_PIX_FMT_RGB48LE: - *channels = 3; - *bytes_per_component = 2; - return true; - case AV_PIX_FMT_RGBA64LE: - *channels = 4; - *bytes_per_component = 2; - return true; - default: - return false; - } - }; - - auto float_dst_from_packed = [&](const olive::AVFramePtr &packed_src, - AVPixelFormat float_fmt, - const olive::AVFramePtr &float_dst) -> bool { - if (!packed_src || !float_dst || !packed_src->data[0] || - !float_dst->data[0]) { - return false; - } - const int dst_channels = float_channels(float_fmt); - if (dst_channels == 0) { - return false; - } - int src_channels = 0; - int bytes_per_component = 0; - if (!dst_packed_info(static_cast(packed_src->format), - &src_channels, &bytes_per_component)) { - return false; - } - const float inv_scale = - (bytes_per_component == 2) ? (1.0f / 65535.0f) - : (1.0f / 255.0f); - for (int y = 0; y < packed_src->height; ++y) { - const uint8_t *src_row = - packed_src->data[0] + y * packed_src->linesize[0]; - float *dst_row = reinterpret_cast( - float_dst->data[0] + y * float_dst->linesize[0]); - if (bytes_per_component == 2) { - const uint16_t *src_u16 = - reinterpret_cast(src_row); - for (int x = 0; x < packed_src->width; ++x) { - const uint16_t *pix = src_u16 + x * src_channels; - float r = pix[0] * inv_scale; - float g = (src_channels > 1) ? pix[1] * inv_scale : r; - float b = (src_channels > 2) ? pix[2] * inv_scale : r; - float a = (src_channels > 3) ? pix[3] * inv_scale : 1.0f; - dst_row[x * dst_channels + 0] = r; - if (dst_channels > 1) { - dst_row[x * dst_channels + 1] = g; - } - if (dst_channels > 2) { - dst_row[x * dst_channels + 2] = b; - } - if (dst_channels > 3) { - dst_row[x * dst_channels + 3] = a; - } - } - } else { - for (int x = 0; x < packed_src->width; ++x) { - const uint8_t *pix = src_row + x * src_channels; - float r = pix[0] * inv_scale; - float g = (src_channels > 1) ? pix[1] * inv_scale : r; - float b = (src_channels > 2) ? pix[2] * inv_scale : r; - float a = (src_channels > 3) ? pix[3] * inv_scale : 1.0f; - dst_row[x * dst_channels + 0] = r; - if (dst_channels > 1) { - dst_row[x * dst_channels + 1] = g; - } - if (dst_channels > 2) { - dst_row[x * dst_channels + 2] = b; - } - if (dst_channels > 3) { - dst_row[x * dst_channels + 3] = a; - } - } - } - } - return true; - }; - - const int dst_float_channels = float_channels(dst_fmt); - if (dst_float_channels > 0) { - int src_channels = 0; - int bytes_per_component = 0; - if (dst_packed_info(static_cast(src->format), - &src_channels, &bytes_per_component)) { - if (float_dst_from_packed(src, dst_fmt, dst)) { - return dst; - } - } else { - olive::AVFramePtr packed = olive::CreateAVFramePtr(); - AVPixelFormat packed_fmt = (dst_float_channels == 4) - ? AV_PIX_FMT_RGBA - : (dst_float_channels == 3) - ? AV_PIX_FMT_RGB24 - : AV_PIX_FMT_GRAY8; - packed->format = packed_fmt; - packed->width = dst->width; - packed->height = dst->height; - if (av_frame_get_buffer(packed.get(), 0) >= 0) { - SwsContext *pre_ctx = sws_getContext( - src->width, src->height, - static_cast(src->format), - packed->width, packed->height, packed_fmt, SWS_POINT, - nullptr, nullptr, nullptr); - if (pre_ctx) { - sws_scale(pre_ctx, src->data, src->linesize, 0, src->height, - packed->data, packed->linesize); - sws_freeContext(pre_ctx); - if (float_dst_from_packed(packed, dst_fmt, dst)) { - return dst; - } - } - } - } - } - - auto clamp01 = [](float v) -> float { - return std::clamp(v, 0.0f, 1.0f); - }; - - const int src_float_channels = float_channels( - static_cast(src->format)); - if (src_float_channels > 0) { - int dst_channels = 0; - int bytes_per_component = 0; - if (dst_packed_info(dst_fmt, &dst_channels, &bytes_per_component) && - src->data[0] && dst->data[0]) { - for (int y = 0; y < src->height; ++y) { - const float *src_row = reinterpret_cast( - src->data[0] + y * src->linesize[0]); - uint8_t *dst_row = dst->data[0] + y * dst->linesize[0]; - if (bytes_per_component == 2) { - auto *dst_row_u16 = - reinterpret_cast(dst_row); - for (int x = 0; x < src->width; ++x) { - const float *pix = - src_row + x * src_float_channels; - float r = pix[0]; - float g = (src_float_channels > 1) ? pix[1] : r; - float b = (src_float_channels > 2) ? pix[2] : r; - float a = (src_float_channels > 3) ? pix[3] : 1.0f; - if (dst_channels == 1) { - float luma = 0.2126f * r + 0.7152f * g + 0.0722f * b; - dst_row_u16[x] = static_cast( - std::lround(clamp01(luma) * 65535.0f)); - continue; - } - dst_row_u16[x * dst_channels + 0] = - static_cast( - std::lround(clamp01(r) * 65535.0f)); - dst_row_u16[x * dst_channels + 1] = - static_cast( - std::lround(clamp01(g) * 65535.0f)); - dst_row_u16[x * dst_channels + 2] = - static_cast( - std::lround(clamp01(b) * 65535.0f)); - if (dst_channels == 4) { - dst_row_u16[x * dst_channels + 3] = - static_cast( - std::lround(clamp01(a) * 65535.0f)); - } - } - } else { - for (int x = 0; x < src->width; ++x) { - const float *pix = - src_row + x * src_float_channels; - float r = pix[0]; - float g = (src_float_channels > 1) ? pix[1] : r; - float b = (src_float_channels > 2) ? pix[2] : r; - float a = (src_float_channels > 3) ? pix[3] : 1.0f; - if (dst_channels == 1) { - float luma = 0.2126f * r + 0.7152f * g + 0.0722f * b; - dst_row[x] = static_cast( - std::lround(clamp01(luma) * 255.0f)); - continue; - } - dst_row[x * dst_channels + 0] = - static_cast( - std::lround(clamp01(r) * 255.0f)); - dst_row[x * dst_channels + 1] = - static_cast( - std::lround(clamp01(g) * 255.0f)); - dst_row[x * dst_channels + 2] = - static_cast( - std::lround(clamp01(b) * 255.0f)); - if (dst_channels == 4) { - dst_row[x * dst_channels + 3] = - static_cast( - std::lround(clamp01(a) * 255.0f)); - } - } - } - } - return dst; - } - } - + // Try FFmpeg sws_scale first SwsContext *sws_ctx = sws_getContext( src->width, src->height, static_cast(src->format), dst->width, dst->height, dst_fmt, SWS_POINT, nullptr, nullptr, nullptr); - if (!sws_ctx) { - return src; + if (sws_ctx) { + int ret = sws_scale(sws_ctx, src->data, src->linesize, 0, src->height, + dst->data, dst->linesize); + sws_freeContext(sws_ctx); + if (ret > 0) { + return dst; + } } - sws_scale(sws_ctx, src->data, src->linesize, 0, src->height, - dst->data, dst->linesize); - sws_freeContext(sws_ctx); + // sws_scale failed (e.g. RGBAF32 not supported). Use GPU if renderer available. + if (renderer && src->data[0]) { + olive::core::PixelFormat src_fmt; + int src_ch; + GetOliveFormatFromAV(static_cast(src->format), &src_fmt, &src_ch); + if (src_fmt != olive::core::PixelFormat::INVALID && src_ch > 0) { + olive::VideoParams src_vp(src->width, src->height, src_fmt, src_ch); + int src_bpp = olive::VideoParams::GetBytesPerPixel(src_fmt, src_ch); + int src_linesize_pixels = (src_bpp > 0) ? src->linesize[0] / src_bpp : src->width; - return dst; + olive::TexturePtr src_tex = renderer->CreateTexture( + src_vp, src->data[0], src_linesize_pixels); + if (src_tex) { + olive::TexturePtr dst_tex = renderer->CreateTexture(dst_params); + if (dst_tex) { + olive::ShaderJob job; + job.Insert(QStringLiteral("ove_maintex"), + olive::NodeValue(olive::NodeValue::kTexture, + QVariant::fromValue(src_tex))); + renderer->BlitToTexture(renderer->GetDefaultShader(), job, + dst_tex.get(), false); + + // Download result back to AVFrame + int dst_bpp = dst_params.GetBytesPerPixel(); + int dst_linesize_pixels = (dst_bpp > 0) ? dst->linesize[0] / dst_bpp : dst->width; + dst_tex->Download(dst->data[0], dst_linesize_pixels); + return dst; + } + } + } + } + + qWarning().noquote() + << "[WARN] ConvertFrameIfNeeded failed (sws_scale + GPU both unavailable). " + << "Returning unconverted source. src_fmt=" << src->format + << " dst_fmt=" << dst_fmt; + return src; } // 作用：从字节行跨度换算像素行跨度。 @@ -1234,8 +1088,8 @@ static int LinesizeToPixels(const olive::VideoParams ¶ms, int linesize_bytes return linesize_bytes / bytes_per_pixel; } -// 作用：将纹理转换为指定 VideoParams（CPU 路径，必要时回读）。 -// Purpose: Convert texture to target VideoParams (CPU path with readback). +// 作用：将纹理转换为指定 VideoParams。优先使用 GPU shader 做格式转换， +// 避免 CPU 回读/转换/上传的性能损失和精度损失。 static olive::TexturePtr ConvertTextureForParams(olive::TexturePtr src, const olive::VideoParams &dst_params) { @@ -1250,6 +1104,23 @@ static olive::TexturePtr ConvertTextureForParams(olive::TexturePtr src, return src; } + // GPU path: blit directly to destination format using default shader. + // OpenGL texture sampling automatically normalizes U8/U16 to float, + // and write-out quantizes float back to U8/U16 when needed. + if (auto *renderer = src->renderer()) { + olive::TexturePtr dst = renderer->CreateTexture(dst_params); + if (dst) { + olive::ShaderJob job; + job.Insert(QStringLiteral("ove_maintex"), + olive::NodeValue(olive::NodeValue::kTexture, + QVariant::fromValue(src))); + renderer->BlitToTexture(renderer->GetDefaultShader(), job, dst.get(), + false); + return dst; + } + } + + // CPU fallback: readback, sws_scale, re-upload olive::AVFramePtr frame = src->frame(); if (!frame || !frame->data[0]) { frame = ReadbackTextureToFrame(src, src_params); @@ -1258,7 +1129,7 @@ static olive::TexturePtr ConvertTextureForParams(olive::TexturePtr src, return nullptr; } - olive::AVFramePtr converted = ConvertFrameIfNeeded(frame, dst_params); + olive::AVFramePtr converted = ConvertFrameIfNeeded(frame, dst_params, nullptr); if (!converted || !converted->data[0]) { return nullptr; } @@ -1290,6 +1161,7 @@ static olive::TexturePtr ConvertTextureForParams(olive::TexturePtr src, return dst; } + // 作用：安全获取插件标识符，便于日志输出。 // Purpose: Safely fetch plugin identifier for logging. static QString PluginIdForInstance(const OFX::Host::ImageEffect::Instance *instance) @@ -1387,7 +1259,72 @@ static olive::AVFramePtr DownloadTextureToFrame(const olive::TexturePtr &tex) const olive::VideoParams ¶ms = tex->params(); return ReadbackTextureToFrame(tex, params); } +inline std::vector GetPluginSupportedDepths(const OFX::Host::ImageEffect::Descriptor& desc) +{ + std::vector depths; + const OFX::Host::Property::Set& props = desc.getProps(); + // 获取数组维度（支持几种深度） + int dim = props.getDimension(kOfxImageEffectPropSupportedPixelDepths); + for (int i = 0; i < dim; ++i) { + // Host Support Library 返回 const std::string& + const std::string& val = props.getStringProperty( + kOfxImageEffectPropSupportedPixelDepths, + i + ); + if (!val.empty() && val != kOfxBitDepthNone) { + depths.push_back(val); + } + } + return depths; +} + +// 查询插件/宿主是否支持「各 clip 不同深度」 +inline bool SupportsMultipleClipDepths(const OFX::Host::ImageEffect::Descriptor& desc) +{ + const OFX::Host::Property::Set& props = desc.getProps(); + // 这是单值 int 属性（0 或 1），n = 0 + int val = props.getIntProperty(kOfxImageEffectPropSupportsMultipleClipDepths, 0); + return val != 0; +} + +// 作用：根据插件描述符声明的 kOfxImageEffectPropSupportedPixelDepths， +// 按优先级 F32 > U16 > U8 > F16 选择最佳输入像素格式。 +// 参考 OpenFX API: OfxImageEffectPropSupportedPixelDepths +// Purpose: Select best input pixel format from plugin descriptor's supported +// depth list. Priority: F32 > U16 > U8 > F16. +static PixelFormat SelectBestPluginInputFormat( + const OFX::Host::ImageEffect::Descriptor& desc) +{ + const OFX::Host::Property::Set& props = desc.getProps(); + int dim = props.getDimension(kOfxImageEffectPropSupportedPixelDepths); + + bool supports_f32 = false; + bool supports_u16 = false; + bool supports_u8 = false; + bool supports_f16 = false; + + for (int i = 0; i < dim; ++i) { + const std::string& depth = props.getStringProperty( + kOfxImageEffectPropSupportedPixelDepths, i); + if (depth == kOfxBitDepthFloat) { + supports_f32 = true; + } else if (depth == kOfxBitDepthShort) { + supports_u16 = true; + } else if (depth == kOfxBitDepthByte) { + supports_u8 = true; + } else if (depth == kOfxBitDepthHalf) { + supports_f16 = true; + } + } + + // 优先级：F32 > U16 > U8 > F16 + if (supports_f32) return PixelFormat::F32; + if (supports_u16) return PixelFormat::U16; + if (supports_u8) return PixelFormat::U8; + if (supports_f16) return PixelFormat::F16; + return PixelFormat::INVALID; +} // 作用：执行 OFX 插件渲染全流程（准备输入、调用动作、处理输出）。 // Purpose: Run full OFX plugin render flow (inputs, actions, outputs). void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin::PluginJob& job, @@ -1494,7 +1431,7 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: input_clips[entry.first] = input_clip; } } - + // call getClipPreferences to know which format plugin requires OFX::Host::Property::Set args; args.setDoubleProperty(kOfxPropTime, frame); @@ -1530,9 +1467,11 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: // effect needs to render a given frame (clipped to the RoD). // // In our example we are doing full frame fetches regardless. - + // set correct format for input + + auto &descriptor = instance->getDescriptor(); for (const auto &entry : input_clips) { if (entry.first == kOfxImageEffectOutputClipName) { continue; @@ -1549,27 +1488,23 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: input_tex->handleFrame(ptr); } if (is_usable_input(input_tex)) { - // Determine the plugin's preferred format for this clip - std::string bitdepth = input_clip->getUnmappedBitDepth(); - std::string component = input_clip->getUnmappedComponents(); + // Query the plugin descriptor for supported pixel depths and pick + // the best one according to our priority: F32 > U16 > U8 > F16. + // OpenFX reference: kOfxImageEffectPropSupportedPixelDepths on + // the image effect descriptor lists all depths the plugin can handle. + PixelFormat chosen_fmt = SelectBestPluginInputFormat(descriptor); VideoParams params = input_tex->params(); - PixelFormat plugin_format = PixelFormat::from_ofx(bitdepth); - if (plugin_format != PixelFormat::INVALID) { - params.set_format(plugin_format); + + if (chosen_fmt != PixelFormat::INVALID && params.format() != chosen_fmt) { + params.set_format(chosen_fmt); + TexturePtr converted_tex = + ConvertTextureForParams(input_tex, params); + if (converted_tex && is_usable_input(converted_tex)) { + input_tex = converted_tex; + input_textures[entry.first] = input_tex; + } } - if (!component.empty() && component != kOfxImageComponentNone) { - params.set_channel_count(component); - } - // Convert the texture to the plugin's preferred format BEFORE - // setting it on the clip, so the OFX Image receives correctly - // formatted pixel data. - TexturePtr converted_tex = - ConvertTextureForParams(input_tex, params); - if (converted_tex && is_usable_input(converted_tex)) { - input_tex = converted_tex; - } - input_textures[entry.first] = input_tex; - // Now set the (possibly converted) texture on the clip + input_clip->setInputTexture(input_tex, frame); OfxRectD rod; rod.x1 = 0; @@ -1589,21 +1524,22 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: return; } // set correct format for output - VideoParams output_params = destination_params; // params for plugin - // First set the destination params on the clip - output_clip->setParams(output_params); - // Get the plugin's preferred bitdepth/component from the instance (not descriptor) - // Use getUnmappedBitDepth/Components which use the instance's params_ - std::string bitdepth = output_clip->getUnmappedBitDepth(); - std::string component = output_clip->getUnmappedComponents(); - // If plugin returns a valid format different from destination, update output_params - PixelFormat plugin_format = PixelFormat::from_ofx(bitdepth); - if (plugin_format != PixelFormat::INVALID) { - output_params.set_format(plugin_format); + // Query plugin-supported depths and pick best according to our priority: + // F32 > U16 > U8 > F16. If plugin supports F32 we render directly to F32 + // (zero conversion). If plugin only supports U8/U16 we let it render in + // that format and ConvertFrameIfNeeded will convert back to F32 afterwards. + VideoParams output_params = destination_params; + PixelFormat best_fmt = SelectBestPluginInputFormat(descriptor); + if (best_fmt != PixelFormat::INVALID) { + output_params.set_format(best_fmt); } + output_clip->setParams(output_params); + std::string component = output_clip->getUnmappedComponents(); if (!component.empty() && component != kOfxImageComponentNone) { output_params.set_channel_count(component); } + // Re-set params so the clip knows the possibly-changed channel count + output_clip->setParams(output_params); // The render window is in pixel coordinates // ie: render scale and a PAR of not 1 @@ -1612,7 +1548,7 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: renderWindow.x2 = destination_params.width(); renderWindow.y2 = destination_params.height(); - + stat = instance->beginRenderAction(frame, numFramesToRender, 1.0, false, renderScale, true, interactive); @@ -1628,7 +1564,7 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: AttachOutputTexture(destination); } #endif - + if (!output_params.is_valid()) { qWarning().noquote() @@ -1695,7 +1631,7 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: } if (!use_opengl) { - AVFramePtr frame_ptr = + AVFramePtr frame_ptr = create_avframe_from_ofx_image_with_params(*output_image, output_params); if (!frame_ptr) { @@ -1706,7 +1642,7 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: renderScale, true, interactive); return; } - AVFramePtr converted = ConvertFrameIfNeeded(frame_ptr, destination_params); + AVFramePtr converted = ConvertFrameIfNeeded(frame_ptr, destination_params, this); const AVPixelFormat expected_fmt = GetDestinationAVPixelFormat(destination_params); destination->handleFrame(converted); @@ -1733,7 +1669,7 @@ void olive::plugin::PluginRenderer::RenderPlugin(TexturePtr src, olive::plugin:: #endif if (frame_ptr && destination) { AVFramePtr converted = - ConvertFrameIfNeeded(frame_ptr, destination_params); + ConvertFrameIfNeeded(frame_ptr, destination_params, this); const AVPixelFormat expected_fmt = GetDestinationAVPixelFormat(destination_params); destination->handleFrame(converted); diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index ac122d8d9..28265f803 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -165,7 +165,8 @@ void RenderProcessor::Run() SetCancelPointer(ticket_->GetCancelAtom()); VideoParams params=ticket_->property("vparam").value(); - SetCacheVideoParams(ticket_->property("vparam").value()); + params.set_format(PixelFormat::F32); + SetCacheVideoParams(params); SetCacheAudioParams(ticket_->property("aparam").value()); if (IsCancelled()) { diff --git a/docs/rgbaf32-global-plan.md b/docs/rgbaf32-global-plan.md new file mode 100644 index 000000000..e68a26390 --- /dev/null +++ b/docs/rgbaf32-global-plan.md @@ -0,0 +1,122 @@ +# 素材读入强制转换为 RGBAF32 并内部全链路使用 F32 处理 — 实施计划 + +## 1. 现状分析 + +### 1.1 视频读入位置 + +视频/图像素材在以下位置被读入并解码为 GPU Texture： + +| 层级 | 文件 | 职责 | +|------|------|------| +| 解码接口 | `app/codec/decoder.h` / `.cpp` | 基类 `Decoder`，定义 `RetrieveVideo(RetrieveVideoParams)` 公共接口 | +| FFmpeg 解码 | `app/codec/ffmpeg/ffmpegdecoder.cpp` | `FFmpegDecoder::RetrieveVideoInternal()` —— 核心视频解码路径 | +| OIIO 解码 | `app/codec/oiio/oiiodecoder.cpp` | `OIIODecoder::RetrieveVideoInternal()` —— 静态图片解码路径 | +| 渲染触发 | `app/render/renderprocessor.cpp` | `ProcessVideoFootage()` —— 在节点图遍历中触发解码，并做颜色管理转换 | +| 遍历调度 | `app/node/traverser.cpp` | `ResolveJobs()` —— 将 `FootageJob` 分发给 `ProcessVideoFootage()` | + +**数据流：** +``` +文件 → FFmpegDecoder::RetrieveVideoInternal() + → RetrieveFrame() 解码出 AVFrame + → PreProcessFrame() CPU 缩放/格式转换 (sws_scale_frame) + → ProcessFrameIntoTexture() 上传为 GPU Texture + → YUV 格式：上传为 3 个 plane texture + YUV→RGB shader + → RGBA/RGBA64LE：直接 glTexSubImage2D 上传 + → RenderProcessor::ProcessVideoFootage() + → BlitColorManaged() OCIO 颜色空间转换 shader + → 进入节点图后续处理 +``` + +### 1.2 像素格式体系 + +- **核心枚举：** `ext/core/include/olive/core/render/pixelformat.h` 定义 `PixelFormat::U8 / U16 / F16 / F32` +- **GPU 格式映射：** `app/render/opengl/openglrenderer.cpp` 已将 `F32 + 4ch` 映射到 `GL_RGBA32F / GL_FLOAT` +- **内部工作格式：** `NodeTraverser::GetCacheVideoParams().format()` 决定节点图内部缓存格式 +- **项目默认配置：** `app/config/config.cpp` 中 `OnlinePixelFormat = F32`，`OfflinePixelFormat = F16`，说明设计意图就是在线编辑使用 F32 + +### 1.3 当前 F32 支持的关键缺失 + +1. **`FFmpegDecoder::GetNativePixelFormat()` 不识别 F32 FFmpeg 格式** + - 仅映射 `RGBA → U8`、`RGBA64 → U16` + - `AV_PIX_FMT_RGBAF32`、`AV_PIX_FMT_RGBF32` 等落入 `default: INVALID` + +2. **`IsPixelFormatGLSLCompatible()` 未将 RGBAF32 列为 GLSL 兼容** + - 这会导致即使解码器输出 RGBAF32，也会强制走 `sws_scale_frame` CPU 转换路径 + +3. **`ProcessFrameIntoTexture()` 直接上传路径缺少 RGBAF32 分支** + - 当前只有 `YUV...` 和 `RGBA / RGBA64LE` 两个直接上传分支，没有 `RGBAF32` 等直接上传路径 + +4. **`PreProcessFrame()` 的 `sws_scale_frame` 目标格式选择需验证 F32 支持** + - `FFmpegUtils::GetCompatiblePixelFormat(..., maximum=F32)` 理论上应返回 `AV_PIX_FMT_RGBAF32`，但需实测验证 + +5. **OIIO 解码器已原生支持 F32（FLOAT → F32），无需修改** + +## 2. 目标 + +- **读入时转换：** 无论源素材格式（YUV、U8、U16、F16 等），在解码器层面统一转换为 **RGBAF32** 后上传 GPU +- **内部全链路 F32：** 节点图遍历、效果处理、合成、缓存等内部环节全部使用 `PixelFormat::F32`（4 通道） +- **导出保持灵活：** 导出/编码时从 F32 转换为目标格式，保持现有编码逻辑 + +## 3. 实施方案：全局强制 F32 + +**思路：** 将 F32 作为唯一的内部工作格式，在解码器出口强制转换。 + +**改动点：** + +1. **解码器层强制 F32 输出** + - `FFmpegDecoder::RetrieveVideoInternal()`： + - 修改 `RetrieveVideoParams` 或内部逻辑，令 `maximum_format` 固定为 `F32` + - 在 `PreProcessFrame()` 中，若源格式非 RGBAF32，通过 `sws_scale_frame` 转换到 `AV_PIX_FMT_RGBAF32` + - 在 `ProcessFrameIntoTexture()` 中增加 `AV_PIX_FMT_RGBAF32` 直接上传分支（`GL_RGBA32F / GL_FLOAT`） + - `OIIODecoder::RetrieveVideoInternal()`： + - OIIO 读入后，若格式非 F32，通过 `Frame::convert(PixelFormat::F32)` 转换，再上传 + +2. **修复 F32 格式映射** + - `FFmpegDecoder::GetNativePixelFormat()` 增加 `AV_PIX_FMT_RGBAF32 → PixelFormat::F32`、`AV_PIX_FMT_RGBF32 → PixelFormat::F32` + - `FFmpegDecoder::GetNativeChannelCount()` 增加对应分支 + - `IsPixelFormatGLSLCompatible()` 增加 `AV_PIX_FMT_RGBAF32`（可选，因为强制转换后解码器输出就是 RGBAF32） + +3. **内部工作格式锁定 F32** + - 在 `NodeTraverser` 初始化或 `RenderProcessor` 创建时，`SetCacheVideoParams()` 强制 `format = PixelFormat::F32` + - 移除用户层对工作格式的可选配置（或保留配置但忽略/默认 F32） + - `traverser.cpp` 中 `FootageJob`、`GenerateJob`、`ColorTransformJob` 的格式设置已经使用 `GetCacheVideoParams().format()`，因此只需确保基类参数是 F32 即可 + +4. **导出层适配** + - `FFmpegEncoder` 的输入当前通过 `avfilter` 图做格式转换，源为 F32 时： + - `FFmpegUtils::GetFFmpegPixelFormat(F32, 4)` 已返回 `AV_PIX_FMT_RGBAF32` + - 验证 filter graph 的 `buffer` source 和 `format` filter 能否正确处理 `RGBAF32` + - `RenderProcessor::GenerateFrame()` 下载 GPU texture 到 `FramePtr` 时，`DownloadFromTexture()` 已支持 `GL_FLOAT`，直接得到 F32 CPU buffer + +## 4. 关键文件与修改清单 + +| 文件 | 修改内容 | +|------|----------| +| `app/codec/ffmpeg/ffmpegdecoder.cpp` | ① `GetNativePixelFormat()` 增加 RGBAF32/RGBF32 → F32 映射
② `GetNativeChannelCount()` 增加对应分支
③ `IsPixelFormatGLSLCompatible()` 增加 RGBAF32
④ `ProcessFrameIntoTexture()` 增加 RGBAF32 直接上传分支
⑤ `PreProcessFrame()` 确保 divider=1 且格式为 RGBAF32 时跳过 CPU 转换 | +| `app/codec/oiio/oiiodecoder.cpp` | `RetrieveVideoInternal()` 上传前若 `frame.format() != F32` 则调用 `convert(F32)` | +| `app/node/traverser.cpp` 或 `app/render/renderprocessor.cpp` | 初始化时强制 `SetCacheVideoParams().format = F32` | +| `app/codec/ffmpeg/ffmpegencoder.cpp` | 验证 filter graph 对 RGBAF32 source 的处理，必要时调整 | +| `app/render/opengl/openglrenderer.cpp` | 确认 `GL_RGBA32F / GL_FLOAT` 路径完整，补充必要错误检查 | +| `app/codec/ffmpeg/ffmpegutils.cpp` | 验证 `GetCompatiblePixelFormat(maximum=F32)` 的行为 | + +## 5. 风险评估 + +| 风险 | 说明 | 缓解措施 | +|------|------|----------| +| 内存带宽 ×4 | F32 是 U8 的 4 倍、U16/F16 的 2 倍，显存和内存占用显著增加 | 这是预期代价；`OfflinePixelFormat` 机制可继续用于代理预览，降低分辨率同时用 F16 减少带宽 | +| FFmpeg swscale 对 RGBAF32 支持 | `sws_scale_frame` 是否能正确处理 `AV_PIX_FMT_RGBAF32` 作为目标格式需验证 | 先写单元测试验证；若不支持，可用 OIIO `Frame::convert()` 作为 fallback，或在 GPU 上通过 shader 做格式转换 | +| OFX 插件兼容性 | 大部分 OFX 插件支持 `kOfxBitDepthFloat`，但仍有少数可能只支持 U8/U16 | `PluginRenderer` 已有格式转换路径，F32 的支持比 F16 更成熟 | +| 性能回归 | YUV→RGB 原来在 GPU 走 shader，若强制先转 RGBAF32 再上传，可能需要调整流程 | YUV 素材仍保留 GPU shader 转换路径，只是 shader 输出目标 texture 格式改为 F32（OpenGL 已支持 `GL_RGBA32F` 作为 render target） | +| 缓存文件体积翻倍 | 帧缓存从 U8/U16 改为 F32 后，磁盘缓存体积增大 | 可接受；必要时调整缓存策略或压缩 | + +## 6. 建议的实施顺序 + +1. **第一阶段：** 修复 `FFmpegDecoder` F32 映射 + 增加 RGBAF32 直接上传分支，编写解码器单元测试 +2. **第二阶段：** 在 `OIIODecoder` 添加强制 F32 转换 +3. **第三阶段：** 锁定内部工作格式为 F32，验证节点图全链路 +4. **第四阶段：** 验证导出编码路径，确认 filter graph 对 RGBAF32 的处理 +5. **第五阶段：** 性能测试与回归测试 + +## 7. 决策点 + +- 是否保留 `OfflinePixelFormat = F16` 的代理降级机制？还是连 proxy 也强制 F32？ +- 若保留代理降级，是否需要在解码器层根据 online/offline 模式选择输出格式？