From 73fa930f020d5d0c230451b0649e87f8d1d86186 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Sun, 7 Aug 2022 14:00:12 -0700 Subject: [PATCH] ffmpegencoder: use avfilter instead of swscale for pixel format conversions --- app/codec/ffmpeg/ffmpegdecoder.h | 7 +- app/codec/ffmpeg/ffmpegencoder.cpp | 157 +++++++++++++++-------------- app/codec/ffmpeg/ffmpegencoder.h | 6 +- app/common/ffmpegutils.h | 6 ++ 4 files changed, 90 insertions(+), 86 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 1c0d25bb8..ad9bc571a 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -36,15 +36,10 @@ extern "C" { #include #include "codec/decoder.h" +#include "common/ffmpegutils.h" namespace olive { -using AVFramePtr = std::shared_ptr; -inline AVFramePtr CreateAVFramePtr(AVFrame *f) -{ - return std::shared_ptr(f, [](AVFrame *g){ av_frame_free(&g); }); -} - /** * @brief A Decoder derivative that wraps FFmpeg functions as on Olive decoder */ diff --git a/app/codec/ffmpeg/ffmpegencoder.cpp b/app/codec/ffmpeg/ffmpegencoder.cpp index 2c9f87775..1a66cbae8 100644 --- a/app/codec/ffmpeg/ffmpegencoder.cpp +++ b/app/codec/ffmpeg/ffmpegencoder.cpp @@ -21,6 +21,8 @@ #include "ffmpegencoder.h" extern "C" { +#include +#include #include } @@ -37,6 +39,8 @@ FFmpegEncoder::FFmpegEncoder(const EncodingParams ¶ms) : video_stream_(nullptr), video_codec_ctx_(nullptr), video_scale_ctx_(nullptr), + video_buffersrc_ctx_(nullptr), + video_buffersink_ctx_(nullptr), audio_stream_(nullptr), audio_codec_ctx_(nullptr), audio_resample_ctx_(nullptr), @@ -141,34 +145,59 @@ bool FFmpegEncoder::Open() // This is the pixel format the encoder wants to encode to AVPixelFormat encoder_pix_fmt = video_codec_ctx_->pix_fmt; - // Set up a scaling context - if the native pixel format is not equal to the encoder's, we'll need to convert it - // before encoding. Even if we don't, this may be useful for converting between linesizes, etc. - video_scale_ctx_ = sws_getContext(params().video_params().width(), - params().video_params().height(), - src_alpha_pix_fmt, - params().video_params().width(), - params().video_params().height(), - encoder_pix_fmt, - 0, - nullptr, - nullptr, - nullptr); + video_scale_ctx_ = avfilter_graph_alloc(); + if (!video_scale_ctx_) { + return false; + } - int *inv_table; - int src_range; - int *table; - int dst_range; - int brightness; - int contrast; - int saturation; + static const int FILTER_ARG_SZ = 1024; + char filter_args[FILTER_ARG_SZ]; - sws_getColorspaceDetails(video_scale_ctx_, &inv_table, &src_range, &table, &dst_range, &brightness, &contrast, &saturation); + snprintf(filter_args, FILTER_ARG_SZ, "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d", + params().video_params().effective_width(), + params().video_params().effective_height(), + src_alpha_pix_fmt, + params().video_params().time_base().numerator(), + params().video_params().time_base().denominator(), + params().video_params().pixel_aspect_ratio().numerator(), + params().video_params().pixel_aspect_ratio().denominator()); - // Set swscale's dst range based on AVCodecContext's color_range. Here, 1 == JPEG range (0-255) - // and 0 == MPEG range (16-235). - dst_range = (video_codec_ctx_->color_range == AVCOL_RANGE_JPEG); + avfilter_graph_create_filter(&video_buffersrc_ctx_, avfilter_get_by_name("buffer"), "in", filter_args, nullptr, video_scale_ctx_); + avfilter_graph_create_filter(&video_buffersink_ctx_, avfilter_get_by_name("buffersink"), "out", nullptr, nullptr, video_scale_ctx_); - sws_setColorspaceDetails(video_scale_ctx_, inv_table, src_range, table, dst_range, brightness, contrast, saturation); + AVFilterContext *last_filter = video_buffersrc_ctx_; + + { + // Set color range + AVFilterContext* range_filter; + + snprintf(filter_args, FILTER_ARG_SZ, "in_range=full:out_range=%s", + params().video_color_range() == EncodingParams::kYUVJPEG0_255 ? "full" : "limited"); + + avfilter_graph_create_filter(&range_filter, avfilter_get_by_name("scale"), "range", filter_args, nullptr, video_scale_ctx_); + + avfilter_link(last_filter, 0, range_filter, 0); + last_filter = range_filter; + } + + if (src_alpha_pix_fmt != encoder_pix_fmt) { + // Transform pixel format + AVFilterContext* format_filter; + + snprintf(filter_args, FILTER_ARG_SZ, "pix_fmts=%u", encoder_pix_fmt); + + avfilter_graph_create_filter(&format_filter, avfilter_get_by_name("format"), "format", filter_args, nullptr, video_scale_ctx_); + + avfilter_link(last_filter, 0, format_filter, 0); + last_filter = format_filter; + } + + avfilter_link(last_filter, 0, video_buffersink_ctx_, 0); + + if (avfilter_graph_config(video_scale_ctx_, nullptr) < 0) { + SetError(tr("Failed to configure filter graph")); + return false; + } } // Initialize an audio stream if it's enabled @@ -207,71 +236,41 @@ bool FFmpegEncoder::Open() bool FFmpegEncoder::WriteFrame(FramePtr frame, rational time) { - bool success = false; - - AVFrame* encoded_frame = av_frame_alloc(); - - int error_code; - const char* input_data; - int input_linesize; - - // Frame must be video - encoded_frame->width = frame->width(); - encoded_frame->height = frame->height(); - encoded_frame->format = video_codec_ctx_->pix_fmt; - encoded_frame->color_range = video_codec_ctx_->color_range; - encoded_frame->color_trc = video_codec_ctx_->color_trc; - encoded_frame->color_primaries = video_codec_ctx_->color_primaries; - encoded_frame->colorspace = video_codec_ctx_->colorspace; - - // Set interlacing - if (frame->video_params().interlacing() != VideoParams::kInterlaceNone) { - encoded_frame->interlaced_frame = 1; - - if (frame->video_params().interlacing() == VideoParams::kInterlacedTopFirst) { - encoded_frame->top_field_first = 1; - } else { - encoded_frame->top_field_first = 0; - } - } - - error_code = av_frame_get_buffer(encoded_frame, 0); - if (error_code < 0) { - FFmpegError(tr("Failed to create AVFrame buffer"), error_code); - goto fail; - } - // We may need to convert this frame to a frame that swscale will understand if (frame->format() != video_conversion_fmt_) { frame = frame->convert(video_conversion_fmt_); } // Use swscale context to convert formats/linesizes - input_data = frame->const_data(); - input_linesize = frame->linesize_bytes(); + AVFramePtr input_frame = CreateAVFramePtr(av_frame_alloc()); + input_frame->width = frame->width(); + input_frame->height = frame->height(); + input_frame->format = FFmpegUtils::GetFFmpegPixelFormat(frame->format(), frame->channel_count()); + input_frame->data[0] = reinterpret_cast(frame->data()); + input_frame->linesize[0] = frame->linesize_bytes(); - error_code = sws_scale(video_scale_ctx_, - reinterpret_cast(&input_data), - &input_linesize, - 0, - frame->height(), - encoded_frame->data, - encoded_frame->linesize); + input_frame->color_primaries = video_codec_ctx_->color_primaries; + input_frame->color_trc = video_codec_ctx_->color_trc; + input_frame->colorspace = video_codec_ctx_->colorspace; + input_frame->color_range = video_codec_ctx_->color_range; + int r; + r = av_buffersrc_add_frame_flags(video_buffersrc_ctx_, input_frame.get(), AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + FFmpegError(tr("Failed to add frame to filter graph"), r); + return false; + } - if (error_code < 0) { - FFmpegError(tr("Failed to scale frame"), error_code); - goto fail; + AVFramePtr encoded_frame = CreateAVFramePtr(av_frame_alloc()); + r = av_buffersink_get_frame(video_buffersink_ctx_, encoded_frame.get()); + if (r < 0) { + FFmpegError(tr("Failed to retrieve frame from buffer sink"), r); + return false; } encoded_frame->pts = qRound64(time.toDouble() / av_q2d(video_codec_ctx_->time_base)); - success = WriteAVFrame(encoded_frame, video_codec_ctx_, video_stream_); - -fail: - av_frame_free(&encoded_frame); - - return success; + return WriteAVFrame(encoded_frame.get(), video_codec_ctx_, video_stream_); } bool FFmpegEncoder::WriteAudio(const SampleBuffer &audio) @@ -493,8 +492,10 @@ void FFmpegEncoder::Close() } if (video_scale_ctx_) { - sws_freeContext(video_scale_ctx_); + avfilter_graph_free(&video_scale_ctx_); video_scale_ctx_ = nullptr; + video_buffersrc_ctx_ = nullptr; + video_buffersink_ctx_ = nullptr; } if (video_codec_ctx_) { @@ -656,7 +657,7 @@ bool FFmpegEncoder::InitializeStream(AVMediaType type, AVStream** stream_ptr, AV if (params().color_transform().output().contains(QStringLiteral("sRGB"), Qt::CaseInsensitive)) { codec_ctx->color_primaries = AVCOL_PRI_BT709; codec_ctx->color_trc = AVCOL_TRC_IEC61966_2_1; - codec_ctx->colorspace = AVCOL_SPC_RGB; + codec_ctx->colorspace = AVCOL_SPC_BT709; } else { // Assume Rec.709 codec_ctx->color_primaries = AVCOL_PRI_BT709; codec_ctx->color_trc = AVCOL_TRC_BT709; diff --git a/app/codec/ffmpeg/ffmpegencoder.h b/app/codec/ffmpeg/ffmpegencoder.h index 4b7dc0edc..9a9f6cae3 100644 --- a/app/codec/ffmpeg/ffmpegencoder.h +++ b/app/codec/ffmpeg/ffmpegencoder.h @@ -23,8 +23,8 @@ extern "C" { #include +#include #include -#include #include #include } @@ -88,7 +88,9 @@ private: AVStream* video_stream_; AVCodecContext* video_codec_ctx_; - SwsContext* video_scale_ctx_; + AVFilterGraph *video_scale_ctx_; + AVFilterContext *video_buffersrc_ctx_; + AVFilterContext *video_buffersink_ctx_; VideoParams::Format video_conversion_fmt_; AVStream* audio_stream_; diff --git a/app/common/ffmpegutils.h b/app/common/ffmpegutils.h index cc74d8c48..f8f85aedd 100644 --- a/app/common/ffmpegutils.h +++ b/app/common/ffmpegutils.h @@ -59,6 +59,12 @@ public: static AVSampleFormat GetFFmpegSampleFormat(const AudioParams::Format &smp_fmt); }; +using AVFramePtr = std::shared_ptr; +inline AVFramePtr CreateAVFramePtr(AVFrame *f) +{ + return std::shared_ptr(f, [](AVFrame *g){ av_frame_free(&g); }); +} + } #endif // FFMPEGABSTRACTION_H