diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index e37f8649c..bf6cb4006 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -93,7 +93,7 @@ bool Decoder::Open(const CodecStream &stream) } } -FramePtr Decoder::RetrieveVideo(const rational &timecode, const int ÷r) +FramePtr Decoder::RetrieveVideo(const rational &timecode, const RetrieveVideoParams ÷r) { QMutexLocker locker(&mutex_); @@ -302,7 +302,7 @@ int64_t Decoder::GetImageSequenceIndex(const QString &filename) return number_only.toLongLong(); } -FramePtr Decoder::RetrieveVideoInternal(const rational &timecode, const int ÷r) +FramePtr Decoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams ÷r) { Q_UNUSED(timecode) Q_UNUSED(divider) diff --git a/app/codec/decoder.h b/app/codec/decoder.h index 7070b2ac0..cea86c922 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -143,6 +143,35 @@ public: static const rational kAnyTimecode; + struct RetrieveVideoParams + { + RetrieveVideoParams() + { + divider = 1; + src_interlacing = VideoParams::kInterlaceNone; + dst_interlacing = VideoParams::kInterlaceNone; + } + + int divider; + VideoParams::Interlacing src_interlacing; + VideoParams::Interlacing dst_interlacing; + + void reset() + { + *this = RetrieveVideoParams(); + } + + bool operator==(const RetrieveVideoParams& rhs) const + { + return divider == rhs.divider && src_interlacing == rhs.src_interlacing && dst_interlacing == rhs.dst_interlacing; + } + + bool operator!=(const RetrieveVideoParams& rhs) const + { + return !(*this == rhs); + } + }; + /** * @brief Retrieves a video frame from footage * @@ -153,7 +182,7 @@ public: * * This function is thread safe and can only run while the decoder is open. \see Open() */ - FramePtr RetrieveVideo(const rational& timecode, const int& divider); + FramePtr RetrieveVideo(const rational& timecode, const RetrieveVideoParams& divider); /** * @brief Retrieve audio data from footage @@ -236,7 +265,7 @@ protected: * Sub-classes must override this function IF they support video. Function is already mutexed * so sub-classes don't need to worry about thread safety. */ - virtual FramePtr RetrieveVideoInternal(const rational& timecode, const int& divider); + virtual FramePtr RetrieveVideoInternal(const rational& timecode, const RetrieveVideoParams& divider); virtual bool ConformAudioInternal(const QString& filename, const AudioParams ¶ms, const QAtomicInt* cancelled); diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index dd8e485ec..2666b0961 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -22,6 +22,8 @@ extern "C" { #include +#include +#include #include #include #include @@ -48,8 +50,9 @@ extern "C" { namespace olive { FFmpegDecoder::FFmpegDecoder() : - scale_ctx_(nullptr), - scale_divider_(0), + filter_graph_(nullptr), + buffersrc_ctx_(nullptr), + buffersink_ctx_(nullptr), pool_(QThread::idealThreadCount()*2), is_working_(false), cache_at_zero_(false), @@ -148,28 +151,12 @@ bool FFmpegDecoder::OpenInternal() return output_frame; }*/ -FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const int ÷r) +FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams ¶ms) { - if (scale_divider_ != divider) { - FreeScaler(); - InitScaler(divider); - } - AVStream* s = instance_.avstream(); - int divided_width = VideoParams::GetScaledDimension(s->codecpar->width, divider); - int divided_height = VideoParams::GetScaledDimension(s->codecpar->height, divider); - - if (pool_.width() != divided_width || pool_.height() != divided_height) { - // Clear all instance queues - ClearFrameCache(); - - // Set new frame pool parameters - pool_.SetParameters(divided_width, divided_height, native_pix_fmt_, native_channel_count_); - } - // Retrieve frame - FFmpegFramePool::ElementPtr return_frame = RetrieveFrame(timecode, divider); + FFmpegFramePool::ElementPtr return_frame = RetrieveFrame(timecode, params); // We found the frame, we'll return a copy if (return_frame) { @@ -180,7 +167,7 @@ FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const in native_channel_count_, av_guess_sample_aspect_ratio(instance_.fmt_ctx(), s, nullptr), // May be incorrect, VideoParams::kInterlaceNone, // May be incorrect - divider)); + filter_params_.divider)); copy->set_timestamp(timecode); copy->allocate(); @@ -198,8 +185,42 @@ void FFmpegDecoder::CloseInternal() ClearFrameCache(); instance_.Close(); +} - FreeScaler(); +int FFmpegDecoder::GetFilteredFrame(AVPacket* packet, AVFrame* output_frame, const RetrieveVideoParams& params) +{ + // Ensure scaler is correct for these parameters + if (!InitScaler(params)) { + return AVERROR(EINVAL); + } + + int ret; + + AVFrame* working_frame = av_frame_alloc(); + + // Try to pull frame from buffersink + while ((ret = av_buffersink_get_frame(buffersink_ctx_, output_frame)) == AVERROR(EAGAIN)) { + // If no frame is ready in the buffersink, pull from codec + ret = instance_.GetFrame(packet, working_frame); + + if (ret >= 0) { + // If succeeded in pulling from codec, send to buffer source + ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, working_frame, AV_BUFFERSRC_FLAG_KEEP_REF); + + if (ret < 0) { + // If failed to send to buffer source, return break and error code + qDebug() << "Failed to feed filter graph:" << FFmpegError(ret); + break; + } + } else { + // If failed to read from decoder, return break and error code + break; + } + } + + av_frame_free(&working_frame); + + return ret; } QString FFmpegDecoder::id() const @@ -538,17 +559,6 @@ uint64_t FFmpegDecoder::ValidateChannelLayout(AVStream* stream) return av_get_default_channel_layout(stream->codecpar->channels); } -void FFmpegDecoder::FFmpegBufferToNativeBuffer(uint8_t **input_data, int *input_linesize, uint8_t** output_buffer, int* output_linesize) -{ - sws_scale(scale_ctx_, - input_data, - input_linesize, - 0, - instance_.avstream()->codecpar->height, - output_buffer, - output_linesize); -} - /* OLD UNUSED CODE: Keeping this around in case the code proves useful void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) @@ -611,9 +621,12 @@ void FFmpegDecoder::ClearFrameCache() cached_frames_.clear(); cache_at_eof_ = false; cache_at_zero_ = false; + + // Filter graph may rely on "continuous" video frames, so we free the scaler here + FreeScaler(); } -FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, int divider) +FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, const RetrieveVideoParams ¶ms) { int64_t target_ts = GetTimeInTimebaseUnits(time, instance_.avstream()->time_base, instance_.avstream()->start_time); int64_t seek_ts = target_ts; @@ -650,7 +663,7 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, i while (true) { // Pull from the decoder - ret = instance_.GetFrame(pkt, working_frame); + ret = GetFilteredFrame(pkt, working_frame, params); // Handle any errors that aren't EOF (EOF is handled later on) if (ret < 0 && ret != AVERROR_EOF) { @@ -707,8 +720,9 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, i // Store in queue, converting to native format uint8_t* destination_data = cached->data(); - int destination_linesize = Frame::generate_linesize_bytes(VideoParams::GetScaledDimension(instance_.avstream()->codecpar->width, divider), native_pix_fmt_, native_channel_count_); - FFmpegBufferToNativeBuffer(working_frame->data, working_frame->linesize, &destination_data, &destination_linesize); + int destination_linesize = Frame::generate_linesize_bytes(working_frame->width, native_pix_fmt_, native_channel_count_); + + av_image_copy(&destination_data, &destination_linesize, const_cast(working_frame->data), working_frame->linesize, static_cast(working_frame->format), working_frame->width, working_frame->height); // Set timestamp so this frame can be identified later cached->set_timestamp(working_frame->pts); @@ -746,81 +760,124 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, i return return_frame; } -void FFmpegDecoder::InitScaler(int divider) +bool FFmpegDecoder::InitScaler(const RetrieveVideoParams& params) { - int src_width = instance_.avstream()->codecpar->width; - int src_height = instance_.avstream()->codecpar->height; - - int scaled_width = VideoParams::GetScaledDimension(src_width, divider); - int scaled_height = VideoParams::GetScaledDimension(src_height, divider); - - scale_ctx_ = sws_getContext(src_width, - src_height, - static_cast(instance_.avstream()->codecpar->format), - scaled_width, - scaled_height, - ideal_pix_fmt_, - SWS_FAST_BILINEAR, - nullptr, - nullptr, - nullptr); - - if (scale_ctx_) { - scale_divider_ = divider; - } else { - scale_divider_ = 0; + if (params == filter_params_ && filter_graph_) { + // We have an appropriate filter for these parameters, just return true + return true; } + + // We need to (re)create the filter, delete current if necessary + ClearFrameCache(); + + // Set our params to this + filter_params_ = params; + + // Allocate filter graph + filter_graph_ = avfilter_graph_alloc(); + if (!filter_graph_) { + qWarning() << "Failed to allocate filter graph"; + return false; + } + + AVStream* s = instance_.avstream(); + + int src_width = s->codecpar->width; + int src_height = s->codecpar->height; + + // Define filter parameters + static const int kFilterArgSz = 1024; + char filter_args[kFilterArgSz]; + snprintf(filter_args, kFilterArgSz, "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d", + src_width, + src_height, + s->codecpar->format, + s->time_base.num, + s->time_base.den, + s->codecpar->sample_aspect_ratio.num, + s->codecpar->sample_aspect_ratio.den); + + // Create path in and out of the filter graph (the buffer in and the buffersink out) + avfilter_graph_create_filter(&buffersrc_ctx_, avfilter_get_by_name("buffer"), "in", filter_args, nullptr, filter_graph_); + avfilter_graph_create_filter(&buffersink_ctx_, avfilter_get_by_name("buffersink"), "out", nullptr, nullptr, filter_graph_); + + // Link filters as necessary + AVFilterContext *last_filter = buffersrc_ctx_; + + // Add interlacing filter if necessary + if (filter_params_.src_interlacing != filter_params_.dst_interlacing) { + // Determine what kind of interlacing we'll be doing + if (filter_params_.dst_interlacing == VideoParams::kInterlaceNone) { + // Deinterlace source + } else if (filter_params_.src_interlacing == VideoParams::kInterlaceNone) { + // We will be interlacing a previous progressive source + } else { + // We will simply be flipping the fields + } + } + + // Add scale filter if necessary + int dst_width, dst_height; + if (filter_params_.divider > 1) { + AVFilterContext* scale_filter; + + dst_width = VideoParams::GetScaledDimension(src_width, filter_params_.divider); + dst_height = VideoParams::GetScaledDimension(src_height, filter_params_.divider); + + snprintf(filter_args, kFilterArgSz, "w=%d:h=%d:flags=fast_bilinear:interl=%d", + dst_width, + dst_height, + params.dst_interlacing != VideoParams::kInterlaceNone); + + avfilter_graph_create_filter(&scale_filter, avfilter_get_by_name("scale"), "scale", filter_args, nullptr, filter_graph_); + + avfilter_link(last_filter, 0, scale_filter, 0); + last_filter = scale_filter; + } else { + dst_width = src_width; + dst_height = src_height; + } + + // Add format filter if necessary + if (ideal_pix_fmt_ != s->codecpar->format) { + AVFilterContext* format_filter; + + snprintf(filter_args, kFilterArgSz, "pix_fmts=%u", ideal_pix_fmt_); + + avfilter_graph_create_filter(&format_filter, avfilter_get_by_name("format"), "format", filter_args, nullptr, filter_graph_); + + avfilter_link(last_filter, 0, format_filter, 0); + last_filter = format_filter; + } + + // Finally, link the last filter with the buffersink + avfilter_link(last_filter, 0, buffersink_ctx_, 0); + + // Configure graph + if (int ret = avfilter_graph_config(filter_graph_, nullptr) < 0) { + qDebug() << "Failed to configure graph:" << FFmpegError(ret); + return false; + } + + // Configure frame pool + if (pool_.width() != dst_width || pool_.height() != dst_height) { + // Set new frame pool parameters + pool_.SetParameters(dst_width, dst_height, native_pix_fmt_, native_channel_count_); + } + + return true; } void FFmpegDecoder::FreeScaler() { - if (scale_ctx_) { - sws_freeContext(scale_ctx_); - scale_ctx_ = nullptr; - - scale_divider_ = 0; + if (filter_graph_) { + avfilter_graph_free(&filter_graph_); + filter_graph_ = nullptr; + buffersrc_ctx_ = nullptr; + buffersink_ctx_ = nullptr; } } -/*int64_t FFmpegDecoder::RangeStart() const -{ - if (cached_frames_.isEmpty()) { - return AV_NOPTS_VALUE; - } - return cached_frames_.first()->timestamp(); -} - -int64_t FFmpegDecoder::RangeEnd() const -{ - if (cached_frames_.isEmpty()) { - return AV_NOPTS_VALUE; - } - return cached_frames_.last()->timestamp(); -} - -bool FFmpegDecoder::CacheContainsTime(const int64_t &t) const -{ - return !cached_frames_.isEmpty() - && ((RangeStart() <= t && RangeEnd() >= t) - || (cache_at_zero_ && t < cached_frames_.first()->timestamp()) - || (cache_at_eof_ && t > cached_frames_.last()->timestamp())); -} - -bool FFmpegDecoder::CacheWillContainTime(const int64_t &t) const -{ - return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= cache_target_time_; -} - -bool FFmpegDecoder::CacheCouldContainTime(const int64_t &t) const -{ - return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= (cache_target_time_ + 2*second_ts_); -} - -bool FFmpegDecoder::CacheIsEmpty() const -{ - return cached_frames_.isEmpty(); -}*/ - FFmpegFramePool::ElementPtr FFmpegDecoder::GetFrameFromCache(const int64_t &t) const { if (t < cached_frames_.first()->timestamp()) { @@ -856,39 +913,6 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::GetFrameFromCache(const int64_t &t) c return nullptr; } -/*void FFmpegDecoder::RemoveFramesBefore(const qint64 &t) -{ - while (!cached_frames_.isEmpty() && cached_frames_.first()->last_accessed() < t) { - RemoveFirstFrame(); - } -} - -int FFmpegDecoder::TruncateCacheRangeToTime(const qint64 &t) -{ - int counter = 0; - - // We keep one frame in memory as an identifier for what pts the decoder is up to - while (cached_frames_.size() > 1 && (RangeEnd() - RangeStart()) > t) { - RemoveFirstFrame(); - counter++; - } - - return counter; -} - -int FFmpegDecoder::TruncateCacheRangeToFrames(int nb_frames) -{ - int counter = 0; - - // We keep one frame in memory as an identifier for what pts the decoder is up to - while (cached_frames_.size() > nb_frames) { - RemoveFirstFrame(); - counter++; - } - - return counter; -}*/ - void FFmpegDecoder::RemoveFirstFrame() { cached_frames_.removeFirst(); diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index a9fecacb9..e74f8514f 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -21,7 +21,11 @@ #ifndef FFMPEGDECODER_H #define FFMPEGDECODER_H +// Fixes weird define issue when including +#include + extern "C" { +#include #include #include #include @@ -60,7 +64,7 @@ public: protected: virtual bool OpenInternal() override; - virtual FramePtr RetrieveVideoInternal(const rational &timecode, const int& divider) override; + virtual FramePtr RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams& params) override; virtual bool ConformAudioInternal(const QString& filename, const AudioParams ¶ms, const QAtomicInt* cancelled) override; virtual void CloseInternal() override; @@ -108,6 +112,8 @@ private: }; + int GetFilteredFrame(AVPacket *packet, AVFrame *frame, const RetrieveVideoParams ¶ms); + /** * @brief Handle an FFmpeg error code * @@ -118,28 +124,26 @@ private: */ static QString FFmpegError(int error_code); - void InitScaler(int divider); + bool InitScaler(const RetrieveVideoParams ¶ms); void FreeScaler(); - //FramePtr RetrieveStillImage(const rational& timecode, const int& divider); - static VideoParams::Format GetNativePixelFormat(AVPixelFormat pix_fmt); static int GetNativeChannelCount(AVPixelFormat pix_fmt); static uint64_t ValidateChannelLayout(AVStream *stream); - void FFmpegBufferToNativeBuffer(uint8_t** input_data, int* input_linesize, uint8_t **output_buffer, int *output_linesize); - FFmpegFramePool::ElementPtr GetFrameFromCache(const int64_t& t) const; void ClearFrameCache(); - FFmpegFramePool::ElementPtr RetrieveFrame(const rational &time, int divider); + FFmpegFramePool::ElementPtr RetrieveFrame(const rational &time, const RetrieveVideoParams ¶ms); void RemoveFirstFrame(); - SwsContext* scale_ctx_; - int scale_divider_; + RetrieveVideoParams filter_params_; + AVFilterGraph* filter_graph_; + AVFilterContext* buffersrc_ctx_; + AVFilterContext* buffersink_ctx_; AVPixelFormat ideal_pix_fmt_; VideoParams::Format native_pix_fmt_; int native_channel_count_; diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index 8edcb80a8..16ff93fe4 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -106,7 +106,7 @@ bool OIIODecoder::OpenInternal() return OpenImageHandler(stream().filename()); } -FramePtr OIIODecoder::RetrieveVideoInternal(const rational &timecode, const int& divider) +FramePtr OIIODecoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams ÷r) { Q_UNUSED(timecode) @@ -118,10 +118,10 @@ FramePtr OIIODecoder::RetrieveVideoInternal(const rational &timecode, const int& channel_count_, OIIOUtils::GetPixelAspectRatioFromOIIO(buffer_->spec()), VideoParams::kInterlaceNone, // FIXME: Does OIIO deinterlace for us? - divider)); + divider.divider)); frame->allocate(); - if (divider == 1) { + if (divider.divider == 1) { OIIOUtils::BufferToFrame(buffer_, frame.get()); diff --git a/app/codec/oiio/oiiodecoder.h b/app/codec/oiio/oiiodecoder.h index e0155051f..30abd7b9e 100644 --- a/app/codec/oiio/oiiodecoder.h +++ b/app/codec/oiio/oiiodecoder.h @@ -44,7 +44,7 @@ public: protected: virtual bool OpenInternal() override; - virtual FramePtr RetrieveVideoInternal(const rational &timecode, const int& divider) override; + virtual FramePtr RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams& divider) override; virtual void CloseInternal() override; private: diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index f06c6c1d8..de9a29fef 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -361,7 +361,12 @@ QVariant RenderProcessor::ProcessVideoFootage(const FootageJob &stream, const ra } if (decoder) { - FramePtr frame = decoder->RetrieveVideo((stream_data.video_type() == VideoParams::kVideoTypeVideo) ? input_time : Decoder::kAnyTimecode, footage_divider); + Decoder::RetrieveVideoParams p; + p.divider = footage_divider; + p.src_interlacing = stream_data.interlacing(); + p.dst_interlacing = GetCacheVideoParams().interlacing(); + + FramePtr frame = decoder->RetrieveVideo((stream_data.video_type() == VideoParams::kVideoTypeVideo) ? input_time : Decoder::kAnyTimecode, p); if (frame) { // Return a texture from the derived class