From b0530cd55d8ef7cbfe9d0ea353ebfa229366fdbd Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Thu, 19 May 2022 16:54:18 -0700 Subject: [PATCH] decoders: upload directly to texture --- app/codec/decoder.cpp | 14 ++++----- app/codec/decoder.h | 9 ++++-- app/codec/ffmpeg/ffmpegdecoder.cpp | 30 +++++++----------- app/codec/ffmpeg/ffmpegdecoder.h | 6 ++-- app/codec/oiio/oiiodecoder.cpp | 36 +++++++++++----------- app/codec/oiio/oiiodecoder.h | 4 ++- app/render/renderprocessor.cpp | 49 +++++++++++++++--------------- 7 files changed, 74 insertions(+), 74 deletions(-) diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index af4b45d34..d63c3bc45 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -86,7 +86,7 @@ bool Decoder::Open(const CodecStream &stream) } } -FramePtr Decoder::RetrieveVideo(const rational &timecode, const RetrieveVideoParams ÷r, const QAtomicInt *cancelled) +bool Decoder::RetrieveVideo(TexturePtr destination, const rational &timecode, const RetrieveVideoParams ÷r, const QAtomicInt *cancelled) { QMutexLocker locker(&mutex_); @@ -94,19 +94,19 @@ FramePtr Decoder::RetrieveVideo(const rational &timecode, const RetrieveVideoPar if (!stream_.IsValid()) { qCritical() << "Can't retrieve video on a closed decoder"; - return nullptr; + return false; } if (!SupportsVideo()) { qCritical() << "Decoder doesn't support video"; - return nullptr; + return false; } if (cancelled && *cancelled) { - return nullptr; + return false; } - return RetrieveVideoInternal(timecode, divider, cancelled); + return RetrieveVideoInternal(destination, timecode, divider, cancelled); } Decoder::RetrieveAudioStatus Decoder::RetrieveAudio(SampleBuffer &dest, const TimeRange &range, const AudioParams ¶ms, const QString& cache_path, Footage::LoopMode loop_mode, RenderMode::Mode mode) @@ -264,12 +264,12 @@ int64_t Decoder::GetImageSequenceIndex(const QString &filename) return number_only.toLongLong(); } -FramePtr Decoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams ÷r, const QAtomicInt *cancelled) +bool Decoder::RetrieveVideoInternal(TexturePtr destination, const rational &timecode, const RetrieveVideoParams ÷r, const QAtomicInt *cancelled) { Q_UNUSED(timecode) Q_UNUSED(divider) Q_UNUSED(cancelled) - return nullptr; + return false; } bool Decoder::ConformAudioInternal(const QVector &filenames, const AudioParams ¶ms, const QAtomicInt* cancelled) diff --git a/app/codec/decoder.h b/app/codec/decoder.h index c3b501cc1..80bf9fd48 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -184,7 +184,12 @@ public: * * This function is thread safe and can only run while the decoder is open. \see Open() */ - FramePtr RetrieveVideo(const rational& timecode, const RetrieveVideoParams& divider, const QAtomicInt *cancelled = nullptr); + bool RetrieveVideo(TexturePtr destination, const rational& timecode, const RetrieveVideoParams& divider, const QAtomicInt *cancelled = nullptr); + + virtual VideoParams GetParamsForTexture(const Decoder::RetrieveVideoParams &p) const + { + return VideoParams(); + } enum RetrieveAudioStatus { kInvalid = -1, @@ -279,7 +284,7 @@ protected: * Sub-classes must override this function IF they support video. Function is already mutexed * so sub-classes don't need to worry about thread safety. */ - virtual FramePtr RetrieveVideoInternal(const rational& timecode, const RetrieveVideoParams& divider, const QAtomicInt *cancelled); + virtual bool RetrieveVideoInternal(TexturePtr destination, const rational& timecode, const RetrieveVideoParams& divider, const QAtomicInt *cancelled); virtual bool ConformAudioInternal(const QVector& filenames, const AudioParams ¶ms, const QAtomicInt* cancelled); diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 5698caf0a..0e47a8cd1 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -151,38 +151,30 @@ bool FFmpegDecoder::OpenInternal() return output_frame; }*/ -FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams ¶ms, const QAtomicInt *cancelled) +bool FFmpegDecoder::RetrieveVideoInternal(TexturePtr destination, const rational &timecode, const RetrieveVideoParams ¶ms, const QAtomicInt *cancelled) { - FramePtr ret = nullptr; - if (AVFramePtr f = RetrieveFrame(timecode, cancelled)) { if (InitScaler(params)) { - qint64 t = QDateTime::currentMSecsSinceEpoch(); - int r; r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF); if (r < 0) { - return nullptr; + return false; } - r = av_buffersink_get_frame(buffersink_ctx_, f.get()); + r = av_buffersink_get_frame(buffersink_ctx_, working_frame_); if (r < 0) { - return nullptr; + return false; } - ret = Frame::Create(); - ret->set_video_params(GetVideoParams()); - ret->allocate(); + VideoParams vp = GetParamsForTexture(params); - uint8_t *destination_data = reinterpret_cast(ret->data()); - int destination_linesize = ret->linesize_bytes(); - av_image_copy(&destination_data, &destination_linesize, const_cast(f->data), f->linesize, AVPixelFormat(f->format), f->width, f->height); + destination->Upload(working_frame_->data[0], working_frame_->linesize[0] / vp.GetBytesPerPixel()); - //sws_scale(sws_ctx_, f->data, f->linesize, 0, f->height, &destination_data, &destination_linesize); - qDebug() << "Converting from" << instance_.avstream()->codecpar->format << "to" << ideal_pix_fmt_ << "took" << (QDateTime::currentMSecsSinceEpoch() - t); + av_frame_unref(working_frame_); + return true; } } - return ret; + return false; } void FFmpegDecoder::CloseInternal() @@ -944,7 +936,7 @@ void FFmpegDecoder::RemoveFirstFrame() cache_at_zero_ = false; } -VideoParams FFmpegDecoder::GetVideoParams() const +VideoParams FFmpegDecoder::GetParamsForTexture(const Decoder::RetrieveVideoParams &p) const { return VideoParams(instance_.avstream()->codecpar->width, instance_.avstream()->codecpar->height, @@ -952,7 +944,7 @@ VideoParams FFmpegDecoder::GetVideoParams() const native_channel_count_, av_guess_sample_aspect_ratio(instance_.fmt_ctx(), instance_.avstream(), nullptr), VideoParams::kInterlaceNone, - filter_params_.divider); + p.divider); } FFmpegDecoder::Instance::Instance() : diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index b424692cc..ac945ae02 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -66,9 +66,11 @@ public: virtual FootageDescription Probe(const QString &filename, const QAtomicInt *cancelled) const override; + virtual VideoParams GetParamsForTexture(const Decoder::RetrieveVideoParams &p) const override; + protected: virtual bool OpenInternal() override; - virtual FramePtr RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams& params, const QAtomicInt *cancelled) override; + virtual bool RetrieveVideoInternal(TexturePtr destination, const rational& timecode, const RetrieveVideoParams& params, const QAtomicInt *cancelled) override; virtual bool ConformAudioInternal(const QVector& filenames, const AudioParams ¶ms, const QAtomicInt* cancelled) override; virtual void CloseInternal() override; @@ -150,8 +152,6 @@ private: void RemoveFirstFrame(); - VideoParams GetVideoParams() const; - RetrieveVideoParams filter_params_; AVFilterGraph* filter_graph_; AVFilterContext* buffersrc_ctx_; diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index 08264ac56..b9b370204 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -117,48 +117,48 @@ FootageDescription OIIODecoder::Probe(const QString &filename, const QAtomicInt* return desc; } +VideoParams OIIODecoder::GetParamsForTexture(const RetrieveVideoParams &p) const +{ + return VideoParams(buffer_->spec().width, + buffer_->spec().height, + pix_fmt_, + channel_count_, + OIIOUtils::GetPixelAspectRatioFromOIIO(buffer_->spec()), + VideoParams::kInterlaceNone, // FIXME: Does OIIO deinterlace for us? + p.divider); +} + bool OIIODecoder::OpenInternal() { // If we can open the filename provided, assume everything is working return OpenImageHandler(stream().filename(), stream().stream()); } -FramePtr OIIODecoder::RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams ÷r, const QAtomicInt *cancelled) +bool OIIODecoder::RetrieveVideoInternal(TexturePtr destination, const rational &timecode, const RetrieveVideoParams ¶ms, const QAtomicInt *cancelled) { Q_UNUSED(timecode) Q_UNUSED(cancelled) - FramePtr frame = Frame::Create(); + VideoParams vp = GetParamsForTexture(params); - VideoParams vp(buffer_->spec().width, - buffer_->spec().height, - pix_fmt_, - channel_count_, - OIIOUtils::GetPixelAspectRatioFromOIIO(buffer_->spec()), - VideoParams::kInterlaceNone, // FIXME: Does OIIO deinterlace for us? - divider.divider); + if (params.divider == 1) { - frame->set_video_params(vp); - frame->allocate(); - - if (divider.divider == 1) { - - OIIOUtils::BufferToFrame(buffer_, frame.get()); + destination->Upload(buffer_->localpixels(), buffer_->scanline_stride() / vp.GetBytesPerPixel()); } else { // Will need to resize the image - OIIO::ImageBuf dst(OIIO::ImageSpec(frame->width(), frame->height(), buffer_->spec().nchannels, buffer_->spec().format)); + OIIO::ImageBuf dst(OIIO::ImageSpec(vp.effective_width(), vp.effective_height(), buffer_->spec().nchannels, buffer_->spec().format)); if (!OIIO::ImageBufAlgo::resample(dst, *buffer_)) { qWarning() << "OIIO resize failed"; } - OIIOUtils::BufferToFrame(&dst, frame.get()); + destination->Upload(dst.localpixels(), dst.scanline_stride() / vp.GetBytesPerPixel()); } - return frame; + return true; } void OIIODecoder::CloseInternal() diff --git a/app/codec/oiio/oiiodecoder.h b/app/codec/oiio/oiiodecoder.h index fd9270e8f..d22a186e4 100644 --- a/app/codec/oiio/oiiodecoder.h +++ b/app/codec/oiio/oiiodecoder.h @@ -42,9 +42,11 @@ public: virtual FootageDescription Probe(const QString& filename, const QAtomicInt* cancelled) const override; + virtual VideoParams GetParamsForTexture(const Decoder::RetrieveVideoParams &p) const override; + protected: virtual bool OpenInternal() override; - virtual FramePtr RetrieveVideoInternal(const rational &timecode, const RetrieveVideoParams& divider, const QAtomicInt *cancelled) override; + virtual bool RetrieveVideoInternal(TexturePtr destination, const rational& timecode, const RetrieveVideoParams& params, const QAtomicInt *cancelled) override; virtual void CloseInternal() override; private: diff --git a/app/render/renderprocessor.cpp b/app/render/renderprocessor.cpp index e9ef9be61..0b2d469b8 100644 --- a/app/render/renderprocessor.cpp +++ b/app/render/renderprocessor.cpp @@ -46,7 +46,6 @@ RenderProcessor::RenderProcessor(RenderTicketPtr ticket, Renderer *render_ctx, D TexturePtr RenderProcessor::GenerateTexture(const rational &time, const rational &frame_length) { - TimeRange range = TimeRange(time, time + frame_length); NodeValueTable table; @@ -451,35 +450,37 @@ void RenderProcessor::ProcessVideoFootage(TexturePtr destination, const FootageJ p.dst_interlacing = GetCacheVideoParams().interlacing(); if (!IsCancelled()) { - FramePtr frame = decoder->RetrieveVideo((stream_data.video_type() == VideoParams::kVideoTypeVideo) ? input_time : Decoder::kAnyTimecode, p, GetCancelPointer()); - if (frame) { - // Return a texture from the derived class - TexturePtr unmanaged_texture = render_ctx_->CreateTexture(frame->video_params(), - frame->data(), - frame->linesize_pixels()); + VideoParams tex_params = decoder->GetParamsForTexture(p); - // We convert to our rendering pixel format, since that will always be float-based which - // is necessary for correct color conversion - ColorProcessorPtr processor = ColorProcessor::Create(color_manager, - using_colorspace, - color_manager->GetReferenceColorSpace()); + if (tex_params.is_valid()) { + TexturePtr unmanaged_texture = render_ctx_->CreateTexture(tex_params); - ColorTransformJob job; + bool frame = decoder->RetrieveVideo(unmanaged_texture, (stream_data.video_type() == VideoParams::kVideoTypeVideo) ? input_time : Decoder::kAnyTimecode, p, GetCancelPointer()); - job.SetColorProcessor(processor); - job.SetInputTexture(unmanaged_texture); + if (frame) { + // We convert to our rendering pixel format, since that will always be float-based which + // is necessary for correct color conversion + ColorProcessorPtr processor = ColorProcessor::Create(color_manager, + using_colorspace, + color_manager->GetReferenceColorSpace()); - if (stream_data.channel_count() != VideoParams::kRGBAChannelCount - || stream_data.colorspace() == color_manager->GetReferenceColorSpace()) { - job.SetInputAlphaAssociation(kAlphaNone); - } else if (stream_data.premultiplied_alpha()) { - job.SetInputAlphaAssociation(kAlphaAssociated); - } else { - job.SetInputAlphaAssociation(kAlphaUnassociated); + ColorTransformJob job; + + job.SetColorProcessor(processor); + job.SetInputTexture(unmanaged_texture); + + if (stream_data.channel_count() != VideoParams::kRGBAChannelCount + || stream_data.colorspace() == color_manager->GetReferenceColorSpace()) { + job.SetInputAlphaAssociation(kAlphaNone); + } else if (stream_data.premultiplied_alpha()) { + job.SetInputAlphaAssociation(kAlphaAssociated); + } else { + job.SetInputAlphaAssociation(kAlphaUnassociated); + } + + render_ctx_->BlitColorManaged(job, destination.get()); } - - render_ctx_->BlitColorManaged(job, destination.get()); } } }