diff --git a/app/codec/decoder.cpp b/app/codec/decoder.cpp index c793f55d0..af4b45d34 100644 --- a/app/codec/decoder.cpp +++ b/app/codec/decoder.cpp @@ -199,9 +199,17 @@ DecoderPtr Decoder::CreateFromID(const QString &id) return nullptr; } -int64_t Decoder::GetTimeInTimebaseUnits(const rational &time, const rational &timebase, int64_t start_time) +int64_t Decoder::GetTimeInTimebaseUnits(const rational &time, const rational &timebase, int64_t start_time, VideoParams::Interlacing interlacing) { - return Timecode::time_to_timestamp(time, timebase) + start_time; + int64_t t = Timecode::time_to_timestamp(time, timebase); + t += start_time; + return t; +} + +rational Decoder::GetTimestampInTimeUnits(int64_t time, const rational &timebase, int64_t start_time, VideoParams::Interlacing interlacing) +{ + time -= start_time; + return Timecode::timestamp_to_time(time, timebase); } void Decoder::SignalProcessingProgress(int64_t ts, int64_t duration) diff --git a/app/codec/decoder.h b/app/codec/decoder.h index 35410c0d1..c3b501cc1 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -295,7 +295,8 @@ protected: return stream_; } - static int64_t GetTimeInTimebaseUnits(const rational& time, const rational& timebase, int64_t start_time); + static int64_t GetTimeInTimebaseUnits(const rational& time, const rational& timebase, int64_t start_time, VideoParams::Interlacing interlacing); + static rational GetTimestampInTimeUnits(int64_t time, const rational& timebase, int64_t start_time, VideoParams::Interlacing interlacing); signals: /** diff --git a/app/codec/ffmpeg/CMakeLists.txt b/app/codec/ffmpeg/CMakeLists.txt index 70d978888..0401fba78 100644 --- a/app/codec/ffmpeg/CMakeLists.txt +++ b/app/codec/ffmpeg/CMakeLists.txt @@ -16,11 +16,9 @@ set(OLIVE_SOURCES ${OLIVE_SOURCES} - codec/ffmpeg/ffmpegdecoder.h codec/ffmpeg/ffmpegdecoder.cpp - codec/ffmpeg/ffmpegencoder.h + codec/ffmpeg/ffmpegdecoder.h codec/ffmpeg/ffmpegencoder.cpp - codec/ffmpeg/ffmpegframepool.h - codec/ffmpeg/ffmpegframepool.cpp + codec/ffmpeg/ffmpegencoder.h PARENT_SCOPE ) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 1da0881ef..3abb9a86a 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -54,7 +54,8 @@ FFmpegDecoder::FFmpegDecoder() : filter_graph_(nullptr), buffersrc_ctx_(nullptr), buffersink_ctx_(nullptr), - pool_(QThread::idealThreadCount()*2), + working_frame_(nullptr), + working_packet_(nullptr), is_working_(false), cache_at_zero_(false), cache_at_eof_(false) @@ -80,11 +81,14 @@ bool FFmpegDecoder::OpenInternal() if (native_pix_fmt_ == VideoParams::kFormatInvalid || native_channel_count_ == 0) { - qDebug() << "Failed to find valid native pixel format for" << ideal_pix_fmt_; + qCritical() << "Failed to find valid native pixel format for" << ideal_pix_fmt_; return false; } } + working_frame_ = av_frame_alloc(); + working_packet_ = av_packet_alloc(); + return true; } @@ -153,35 +157,21 @@ FramePtr FFmpegDecoder::RetrieveVideoInternal(const rational &timecode, const Re return nullptr; } - AVStream* s = instance_.avstream(); - - // Retrieve frame - FFmpegFramePool::ElementPtr return_frame = RetrieveFrame(timecode, cancelled); - - // We found the frame, we'll return a copy - if (return_frame) { - FramePtr copy = Frame::Create(); - copy->set_video_params(VideoParams(s->codecpar->width, - s->codecpar->height, - native_pix_fmt_, - native_channel_count_, - av_guess_sample_aspect_ratio(instance_.fmt_ctx(), s, nullptr), // May be incorrect, - VideoParams::kInterlaceNone, - filter_params_.divider)); - copy->set_timestamp(timecode); - copy->allocate(); - - // This data will already match the frame - memcpy(copy->data(), return_frame->data(), copy->allocated_size()); - - return copy; - } - - return nullptr; + return RetrieveFrame(timecode, cancelled); } void FFmpegDecoder::CloseInternal() { + if (working_packet_) { + av_packet_free(&working_packet_); + working_packet_ = nullptr; + } + + if (working_frame_) { + av_frame_free(&working_frame_); + working_frame_ = nullptr; + } + ClearFrameCache(); instance_.Close(); @@ -192,35 +182,35 @@ int FFmpegDecoder::GetFilteredFrame(AVPacket* packet, AVFrame* output_frame) // Ensure scaler is correct for these parameters int ret; - AVFrame* working_frame = av_frame_alloc(); - // Try to pull frame from buffersink while ((ret = av_buffersink_get_frame(buffersink_ctx_, output_frame)) == AVERROR(EAGAIN)) { // If no frame is ready in the buffersink, pull from codec - ret = instance_.GetFrame(packet, working_frame); + ret = instance_.GetFrame(packet, output_frame); if (ret >= 0) { // Override this frame's interlacing parameters from user switch (filter_params_.src_interlacing) { case VideoParams::kInterlaceNone: - working_frame->interlaced_frame = 0; + output_frame->interlaced_frame = 0; break; case VideoParams::kInterlacedTopFirst: - working_frame->interlaced_frame = 1; - working_frame->top_field_first = 1; + output_frame->interlaced_frame = 1; + output_frame->top_field_first = 1; break; case VideoParams::kInterlacedBottomFirst: - working_frame->interlaced_frame = 1; - working_frame->top_field_first = 0; + output_frame->interlaced_frame = 1; + output_frame->top_field_first = 0; break; } // If succeeded in pulling from codec, send to buffer source - ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, working_frame, AV_BUFFERSRC_FLAG_KEEP_REF); + ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, output_frame, AV_BUFFERSRC_FLAG_KEEP_REF); + + av_frame_unref(output_frame); if (ret < 0) { // If failed to send to buffer source, return break and error code - qDebug() << "Failed to feed filter graph:" << FFmpegError(ret); + qCritical() << "Failed to feed filter graph:" << FFmpegError(ret); break; } } else { @@ -229,8 +219,6 @@ int FFmpegDecoder::GetFilteredFrame(AVPacket* packet, AVFrame* output_frame) } } - av_frame_free(&working_frame); - return ret; } @@ -685,7 +673,7 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) void FFmpegDecoder::ClearFrameCache() { - if (!cached_frames_.isEmpty()) { + if (!cached_frames_.empty()) { cached_frames_.clear(); cache_at_eof_ = false; cache_at_zero_ = false; @@ -696,24 +684,18 @@ void FFmpegDecoder::ClearFrameCache() } } -FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, const QAtomicInt *cancelled) +FramePtr FFmpegDecoder::RetrieveFrame(const rational& time, const QAtomicInt *cancelled) { - int64_t target_ts = GetTimeInTimebaseUnits(time, instance_.avstream()->time_base, instance_.avstream()->start_time); + int64_t target_ts = GetTimeInTimebaseUnits(time, instance_.avstream()->time_base, instance_.avstream()->start_time, filter_params_.src_interlacing); const int64_t min_seek = -instance_.avstream()->start_time; int64_t seek_ts = target_ts; bool still_seeking = false; - if (filter_params_.src_interlacing != VideoParams::kInterlaceNone) { - // If we are de-interlacing, the timebase is doubled because we get one frame per field, so we - // double the target timestamp too - target_ts *= 2; - } - if (time != kAnyTimecode) { // If the frame wasn't in the frame cache, see if this frame cache is too old to use - if (cached_frames_.isEmpty() - || (target_ts < cached_frames_.first()->timestamp() || target_ts > cached_frames_.last()->timestamp() + 2*second_ts_)) { + if (cached_frames_.empty() + || (time < cached_frames_.front()->timestamp() || time > cached_frames_.back()->timestamp() + 2)) { ClearFrameCache(); instance_.Seek(seek_ts); @@ -724,7 +706,7 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, c still_seeking = true; } else { // Search cache for frame - FFmpegFramePool::ElementPtr cached_frame = GetFrameFromCache(target_ts); + FramePtr cached_frame = GetFrameFromCache(time); if (cached_frame) { return cached_frame; } @@ -732,11 +714,7 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, c } int ret; - AVPacket* pkt = av_packet_alloc(); - FFmpegFramePool::ElementPtr return_frame = nullptr; - - // Allocate a new frame - AVFrame* working_frame = av_frame_alloc(); + FramePtr return_frame = nullptr; while (true) { // Break out of loop if we've cancelled @@ -745,8 +723,8 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, c } // Pull from the decoder - av_frame_unref(working_frame); - ret = GetFilteredFrame(pkt, working_frame); + av_frame_unref(working_frame_); + ret = GetFilteredFrame(working_packet_, working_frame_); // Handle any errors that aren't EOF (EOF is handled later on) if (ret < 0 && ret != AVERROR_EOF) { @@ -757,7 +735,7 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, c if (still_seeking) { // Handle a failure to seek (occurs on some media) // We'll only be here if the frame cache was emptied earlier - if (!cache_at_zero_ && (ret == AVERROR_EOF || working_frame->best_effort_timestamp > target_ts)) { + if (!cache_at_zero_ && (ret == AVERROR_EOF || working_frame_->best_effort_timestamp > target_ts)) { seek_ts = qMax(min_seek, seek_ts - second_ts_); instance_.Seek(seek_ts); @@ -779,10 +757,10 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, c // Handle an "expected" EOF by using the last frame of our cache cache_at_eof_ = true; - if (cached_frames_.isEmpty()) { + if (cached_frames_.empty()) { qCritical() << "Unexpected codec EOF - unable to retrieve frame"; } else { - return_frame = cached_frames_.last(); + return_frame = cached_frames_.back(); } break; @@ -790,42 +768,39 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, c } else { // Cut down to thread count - 1 before we acquire a new frame - if (cached_frames_.size() == QThread::idealThreadCount()) { + if (cached_frames_.size() == size_t(QThread::idealThreadCount())) { RemoveFirstFrame(); } - FFmpegFramePool::ElementPtr cached = pool_.Get(); - - if (!cached) { - qCritical() << "Frame pool failed to return a valid frame - out of memory?"; - break; - } + FramePtr cached = Frame::Create(); + cached->set_video_params(GetVideoParams()); + cached->allocate(); // Store in queue, converting to native format - uint8_t* destination_data = cached->data(); - int destination_linesize = Frame::generate_linesize_bytes(working_frame->width, native_pix_fmt_, native_channel_count_); + uint8_t* destination_data = reinterpret_cast(cached->data()); + int destination_linesize = cached->linesize_bytes(); - av_image_copy(&destination_data, &destination_linesize, const_cast(working_frame->data), working_frame->linesize, static_cast(working_frame->format), working_frame->width, working_frame->height); + av_image_copy(&destination_data, &destination_linesize, const_cast(working_frame_->data), working_frame_->linesize, static_cast(working_frame_->format), working_frame_->width, working_frame_->height); // Set timestamp so this frame can be identified later - cached->set_timestamp(working_frame->best_effort_timestamp); + cached->set_timestamp(GetTimestampInTimeUnits(working_frame_->best_effort_timestamp, instance_.avstream()->time_base, instance_.avstream()->start_time, filter_params_.src_interlacing)); // Store frame before just in case - FFmpegFramePool::ElementPtr previous; - if (cached_frames_.isEmpty()) { + FramePtr previous; + if (cached_frames_.empty()) { previous = nullptr; } else { - previous = cached_frames_.last(); + previous = cached_frames_.back(); } // Append this frame and signal to other threads that a new frame has arrived - cached_frames_.append(cached); + cached_frames_.push_back(cached); // If this is a valid frame, see if this or the frame before it are the one we need - if (cached->timestamp() == target_ts || time == kAnyTimecode) { + if (cached->timestamp() == time || time == kAnyTimecode) { return_frame = cached; break; - } else if (cached->timestamp() > target_ts) { + } else if (cached->timestamp() > time) { if (!previous && cache_at_zero_) { return_frame = cached; break; @@ -837,8 +812,8 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::RetrieveFrame(const rational& time, c } } - av_frame_free(&working_frame); - av_packet_free(&pkt); + av_frame_unref(working_frame_); + av_packet_unref(working_packet_); return return_frame; } @@ -939,16 +914,10 @@ bool FFmpegDecoder::InitScaler(const RetrieveVideoParams& params) // Configure graph if (int ret = avfilter_graph_config(filter_graph_, nullptr) < 0) { - qDebug() << "Failed to configure graph:" << FFmpegError(ret); + qCritical() << "Failed to configure graph:" << FFmpegError(ret); return false; } - // Configure frame pool - if (pool_.width() != dst_width || pool_.height() != dst_height) { - // Set new frame pool parameters - pool_.SetParameters(dst_width, dst_height, native_pix_fmt_, native_channel_count_); - } - return true; } @@ -962,32 +931,32 @@ void FFmpegDecoder::FreeScaler() } } -FFmpegFramePool::ElementPtr FFmpegDecoder::GetFrameFromCache(const int64_t &t) const +FramePtr FFmpegDecoder::GetFrameFromCache(const rational &t) const { - if (t < cached_frames_.first()->timestamp()) { + if (t < cached_frames_.front()->timestamp()) { if (cache_at_zero_) { - cached_frames_.first()->access(); - return cached_frames_.first(); + return cached_frames_.front(); } - } else if (t > cached_frames_.last()->timestamp()) { + } else if (t > cached_frames_.back()->timestamp()) { if (cache_at_eof_) { - cached_frames_.last()->access(); - return cached_frames_.last(); + return cached_frames_.back(); } } else { // We already have this frame in the cache, find it - for (int i=0;itimestamp() == t // Test for an exact match - || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->timestamp() > t)) { // Or for this frame to be the "closest" + || (next != cached_frames_.cend() && (*next)->timestamp() > t)) { // Or for this frame to be the "closest" - this_frame->access(); return this_frame; } @@ -999,10 +968,21 @@ FFmpegFramePool::ElementPtr FFmpegDecoder::GetFrameFromCache(const int64_t &t) c void FFmpegDecoder::RemoveFirstFrame() { - cached_frames_.removeFirst(); + cached_frames_.pop_front(); cache_at_zero_ = false; } +VideoParams FFmpegDecoder::GetVideoParams() const +{ + return VideoParams(instance_.avstream()->codecpar->width, + instance_.avstream()->codecpar->height, + native_pix_fmt_, + native_channel_count_, + av_guess_sample_aspect_ratio(instance_.fmt_ctx(), instance_.avstream(), nullptr), + VideoParams::kInterlaceNone, + filter_params_.divider); +} + FFmpegDecoder::Instance::Instance() : fmt_ctx_(nullptr), codec_ctx_(nullptr), diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 3d8c56f97..2f377af5d 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -37,7 +37,6 @@ extern "C" { #include #include "codec/decoder.h" -#include "ffmpegframepool.h" namespace olive { @@ -139,14 +138,16 @@ private: static const char* GetInterlacingModeInFFmpeg(VideoParams::Interlacing interlacing); - FFmpegFramePool::ElementPtr GetFrameFromCache(const int64_t& t) const; + FramePtr GetFrameFromCache(const rational &t) const; void ClearFrameCache(); - FFmpegFramePool::ElementPtr RetrieveFrame(const rational &time, const QAtomicInt *cancelled); + FramePtr RetrieveFrame(const rational &time, const QAtomicInt *cancelled); void RemoveFirstFrame(); + VideoParams GetVideoParams() const; + RetrieveVideoParams filter_params_; AVFilterGraph* filter_graph_; AVFilterContext* buffersrc_ctx_; @@ -155,11 +156,12 @@ private: VideoParams::Format native_pix_fmt_; int native_channel_count_; - FFmpegFramePool pool_; + AVFrame *working_frame_; + AVPacket *working_packet_; int64_t second_ts_; - QList cached_frames_; + std::list cached_frames_; bool is_working_; QMutex is_working_mutex_; diff --git a/app/codec/ffmpeg/ffmpegframepool.cpp b/app/codec/ffmpeg/ffmpegframepool.cpp deleted file mode 100644 index 1337b14e7..000000000 --- a/app/codec/ffmpeg/ffmpegframepool.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2022 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "ffmpegframepool.h" - -#include "codec/frame.h" - -namespace olive { - -FFmpegFramePool::FFmpegFramePool(int element_count) : - MemoryPool(element_count), - width_(0), - height_(0), - format_(VideoParams::kFormatInvalid), - channel_count_(0) -{ -} - -void FFmpegFramePool::SetParameters(int width, int height, VideoParams::Format format, int channel_count) -{ - Clear(); - - width_ = width; - height_ = height; - format_ = format; - channel_count_ = channel_count; -} - -size_t FFmpegFramePool::GetElementSize() -{ - return Frame::generate_linesize_bytes(width_, format_, channel_count_) * height_; -} - -} diff --git a/app/codec/ffmpeg/ffmpegframepool.h b/app/codec/ffmpeg/ffmpegframepool.h deleted file mode 100644 index 8107ec324..000000000 --- a/app/codec/ffmpeg/ffmpegframepool.h +++ /dev/null @@ -1,63 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2022 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef FFMPEGFRAMEPOOL_H -#define FFMPEGFRAMEPOOL_H - -#include "common/memorypool.h" -#include "render/videoparams.h" - -namespace olive { - -class FFmpegFramePool : public MemoryPool -{ - Q_OBJECT -public: - FFmpegFramePool(int element_count); - - void SetParameters(int width, int height, VideoParams::Format format, int channel_count); - - const int& width() const - { - return width_; - } - - const int& height() const - { - return height_; - } - -protected: - virtual size_t GetElementSize() override; - -private: - int width_; - - int height_; - - VideoParams::Format format_; - - int channel_count_; - -}; - -} - -#endif // FFMPEGFRAMEPOOL_H diff --git a/app/common/ffmpegutils.cpp b/app/common/ffmpegutils.cpp index d05252bb9..411f0a115 100644 --- a/app/common/ffmpegutils.cpp +++ b/app/common/ffmpegutils.cpp @@ -25,9 +25,9 @@ namespace olive { AVPixelFormat FFmpegUtils::GetCompatiblePixelFormat(const AVPixelFormat &pix_fmt) { AVPixelFormat possible_pix_fmts[] = { - AV_PIX_FMT_RGB24, + // RGBA formats only because GPUs always upconvert to RGBA, so if it's RGB, that adds extra + // conversion overhead AV_PIX_FMT_RGBA, - AV_PIX_FMT_RGB48, AV_PIX_FMT_RGBA64, AV_PIX_FMT_NONE }; diff --git a/app/node/traverser.cpp b/app/node/traverser.cpp index 75ff4b3a8..a7738ba20 100644 --- a/app/node/traverser.cpp +++ b/app/node/traverser.cpp @@ -260,8 +260,23 @@ NodeTraverser::NodeTraverser() : { } +class GTTTime +{ +public: + GTTTime(const Node *n) { t = QDateTime::currentMSecsSinceEpoch(); node = n; } + + ~GTTTime() { qDebug() << "GT for" << node << "took" << (QDateTime::currentMSecsSinceEpoch() - t); } + + qint64 t; + const Node *node; + +}; + NodeValueTable NodeTraverser::GenerateTable(const Node *n, const TimeRange& range) { + // NOTE: Times how long a node takes to process, useful for profiling. + //GTTTime gtt(n);Q_UNUSED(gtt); + const Track* track = dynamic_cast(n); if (track) { // If the range is not wholly contained in this Block, we'll need to do some extra processing @@ -297,7 +312,6 @@ NodeValueTable NodeTraverser::GenerateTable(const Node *n, const TimeRange& rang if (!transform_ignore_.contains(n)) { QTransform t = n->GizmoTransformation(row, globals); if (!t.isIdentity()) { - qDebug() << "transforming" << n; (*transform_) *= t; } diff --git a/app/render/rendermanager.cpp b/app/render/rendermanager.cpp index e779820ff..772048134 100644 --- a/app/render/rendermanager.cpp +++ b/app/render/rendermanager.cpp @@ -56,10 +56,6 @@ RenderManager::RenderManager(QObject *parent) : decoder_cache_ = new DecoderCache(); shader_cache_ = new ShaderCache(); default_shader_ = context_->CreateNativeShader(ShaderCode(QString(), QString())); - - decoder_clear_timer_.setInterval(kDecoderMaximumInactivity); - connect(&decoder_clear_timer_, &QTimer::timeout, this, &RenderManager::ClearOldDecoders); - decoder_clear_timer_.start(); } else { qCritical() << "Tried to initialize unknown graphics backend"; context_ = nullptr; @@ -81,24 +77,6 @@ RenderManager::~RenderManager() } } -void RenderManager::ClearOldDecoders() -{ - QMutexLocker locker(decoder_cache_->mutex()); - - qint64 min_age = QDateTime::currentMSecsSinceEpoch() - kDecoderMaximumInactivity; - - for (auto it=decoder_cache_->begin(); it!=decoder_cache_->end(); ) { - DecoderPair decoder = it.value(); - - if (decoder.decoder->GetLastAccessedTime() < min_age) { - decoder.decoder->Close(); - it = decoder_cache_->erase(it); - } else { - it++; - } - } -} - QByteArray RenderManager::Hash(const Node *n, const Node::ValueHint &output, const VideoParams ¶ms, const rational &time) { Q_ASSERT(n); diff --git a/app/render/rendermanager.h b/app/render/rendermanager.h index 39b52fc9f..c5d4fac12 100644 --- a/app/render/rendermanager.h +++ b/app/render/rendermanager.h @@ -142,13 +142,6 @@ private: QVariant default_shader_; - QTimer decoder_clear_timer_; - - static constexpr auto kDecoderMaximumInactivity = 10000; - -private slots: - void ClearOldDecoders(); - }; }