From 0c3ce03348d2a2c65616c427524a28b8c8bbf6d5 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 11 Apr 2020 13:18:31 +1000 Subject: [PATCH 01/14] renderer: use a dedicated decoder per thread --- app/render/backend/audio/audiobackend.cpp | 2 +- app/render/backend/audio/audioworker.cpp | 4 ++-- app/render/backend/audio/audioworker.h | 2 +- app/render/backend/audiorenderworker.cpp | 4 ++-- app/render/backend/audiorenderworker.h | 2 +- app/render/backend/opengl/openglbackend.cpp | 2 +- app/render/backend/opengl/openglworker.cpp | 4 ++-- app/render/backend/opengl/openglworker.h | 2 +- app/render/backend/renderbackend.cpp | 7 ------- app/render/backend/renderbackend.h | 4 ---- app/render/backend/renderworker.cpp | 13 ++++++------- app/render/backend/renderworker.h | 4 ++-- app/render/backend/videorenderworker.cpp | 4 ++-- app/render/backend/videorenderworker.h | 2 +- 14 files changed, 22 insertions(+), 34 deletions(-) diff --git a/app/render/backend/audio/audiobackend.cpp b/app/render/backend/audio/audiobackend.cpp index 92705500e..60923cf89 100644 --- a/app/render/backend/audio/audiobackend.cpp +++ b/app/render/backend/audio/audiobackend.cpp @@ -46,7 +46,7 @@ bool AudioBackend::InitInternal() // Initiate one thread per CPU core for (int i=0;iSetParameters(params()); processors_.append(processor); } diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index 807bec957..bdaf4d6b9 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -22,8 +22,8 @@ OLIVE_NAMESPACE_ENTER -AudioWorker::AudioWorker(DecoderCache* decoder_cache, QHash *copy_map, QObject *parent) : - AudioRenderWorker(decoder_cache, copy_map, parent) +AudioWorker::AudioWorker(QHash *copy_map, QObject *parent) : + AudioRenderWorker(copy_map, parent) { } diff --git a/app/render/backend/audio/audioworker.h b/app/render/backend/audio/audioworker.h index 269b8547f..db49a1902 100644 --- a/app/render/backend/audio/audioworker.h +++ b/app/render/backend/audio/audioworker.h @@ -28,7 +28,7 @@ OLIVE_NAMESPACE_ENTER class AudioWorker : public AudioRenderWorker { public: - AudioWorker(DecoderCache* decoder_cache, QHash* copy_map, QObject* parent = nullptr); + AudioWorker(QHash* copy_map, QObject* parent = nullptr); protected: virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override; diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index aaf68905a..b60417c37 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -30,8 +30,8 @@ OLIVE_NAMESPACE_ENTER -AudioRenderWorker::AudioRenderWorker(DecoderCache* decoder_cache, QHash *copy_map, QObject *parent) : - RenderWorker(decoder_cache, parent), +AudioRenderWorker::AudioRenderWorker(QHash *copy_map, QObject *parent) : + RenderWorker(parent), copy_map_(copy_map) { } diff --git a/app/render/backend/audiorenderworker.h b/app/render/backend/audiorenderworker.h index a8614b685..7da959984 100644 --- a/app/render/backend/audiorenderworker.h +++ b/app/render/backend/audiorenderworker.h @@ -29,7 +29,7 @@ class AudioRenderWorker : public RenderWorker { Q_OBJECT public: - AudioRenderWorker(DecoderCache* decoder_cache, QHash* copy_map, QObject* parent = nullptr); + AudioRenderWorker(QHash* copy_map, QObject* parent = nullptr); void SetParameters(const AudioRenderingParams& audio_params); diff --git a/app/render/backend/opengl/openglbackend.cpp b/app/render/backend/opengl/openglbackend.cpp index df6cd32cb..2db0cefa6 100644 --- a/app/render/backend/opengl/openglbackend.cpp +++ b/app/render/backend/opengl/openglbackend.cpp @@ -63,7 +63,7 @@ bool OpenGLBackend::InitInternal() // Initiate one thread per CPU core for (int i=0;iSetParameters(params()); processors_.append(processor); diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index 3508ab3ed..edff4f17b 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -31,8 +31,8 @@ OLIVE_NAMESPACE_ENTER -OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, DecoderCache* decoder_cache, QObject *parent) : - VideoRenderWorker(frame_cache, decoder_cache, parent) +OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, QObject *parent) : + VideoRenderWorker(frame_cache, parent) { } diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index 1fc920523..d8321d674 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -34,7 +34,7 @@ OLIVE_NAMESPACE_ENTER class OpenGLWorker : public VideoRenderWorker { Q_OBJECT public: - OpenGLWorker(VideoRenderFrameCache* frame_cache, DecoderCache *decoder_cache, + OpenGLWorker(VideoRenderFrameCache* frame_cache, QObject* parent = nullptr); signals: diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index 48b2fd66a..0cfb0eae4 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -105,8 +105,6 @@ void RenderBackend::Close() threads_.clear(); processors_.clear(); - - decoder_cache_.Clear(); } const QString &RenderBackend::GetError() const @@ -404,11 +402,6 @@ void RenderBackend::SetWorkerBusyState(RenderWorker *worker, bool busy) processor_busy_state_.replace(processors_.indexOf(worker), busy); } -DecoderCache *RenderBackend::decoder_cache() -{ - return &decoder_cache_; -} - bool RenderBackend::AllProcessorsAreAvailable() const { foreach (bool busy, processor_busy_state_) { diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index 0beb63700..db53d503e 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -118,14 +118,10 @@ protected: bool WorkerIsBusy(RenderWorker* worker) const; void SetWorkerBusyState(RenderWorker* worker, bool busy); - DecoderCache* decoder_cache(); - TimeRangeList cache_queue_; QVector processors_; - DecoderCache decoder_cache_; - bool compiled_; QHash render_job_info_; diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index 07fee972a..58b14aef2 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -26,10 +26,9 @@ OLIVE_NAMESPACE_ENTER -RenderWorker::RenderWorker(DecoderCache *decoder_cache, QObject *parent) : +RenderWorker::RenderWorker(QObject *parent) : QObject(parent), - started_(false), - decoder_cache_(decoder_cache) + started_(false) { } @@ -50,6 +49,8 @@ void RenderWorker::Close() { CloseInternal(); + decoder_cache_.Clear(); + started_ = false; } @@ -82,11 +83,9 @@ StreamPtr RenderWorker::ResolveStreamFromInput(NodeInput *input) DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream) { - QMutexLocker locker(decoder_cache_->lock()); - // Access a map of Node inputs and decoder instances and retrieve a frame! - DecoderPtr decoder = decoder_cache_->Get(stream.get()); + DecoderPtr decoder = decoder_cache_.Get(stream.get()); if (!decoder && stream) { // Create a new Decoder here @@ -94,7 +93,7 @@ DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream) decoder->set_stream(stream); if (decoder->Open()) { - decoder_cache_->Add(stream.get(), decoder); + decoder_cache_.Add(stream.get(), decoder); } else { decoder = nullptr; qWarning() << "Failed to open decoder for" << stream->footage()->filename() << "::" << stream->index(); diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index 1125015a2..224b8af08 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -35,7 +35,7 @@ class RenderWorker : public QObject, public NodeTraverser { Q_OBJECT public: - RenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); + RenderWorker(QObject* parent = nullptr); bool Init(); @@ -77,7 +77,7 @@ protected: private: bool started_; - DecoderCache* decoder_cache_; + DecoderCache decoder_cache_; NodeDependency path_; diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index 6061975a4..492022824 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -34,8 +34,8 @@ OLIVE_NAMESPACE_ENTER -VideoRenderWorker::VideoRenderWorker(VideoRenderFrameCache *frame_cache, DecoderCache* decoder_cache, QObject *parent) : - RenderWorker(decoder_cache, parent), +VideoRenderWorker::VideoRenderWorker(VideoRenderFrameCache *frame_cache, QObject *parent) : + RenderWorker(parent), frame_cache_(frame_cache), operating_mode_(kHashRenderCache) { diff --git a/app/render/backend/videorenderworker.h b/app/render/backend/videorenderworker.h index 3d7b5e6b7..4a69f060c 100644 --- a/app/render/backend/videorenderworker.h +++ b/app/render/backend/videorenderworker.h @@ -63,7 +63,7 @@ public: kHashRenderCache = 0x7 }; - VideoRenderWorker(VideoRenderFrameCache* frame_cache, DecoderCache *decoder_cache, QObject* parent = nullptr); + VideoRenderWorker(VideoRenderFrameCache* frame_cache, QObject* parent = nullptr); void SetParameters(const VideoRenderingParams& video_params); From ce7fa12520371c7f67e64b8f1b635ef286a89675 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 11 Apr 2020 13:18:44 +1000 Subject: [PATCH 02/14] ffmpegdecoder: implemented basic decoder crosstalk --- app/codec/ffmpeg/ffmpegdecoder.cpp | 303 +++++++++++++++++++---------- app/codec/ffmpeg/ffmpegdecoder.h | 22 ++- 2 files changed, 219 insertions(+), 106 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 3117b0fb6..e478989da 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -44,6 +44,9 @@ extern "C" { OLIVE_NAMESPACE_ENTER +QHash< Stream*, QList > FFmpegDecoder::instances_; +QMutex FFmpegDecoder::instance_lock_; + FFmpegDecoder::FFmpegDecoder() : fmt_ctx_(nullptr), codec_ctx_(nullptr), @@ -53,10 +56,12 @@ FFmpegDecoder::FFmpegDecoder() : cache_at_eof_(false), opts_(nullptr) { + /* // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints clear_timer_.setInterval(250); clear_timer_.moveToThread(qApp->thread()); connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent); + */ } FFmpegDecoder::~FFmpegDecoder() @@ -130,7 +135,8 @@ bool FFmpegDecoder::Open() } // Set multithreading setting - error_code = av_dict_set(&opts_, "threads", "auto", 0); + error_code = av_dict_set(&opts_, "threads", "1", 0); + //error_code = av_dict_set(&opts_, "threads", "auto", 0); // Handle failure to set multithreaded decoding if (error_code < 0) { @@ -171,12 +177,20 @@ bool FFmpegDecoder::Open() second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base))); - QMetaObject::invokeMethod(&clear_timer_, "start"); + //QMetaObject::invokeMethod(&clear_timer_, "start"); } // All allocation succeeded so we set the state to open open_ = true; + { + QMutexLocker l(&instance_lock_); + + QList list = instances_.value(stream().get()); + list.append(this); + instances_.insert(stream().get(), list); + } + return true; } @@ -190,6 +204,8 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time) if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + // Do nothing + } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { AudioStreamPtr audio_stream = std::static_pointer_cast(stream()); @@ -204,6 +220,8 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time) FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int ÷r) { QMutexLocker locker(&mutex_); + QMutexLocker cache_locker(&cache_lock_); + QMutexLocker retrieve_locker(&retrieve_lock_); if (!open_) { qWarning() << "Tried to retrieve video on a decoder that's still closed"; @@ -214,78 +232,57 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid return nullptr; } + cache_target_time_ = timecode; + int64_t target_ts = Timecode::time_to_timestamp(timecode, avstream_->time_base) + avstream_->start_time; - Frame* return_frame = nullptr; - - if (divider != scale_divider_) { - ClearFrameCache(); - FreeScaler(); - SetupScaler(divider); - } + AVFrame* return_frame = nullptr; // See if our RAM cache already has a frame that matches this timestamp - if (!cached_frames_.isEmpty()) { + if (CacheContainsTime(target_ts)) { + return_frame = GetFrameFromCache(target_ts); + } - if (target_ts < cached_frames_.first()->native_timestamp()) { + if (!return_frame) { + // Check siblings for any others rendering at this time + QMutexLocker list_locker(&instance_lock_); - if (cache_at_zero_) { - return_frame = cached_frames_.first(); - cached_frames_.accessedFirst(); + QList siblings = instances_.value(stream().get()); + + foreach (FFmpegDecoder* sibling, siblings) { + if (sibling == this) { + continue; } - } else if (target_ts > cached_frames_.last()->native_timestamp()) { + QMutexLocker sibling_locker(&sibling->cache_lock_); - if (cache_at_eof_) { - return_frame = cached_frames_.last(); - cached_frames_.accessedLast(); - } + if (sibling->CacheContainsTime(target_ts)) { + return_frame = sibling->GetFrameFromCache(target_ts); + break; + } else { + bool cache_will_contain = sibling->CacheWillContainTime(timecode); + bool cache_could_contain = sibling->CacheCouldContainTime(timecode); - } else { + do { + if (cache_will_contain) { + sibling->cache_wait_cond_.wait(&sibling->cache_lock_); - // We already have this frame in the cache, find it - for (int i=0;iGetFrameFromCache(target_ts); + } else if (cache_could_contain) { + sibling->cache_retrieve_cond_.wait(&sibling->retrieve_lock_); - if (this_frame->native_timestamp() == target_ts // Test for an exact match - || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->native_timestamp() > target_ts)) { // Or for this frame to be the "closest" - - return_frame = this_frame; - cached_frames_.accessed(i); - - break; - - } + if (CacheWillContainTime(timecode)) { + cache_will_contain = true; + } else { + // Swap and continue this instance ourselves + break; + } + } + } while (!return_frame); } } } - // See if we stored this frame in the disk cache - /* - QByteArray frame_loader; - if (!got_frame) { - QFile compressed_frame(GetIndexFilename().append(QString::number(target_ts))); - if (compressed_frame.exists() - && compressed_frame.size() > 0 - && compressed_frame.open(QFile::ReadOnly)) { - DiskManager::instance()->Accessed(compressed_frame.fileName()); - - // Read data - frame_loader = qUncompress(compressed_frame.readAll()); - - av_image_fill_arrays(input_data, - input_linesize, - reinterpret_cast(frame_loader.data()), - static_cast(avstream_->codecpar->format), - avstream_->codecpar->width, - avstream_->codecpar->height, - 1); - - got_frame = true; - } - } - */ - // If we have no disk cache, we'll need to find this frame ourselves if (!return_frame) { int64_t seek_ts = target_ts; @@ -293,8 +290,8 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid // If the frame wasn't in the frame cache, see if this frame cache is too old to use if (cached_frames_.isEmpty() - || target_ts < cached_frames_.first()->native_timestamp() - || target_ts > cached_frames_.last()->native_timestamp() + 2*second_ts_) { + || target_ts < cached_frames_.first()->pts + || target_ts > cached_frames_.last()->pts + 2*second_ts_) { ClearFrameCache(); Seek(seek_ts); @@ -307,16 +304,17 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid int ret; AVPacket* pkt = av_packet_alloc(); - AVFrame* working_frame = av_frame_alloc(); while (true) { // Allocate a new frame + AVFrame* working_frame = av_frame_alloc(); // Pull from the decoder ret = GetFrame(pkt, working_frame); // Handle any errors that aren't EOF (EOF is handled later on) if (ret < 0 && ret != AVERROR_EOF) { + av_frame_free(&working_frame); FFmpegError(ret); break; } @@ -331,6 +329,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid if (seek_ts == 0) { cache_at_zero_ = true; } + av_frame_free(&working_frame); continue; } else { @@ -345,69 +344,68 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid // Handle an "expected" EOF by using the last frame of our cache cache_at_eof_ = true; return_frame = cached_frames_.last(); + av_frame_free(&working_frame); + break; } else { - bool working_frame_is_the_one = false; + // Whatever it is, keep this frame in memory for the time being just in case + if (cached_frames_.size() > 0) { + cache_locker.relock(); + } + + cached_frames_.append(working_frame); + cache_wait_cond_.wakeAll(); + + cache_locker.unlock(); // If this is a valid frame, see if this or the frame before it are the one we need if (working_frame->pts == target_ts) { - working_frame_is_the_one = true; + return_frame = working_frame; + break; } else if (working_frame->pts > target_ts) { if (cached_frames_.isEmpty() && cache_at_zero_) { - working_frame_is_the_one = true; + return_frame = working_frame; + break; } else { - return_frame = cached_frames_.last(); + return_frame = cached_frames_.at(cached_frames_.size() - 2); + break; } } - - // Whatever it is, keep this frame in memory for the time being just in case - Frame* working_frame_converted = cached_frames_.append(VideoRenderingParams(avstream_->codecpar->width / divider, - avstream_->codecpar->height / divider, - avstream_->time_base, - native_pix_fmt_, - RenderMode::kOffline)); - - working_frame_converted->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base)); - working_frame_converted->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr)); - working_frame_converted->set_native_timestamp(working_frame->pts); - - // Convert frame to RGBA for the rest of the pipeline - uint8_t* output_data = reinterpret_cast(working_frame_converted->data()); - int output_linesize = working_frame_converted->width() * PixelFormat::ChannelCount(native_pix_fmt_) * PixelFormat::BytesPerChannel(native_pix_fmt_); - - sws_scale(scale_ctx_, - working_frame->data, - working_frame->linesize, - 0, - avstream_->codecpar->height, - &output_data, - &output_linesize); - - if (working_frame_is_the_one) { - // We found the frame we want - return_frame = working_frame_converted; - } - } - - if (return_frame) { - break; } } av_packet_free(&pkt); - av_frame_free(&working_frame); } + retrieve_locker.unlock(); + // We found the frame, we'll return a copy if (return_frame) { + if (divider != scale_divider_) { + FreeScaler(); + SetupScaler(divider); + } + FramePtr copy = Frame::Create(); - copy->set_video_params(return_frame->video_params()); - copy->set_timestamp(return_frame->timestamp()); - copy->set_sample_aspect_ratio(return_frame->sample_aspect_ratio()); + copy->set_video_params(VideoRenderingParams(avstream_->codecpar->width / divider, + avstream_->codecpar->height / divider, + native_pix_fmt_)); + copy->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base)); + copy->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr)); copy->allocate(); - memcpy(copy->data(), return_frame->data(), copy->allocated_size()); + // Convert frame to RGBA for the rest of the pipeline + uint8_t* output_data = reinterpret_cast(copy->data()); + int output_linesize = copy->width() * PixelFormat::ChannelCount(native_pix_fmt_) * PixelFormat::BytesPerChannel(native_pix_fmt_); + + sws_scale(scale_ctx_, + return_frame->data, + return_frame->linesize, + 0, + avstream_->codecpar->height, + &output_data, + &output_linesize); return copy; } @@ -450,9 +448,17 @@ void FFmpegDecoder::Close() { QMutexLocker locker(&mutex_); + { + QMutexLocker l(&instance_lock_); + + QList list = instances_.value(stream().get()); + list.removeOne(this); + instances_.insert(stream().get(), list); + } + ClearResources(); - clear_timer_.stop(); + //clear_timer_.stop(); } QString FFmpegDecoder::id() @@ -903,6 +909,32 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) DiskManager::instance()->CreatedFile(save_frame.fileName(), QByteArray()); } + + // See if we stored this frame in the disk cache + /* + QByteArray frame_loader; + if (!got_frame) { + QFile compressed_frame(GetIndexFilename().append(QString::number(target_ts))); + if (compressed_frame.exists() + && compressed_frame.size() > 0 + && compressed_frame.open(QFile::ReadOnly)) { + DiskManager::instance()->Accessed(compressed_frame.fileName()); + + // Read data + frame_loader = qUncompress(compressed_frame.readAll()); + + av_image_fill_arrays(input_data, + input_linesize, + reinterpret_cast(frame_loader.data()), + static_cast(avstream_->codecpar->format), + avstream_->codecpar->width, + avstream_->codecpar->height, + 1); + + got_frame = true; + } + } + */ } /*void FFmpegDecoder::RemoveFirstFromFrameCache() @@ -987,6 +1019,70 @@ void FFmpegDecoder::FreeScaler() } } +int64_t FFmpegDecoder::RangeStart() const +{ + if (cached_frames_.isEmpty()) { + return AV_NOPTS_VALUE; + } + return cached_frames_.first()->pts; +} + +int64_t FFmpegDecoder::RangeEnd() const +{ + if (cached_frames_.isEmpty()) { + return AV_NOPTS_VALUE; + } + return cached_frames_.last()->pts; +} + +bool FFmpegDecoder::CacheContainsTime(const int64_t &t) const +{ + return (RangeStart() <= t && RangeEnd() >= t); +} + +bool FFmpegDecoder::CacheWillContainTime(const rational &t) const +{ + return !cached_frames_.isEmpty() && t <= cache_target_time_; +} + +bool FFmpegDecoder::CacheCouldContainTime(const rational &t) const +{ + return !cached_frames_.isEmpty() && t <= (cache_target_time_ + 2); +} + +AVFrame *FFmpegDecoder::GetFrameFromCache(const int64_t &t) const +{ + if (t < cached_frames_.first()->pts) { + + if (cache_at_zero_) { + return cached_frames_.first(); + } + + } else if (t > cached_frames_.last()->pts) { + + if (cache_at_eof_) { + return cached_frames_.last(); + } + + } else { + + // We already have this frame in the cache, find it + for (int i=0;ipts == t // Test for an exact match + || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->pts > t)) { // Or for this frame to be the "closest" + + return this_frame; + + } + } + } + + return nullptr; +} + +/* void FFmpegDecoder::ClearTimerEvent() { QMutexLocker locker(&mutex_); @@ -994,5 +1090,6 @@ void FFmpegDecoder::ClearTimerEvent() cache_at_zero_ = false; cached_frames_.remove_old_frames(QDateTime::currentMSecsSinceEpoch() - clear_timer_.interval()); } +*/ OLIVE_NAMESPACE_EXIT diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 8213426c1..dc0bff919 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -30,6 +30,7 @@ extern "C" { #include #include #include +#include #include "audio/sampleformat.h" #include "codec/decoder.h" @@ -111,6 +112,13 @@ private: void SetupScaler(const int& divider); void FreeScaler(); + int64_t RangeStart() const; + int64_t RangeEnd() const; + bool CacheContainsTime(const int64_t& t) const; + bool CacheWillContainTime(const rational& t) const; + bool CacheCouldContainTime(const rational& t) const; + AVFrame* GetFrameFromCache(const int64_t& t) const; + AVFormatContext* fmt_ctx_; AVCodecContext* codec_ctx_; AVStream* avstream_; @@ -121,7 +129,12 @@ private: SwsContext* scale_ctx_; int scale_divider_; - FFmpegFrameCache::Client cached_frames_; + QWaitCondition cache_wait_cond_; + QWaitCondition cache_retrieve_cond_; + QMutex cache_lock_; + QMutex retrieve_lock_; + QList cached_frames_; + rational cache_target_time_; bool cache_at_zero_; bool cache_at_eof_; @@ -129,10 +142,13 @@ private: AVDictionary* opts_; - QTimer clear_timer_; + //QTimer clear_timer_; + + static QHash< Stream*, QList > instances_; + static QMutex instance_lock_; private slots: - void ClearTimerEvent(); + //void ClearTimerEvent(); }; From f08734d18e0a86e3e8246babf48b1c14fd22f673 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 11 Apr 2020 17:36:16 +1000 Subject: [PATCH 03/14] index/confirm tasks: these tasks no longer need to open the decoder to perform said tasks --- app/task/conform/conform.cpp | 2 -- app/task/index/index.cpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/app/task/conform/conform.cpp b/app/task/conform/conform.cpp index 6be547d10..2350f5f19 100644 --- a/app/task/conform/conform.cpp +++ b/app/task/conform/conform.cpp @@ -42,9 +42,7 @@ void ConformTask::Action() connect(decoder.get(), &Decoder::IndexProgress, this, &ConformTask::ProgressChanged); - decoder->Open(); decoder->Conform(params_, &IsCancelled()); - decoder->Close(); emit Succeeded(); } diff --git a/app/task/index/index.cpp b/app/task/index/index.cpp index 248901a73..0fb6393b7 100644 --- a/app/task/index/index.cpp +++ b/app/task/index/index.cpp @@ -42,9 +42,7 @@ void IndexTask::Action() connect(decoder.get(), &Decoder::IndexProgress, this, &IndexTask::ProgressChanged); - decoder->Open(); decoder->Index(&IsCancelled()); - decoder->Close(); emit Succeeded(); } From a8fbe686e21be69936a67d985f9bc233ed6e250f Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 11 Apr 2020 17:37:00 +1000 Subject: [PATCH 04/14] openglrenderer: retrieve frame in each worker thread rather than in the proxy thread Since we now have decoders in each thread, we can parallelize by retrieving frames in each thread as well. --- app/render/backend/opengl/openglproxy.cpp | 9 +-------- app/render/backend/opengl/openglproxy.h | 2 +- app/render/backend/opengl/openglworker.cpp | 6 +++++- app/render/backend/opengl/openglworker.h | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/app/render/backend/opengl/openglproxy.cpp b/app/render/backend/opengl/openglproxy.cpp index d3510db4a..04f6e7d43 100644 --- a/app/render/backend/opengl/openglproxy.cpp +++ b/app/render/backend/opengl/openglproxy.cpp @@ -67,7 +67,7 @@ bool OpenGLProxy::Init() return true; } -void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) +void OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream, const TimeRange &range, NodeValueTable* table) { // Ensure stream is video or image type if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) { @@ -105,13 +105,6 @@ void OpenGLProxy::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode()); - FramePtr frame = decoder->RetrieveVideo(range.in(), video_params_.divider()); - - if (!frame) { - // Nothing to be done - return; - } - // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU if (ocio_method == ColorManager::kOCIOAccurate) { bool has_alpha = PixelFormat::FormatHasAlphaChannel(frame->format()); diff --git a/app/render/backend/opengl/openglproxy.h b/app/render/backend/opengl/openglproxy.h index 77ee7a89b..1203b9330 100644 --- a/app/render/backend/opengl/openglproxy.h +++ b/app/render/backend/opengl/openglproxy.h @@ -63,7 +63,7 @@ public: void Close(); - void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table); + void FrameToValue(FramePtr frame, StreamPtr stream, const TimeRange &range, NodeValueTable* table); void RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params); diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index edff4f17b..b141cb3b0 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -38,7 +38,11 @@ OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, QObject *parent) void OpenGLWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable *table) { - emit RequestFrameToValue(decoder, stream, range, table); + FramePtr frame = decoder->RetrieveVideo(range.in(), video_params().divider()); + + if (frame) { + emit RequestFrameToValue(frame, stream, range, table); + } } void OpenGLWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable *output_params) diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index d8321d674..1ec1846e9 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -38,7 +38,7 @@ public: QObject* parent = nullptr); signals: - void RequestFrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table); + void RequestFrameToValue(FramePtr frame, StreamPtr stream, const TimeRange &range, NodeValueTable* table); void RequestRunNodeAccelerated(const Node *node, const TimeRange &range, const NodeValueDatabase &input_params, NodeValueTable* output_params); From e6b47a817c7a6afe8abdadafad7dd689245dcc78 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 11 Apr 2020 17:37:52 +1000 Subject: [PATCH 05/14] ffmpegdecoder: separated instance from decoder entity Should allow for different decoders in different threads to share a single instance. --- app/codec/ffmpeg/ffmpegdecoder.cpp | 701 +++++++++++++++++------------ app/codec/ffmpeg/ffmpegdecoder.h | 98 ++-- 2 files changed, 479 insertions(+), 320 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index e478989da..8edd2bfa1 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -48,13 +48,9 @@ QHash< Stream*, QList > FFmpegDecoder::instances_; QMutex FFmpegDecoder::instance_lock_; FFmpegDecoder::FFmpegDecoder() : - fmt_ctx_(nullptr), - codec_ctx_(nullptr), scale_ctx_(nullptr), scale_divider_(-1), - cache_at_zero_(false), - cache_at_eof_(false), - opts_(nullptr) + instance_(nullptr) { /* // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints @@ -79,81 +75,20 @@ bool FFmpegDecoder::Open() Q_ASSERT(stream()); - int error_code; - // Convert QString to a C string - QByteArray ba = stream()->footage()->filename().toUtf8(); - const char* filename = ba.constData(); + QByteArray fn_bytes = stream()->footage()->filename().toUtf8(); - // Open file in a format context - error_code = avformat_open_input(&fmt_ctx_, filename, nullptr, nullptr); + instance_ = new FFmpegDecoderInstance(fn_bytes.constData(), stream()->index()); - // Handle format context error - if (error_code != 0) { - FFmpegError(error_code); + if (!instance_->IsValid()) { + delete instance_; return false; } - // Get stream information from format - error_code = avformat_find_stream_info(fmt_ctx_, nullptr); - - // Handle get stream information error - if (error_code < 0) { - FFmpegError(error_code); - return false; - } - - // Get reference to correct AVStream - avstream_ = fmt_ctx_->streams[stream()->index()]; - - // Find decoder - AVCodec* codec = avcodec_find_decoder(avstream_->codecpar->codec_id); - - // Handle failure to find decoder - if (codec == nullptr) { - Error(QStringLiteral("Failed to find appropriate decoder for this codec (%1:%2 - %3)") - .arg(stream()->footage()->filename(), - QString::number(avstream_->index), - QString::number(avstream_->codecpar->codec_id))); - return false; - } - - // Allocate context for the decoder - codec_ctx_ = avcodec_alloc_context3(codec); - if (codec_ctx_ == nullptr) { - Error(QStringLiteral("Failed to allocate codec context (%1 :: %2)").arg(stream()->footage()->filename(), stream()->index())); - return false; - } - - // Copy parameters from the AVStream to the AVCodecContext - error_code = avcodec_parameters_to_context(codec_ctx_, avstream_->codecpar); - - // Handle failure to copy parameters - if (error_code < 0) { - FFmpegError(error_code); - return false; - } - - // Set multithreading setting - error_code = av_dict_set(&opts_, "threads", "1", 0); - //error_code = av_dict_set(&opts_, "threads", "auto", 0); - - // Handle failure to set multithreaded decoding - if (error_code < 0) { - FFmpegError(error_code); - return false; - } - - // Open codec - error_code = avcodec_open2(codec_ctx_, codec, &opts_); - if (error_code < 0) { - FFmpegError(error_code); - return false; - } - - if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + if (stream()->type() == Stream::kVideo) { // Get an Olive compatible AVPixelFormat - ideal_pix_fmt_ = FFmpegCommon::GetCompatiblePixelFormat(static_cast(avstream_->codecpar->format)); + src_pix_fmt_ = static_cast(instance_->stream()->codecpar->format); + ideal_pix_fmt_ = FFmpegCommon::GetCompatiblePixelFormat(src_pix_fmt_); // Determine which Olive native pixel format we retrieved // Note that FFmpeg doesn't support float formats @@ -175,11 +110,14 @@ bool FFmpegDecoder::Open() qFatal("Invalid output format"); } - second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base))); + aspect_ratio_ = instance_->sample_aspect_ratio(); //QMetaObject::invokeMethod(&clear_timer_, "start"); } + time_base_ = instance_->stream()->time_base; + start_time_ = instance_->stream()->start_time; + // All allocation succeeded so we set the state to open open_ = true; @@ -202,11 +140,11 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time) return kFailedToOpen; } - if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + if (stream()->type() == Stream::kVideo) { // Do nothing - } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + } else if (stream()->type() == Stream::kAudio) { AudioStreamPtr audio_stream = std::static_pointer_cast(stream()); if (time > audio_stream->index_length() && !audio_stream->index_done()) { @@ -220,165 +158,140 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time) FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int ÷r) { QMutexLocker locker(&mutex_); - QMutexLocker cache_locker(&cache_lock_); - QMutexLocker retrieve_locker(&retrieve_lock_); if (!open_) { qWarning() << "Tried to retrieve video on a decoder that's still closed"; return nullptr; } - if (avstream_->codecpar->codec_type != AVMEDIA_TYPE_VIDEO) { + if (stream()->type() != Stream::kVideo) { return nullptr; } - cache_target_time_ = timecode; + int decoder_index = instances_.value(stream().get()).indexOf(this); - int64_t target_ts = Timecode::time_to_timestamp(timecode, avstream_->time_base) + avstream_->start_time; + int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_; + + qDebug() << decoder_index << "is retrieving time" << target_ts << "!"; AVFrame* return_frame = nullptr; // See if our RAM cache already has a frame that matches this timestamp - if (CacheContainsTime(target_ts)) { - return_frame = GetFrameFromCache(target_ts); - } - - if (!return_frame) { + if (instance_->CacheContainsTime(target_ts)) { + qDebug() << decoder_index << "found this frame in its cache"; + return_frame = instance_->GetFrameFromCache(target_ts); + } else { // Check siblings for any others rendering at this time + qDebug() << decoder_index << "is waiting for list locker"; QMutexLocker list_locker(&instance_lock_); + qDebug() << decoder_index << "acquired list locker"; QList siblings = instances_.value(stream().get()); - - foreach (FFmpegDecoder* sibling, siblings) { + for (int i=0;icache_lock_); + qDebug() << decoder_index << "waiting for sibling" << i << "'s cache"; + FFmpegDecoderInstance* sibling_instance = sibling->instance_; + QMutexLocker sibling_cache_locker(sibling_instance->cache_lock()); - if (sibling->CacheContainsTime(target_ts)) { - return_frame = sibling->GetFrameFromCache(target_ts); + qDebug() << decoder_index << "locked cache with range" << sibling_instance->RangeStart() << "-" << sibling_instance->RangeEnd(); + + if (sibling_instance->CacheContainsTime(target_ts)) { + + qDebug() << decoder_index << "found the frame it needed in" << i << "'s cache"; + + // We don't need the list anymore + qDebug() << decoder_index << "is unlocking the list locker"; + list_locker.unlock(); + + // Retrieve the frame we need from our sibling + return_frame = sibling_instance->GetFrameFromCache(target_ts); break; - } else { - bool cache_will_contain = sibling->CacheWillContainTime(timecode); - bool cache_could_contain = sibling->CacheCouldContainTime(timecode); - do { + } else { + + bool cache_will_contain = sibling_instance->CacheWillContainTime(target_ts); + bool cache_could_contain = sibling_instance->CacheCouldContainTime(target_ts); + + if (cache_will_contain || cache_could_contain) { + // We found an instance that we can use, we no longer need the list if (cache_will_contain) { - sibling->cache_wait_cond_.wait(&sibling->cache_lock_); - - return_frame = sibling->GetFrameFromCache(target_ts); - } else if (cache_could_contain) { - sibling->cache_retrieve_cond_.wait(&sibling->retrieve_lock_); - - if (CacheWillContainTime(timecode)) { - cache_will_contain = true; - } else { - // Swap and continue this instance ourselves - break; - } - } - } while (!return_frame); - } - } - } - - // If we have no disk cache, we'll need to find this frame ourselves - if (!return_frame) { - int64_t seek_ts = target_ts; - bool still_seeking = false; - - // If the frame wasn't in the frame cache, see if this frame cache is too old to use - if (cached_frames_.isEmpty() - || target_ts < cached_frames_.first()->pts - || target_ts > cached_frames_.last()->pts + 2*second_ts_) { - ClearFrameCache(); - - Seek(seek_ts); - if (seek_ts == 0) { - cache_at_zero_ = true; - } - - still_seeking = true; - } - - int ret; - AVPacket* pkt = av_packet_alloc(); - - while (true) { - // Allocate a new frame - AVFrame* working_frame = av_frame_alloc(); - - // Pull from the decoder - ret = GetFrame(pkt, working_frame); - - // Handle any errors that aren't EOF (EOF is handled later on) - if (ret < 0 && ret != AVERROR_EOF) { - av_frame_free(&working_frame); - FFmpegError(ret); - break; - } - - if (still_seeking) { - // Handle a failure to seek (occurs on some media) - // We'll only be here if the frame cache was emptied earlier - if (!cache_at_zero_ && (ret == AVERROR_EOF || working_frame->pts > target_ts)) { - - seek_ts = qMax(static_cast(0), seek_ts - second_ts_); - Seek(seek_ts); - if (seek_ts == 0) { - cache_at_zero_ = true; - } - av_frame_free(&working_frame); - continue; - - } else { - - still_seeking = false; - - } - } - - if (ret == AVERROR_EOF) { - - // Handle an "expected" EOF by using the last frame of our cache - cache_at_eof_ = true; - return_frame = cached_frames_.last(); - av_frame_free(&working_frame); - break; - - } else { - - // Whatever it is, keep this frame in memory for the time being just in case - if (cached_frames_.size() > 0) { - cache_locker.relock(); - } - - cached_frames_.append(working_frame); - cache_wait_cond_.wakeAll(); - - cache_locker.unlock(); - - // If this is a valid frame, see if this or the frame before it are the one we need - if (working_frame->pts == target_ts) { - return_frame = working_frame; - break; - } else if (working_frame->pts > target_ts) { - if (cached_frames_.isEmpty() && cache_at_zero_) { - return_frame = working_frame; - break; + qDebug() << decoder_index << "found that cache" << i << "WILL contain our time (Target:" << target_ts << "Cache Target:" << sibling_instance->cache_target_time_ << "End:" << sibling_instance->RangeEnd() << ")"; } else { - return_frame = cached_frames_.at(cached_frames_.size() - 2); - break; + qDebug() << decoder_index << "found that cache" << i << "COULD contain our time"; } + + qDebug() << decoder_index << "is unlocking the list locker since it found an instance to use"; + list_locker.unlock(); + + do { + if (cache_will_contain) { + + // Wait for next frame to arrive + qDebug() << decoder_index << "is WAITING for" << i << "to retrieve another frame"; + sibling_instance->cache_wait_cond()->wait(sibling_instance->cache_lock()); + qDebug() << decoder_index << "FINISHED waiting for" << i; + + // Attempt to pull frame from other thread + return_frame = sibling_instance->GetFrameFromCache(target_ts); + + } else if (cache_could_contain) { + + // Allow sibling instance to continue + qDebug() << decoder_index << "is unlocking" << i << "'s cache lock preparing for swap"; + sibling_cache_locker.unlock(); + + // Wait for sibling to finish its current job + qDebug() << decoder_index << "is waiting for sibling" << i << "to finish so we can swap"; + QMutexLocker sibling_locker(&sibling->mutex_); + + // Re-lock list + qDebug() << decoder_index << "is re-locking list locker for swap"; + list_locker.relock(); + + // Check if another thread has already swapped this isntance + if (sibling->instance_ == sibling_instance) { + + // Swap this instance with ours + qDebug() << decoder_index << "SWAPPING instance with" << i; + std::swap(sibling->instance_, instance_); + + // Continue rendering ourselves + qDebug() << decoder_index << "is continuing on its own"; + goto exit_sibling_loop; + + } else { + + // This instance has already swapped, see if they'll do anything about it + qDebug() << decoder_index << "waiting instance has already swapped, redoing loop"; + cache_will_contain = sibling_instance->CacheWillContainTime(target_ts); + continue; + + } + + } + } while (!return_frame); + } else { + qDebug() << decoder_index << "is ignoring" << i; } } } - av_packet_free(&pkt); - } + if (!return_frame) { +exit_sibling_loop: + // Before we unlock `list_locker`, we lock our own before we start encoding + instance_->cache_lock()->lock(); - retrieve_locker.unlock(); + list_locker.unlock(); + + // If we have no RAM cache, we'll need to find this frame ourselves + return_frame = instance_->RetrieveFrame(target_ts, true); + } + } // We found the frame, we'll return a copy if (return_frame) { @@ -387,12 +300,14 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid SetupScaler(divider); } + VideoStream* vs = static_cast(stream().get()); + FramePtr copy = Frame::Create(); - copy->set_video_params(VideoRenderingParams(avstream_->codecpar->width / divider, - avstream_->codecpar->height / divider, + copy->set_video_params(VideoRenderingParams(vs->width() / divider, + vs->height() / divider, native_pix_fmt_)); - copy->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base)); - copy->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr)); + copy->set_timestamp(Timecode::timestamp_to_time(target_ts, time_base_)); + copy->set_sample_aspect_ratio(aspect_ratio_); copy->allocate(); // Convert frame to RGBA for the rest of the pipeline @@ -403,7 +318,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid return_frame->data, return_frame->linesize, 0, - avstream_->codecpar->height, + return_frame->height, &output_data, &output_linesize); @@ -422,7 +337,7 @@ SampleBufferPtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rat return nullptr; } - if (avstream_->codecpar->codec_type != AVMEDIA_TYPE_AUDIO) { + if (stream()->type() != Stream::kAudio) { return nullptr; } @@ -494,7 +409,8 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled) const char* filename = ba.constData(); // Open file in a format context - error_code = avformat_open_input(&fmt_ctx_, filename, nullptr, nullptr); + AVFormatContext* fmt_ctx = nullptr; + error_code = avformat_open_input(&fmt_ctx, filename, nullptr, nullptr); QList streams_that_need_manual_duration; @@ -502,40 +418,40 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled) if (error_code == 0) { // Retrieve metadata about the media - avformat_find_stream_info(fmt_ctx_, nullptr); + avformat_find_stream_info(fmt_ctx, nullptr); // Dump it into the Footage object - for (unsigned int i=0;inb_streams;i++) { + for (unsigned int i=0;inb_streams;i++) { - avstream_ = fmt_ctx_->streams[i]; + AVStream* avstream = fmt_ctx->streams[i]; StreamPtr str; - if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + if (avstream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { // Create a video stream object VideoStreamPtr video_stream = std::make_shared(); - video_stream->set_width(avstream_->codecpar->width); - video_stream->set_height(avstream_->codecpar->height); - video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx_, avstream_, nullptr)); - video_stream->set_start_time(avstream_->start_time); + video_stream->set_width(avstream->codecpar->width); + video_stream->set_height(avstream->codecpar->height); + video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx, avstream, nullptr)); + video_stream->set_start_time(avstream->start_time); str = video_stream; - } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + } else if (avstream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { // Create an audio stream object AudioStreamPtr audio_stream = std::make_shared(); - uint64_t channel_layout = avstream_->codecpar->channel_layout; + uint64_t channel_layout = avstream->codecpar->channel_layout; if (!channel_layout) { - channel_layout = static_cast(av_get_default_channel_layout(avstream_->codecpar->channels)); + channel_layout = static_cast(av_get_default_channel_layout(avstream->codecpar->channels)); } audio_stream->set_channel_layout(channel_layout); - audio_stream->set_channels(avstream_->codecpar->channels); - audio_stream->set_sample_rate(avstream_->codecpar->sample_rate); + audio_stream->set_channels(avstream->codecpar->channels); + audio_stream->set_sample_rate(avstream->codecpar->sample_rate); str = audio_stream; @@ -545,7 +461,7 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled) str = std::make_shared(); // Set the correct codec type based on FFmpeg's result - switch (avstream_->codecpar->codec_type) { + switch (avstream->codecpar->codec_type) { case AVMEDIA_TYPE_UNKNOWN: str->set_type(Stream::kUnknown); break; @@ -566,12 +482,12 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled) } - str->set_index(avstream_->index); - str->set_timebase(avstream_->time_base); - str->set_duration(avstream_->duration); + str->set_index(avstream->index); + str->set_timebase(avstream->time_base); + str->set_duration(avstream->duration); // The container/stream info may not contain a duration, so we'll need to manually retrieve it - if (avstream_->duration == AV_NOPTS_VALUE) { + if (avstream->duration == AV_NOPTS_VALUE) { streams_that_need_manual_duration.append(str.get()); } @@ -599,7 +515,7 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled) av_packet_unref(pkt); // Read packet from file - int ret = av_read_frame(fmt_ctx_, pkt); + int ret = av_read_frame(fmt_ctx, pkt); if (ret < 0) { // Handle errors that aren't EOF (which simply means the file is finished) @@ -628,7 +544,7 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled) } // Free all memory - Close(); + avformat_close_input(&fmt_ctx); return result; } @@ -652,11 +568,6 @@ void FFmpegDecoder::Error(const QString &s) void FFmpegDecoder::Index(const QAtomicInt* cancelled) { - if (!open_) { - qWarning() << "Indexing function tried to run while decoder was closed"; - return; - } - QMutexLocker locker(stream()->index_process_lock()); if (stream()->type() == Stream::kAudio) { @@ -678,29 +589,26 @@ void FFmpegDecoder::Index(const QAtomicInt* cancelled) QString FFmpegDecoder::GetIndexFilename() { - if (!open_) { - qWarning() << "GetIndexFilename tried to run while decoder was closed"; - return QString(); - } - return GetMediaIndexFilename(GetUniqueFileIdentifier(stream()->footage()->filename())) - .append(QString::number(avstream_->index)); + .append(QString::number(stream()->index())); } void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) { // Iterate through each audio frame and extract the PCM data - Seek(0); + QByteArray fn_bytes = stream()->footage()->filename().toUtf8(); - uint64_t channel_layout = avstream_->codecpar->channel_layout; + FFmpegDecoderInstance index_instance(fn_bytes.constData(), stream()->index()); + + uint64_t channel_layout = index_instance.stream()->codecpar->channel_layout; if (!channel_layout) { - if (!avstream_->codecpar->channels) { + if (!index_instance.stream()->codecpar->channels) { // No channel data - we can't do anything with this return; } - channel_layout = static_cast(av_get_default_channel_layout(avstream_->codecpar->channels)); + channel_layout = static_cast(av_get_default_channel_layout(index_instance.stream()->codecpar->channels)); } AudioStreamPtr audio_stream = std::static_pointer_cast(stream()); @@ -709,7 +617,7 @@ void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) audio_stream->clear_index(); SwrContext* resampler = nullptr; - AVSampleFormat src_sample_fmt = static_cast(avstream_->codecpar->format); + AVSampleFormat src_sample_fmt = static_cast(index_instance.stream()->codecpar->format); AVSampleFormat dst_sample_fmt; // We don't use planar types internally, so if this is a planar format convert it now @@ -717,12 +625,12 @@ void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) dst_sample_fmt = av_get_packed_sample_fmt(src_sample_fmt); resampler = swr_alloc_set_opts(nullptr, - static_cast(avstream_->codecpar->channel_layout), + static_cast(index_instance.stream()->codecpar->channel_layout), dst_sample_fmt, - avstream_->codecpar->sample_rate, - static_cast(avstream_->codecpar->channel_layout), + index_instance.stream()->codecpar->sample_rate, + static_cast(index_instance.stream()->codecpar->channel_layout), src_sample_fmt, - avstream_->codecpar->sample_rate, + index_instance.stream()->codecpar->sample_rate, 0, nullptr); @@ -731,7 +639,7 @@ void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) dst_sample_fmt = src_sample_fmt; } - AudioRenderingParams wave_params(avstream_->codecpar->sample_rate, + AudioRenderingParams wave_params(index_instance.stream()->codecpar->sample_rate, channel_layout, FFmpegCommon::GetNativeSampleFormat(dst_sample_fmt)); WaveOutput wave_out(GetIndexFilename(), wave_params); @@ -749,7 +657,7 @@ void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) break; } - ret = GetFrame(pkt, frame); + ret = index_instance.GetFrame(pkt, frame); if (ret < 0) { @@ -827,11 +735,9 @@ void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) av_frame_free(&frame); av_packet_free(&pkt); - - Seek(0); } -int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame) +int FFmpegDecoderInstance::GetFrame(AVPacket *pkt, AVFrame *frame) { bool eof = false; @@ -876,7 +782,17 @@ int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame) return ret; } -void FFmpegDecoder::Seek(int64_t timestamp) +QMutex *FFmpegDecoderInstance::cache_lock() +{ + return &cache_lock_; +} + +QWaitCondition *FFmpegDecoderInstance::cache_wait_cond() +{ + return &cache_wait_cond_; +} + +void FFmpegDecoderInstance::Seek(int64_t timestamp) { avcodec_flush_buffers(codec_ctx_); av_seek_frame(fmt_ctx_, avstream_->index, timestamp, AVSEEK_FLAG_BACKWARD); @@ -959,44 +875,144 @@ void FFmpegDecoder::RemoveLastFromFrameCache() cache_at_eof_ = false; }*/ -void FFmpegDecoder::ClearFrameCache() +void FFmpegDecoderInstance::ClearFrameCache() { cached_frames_.clear(); cache_at_eof_ = false; cache_at_zero_ = false; } +AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) +{ + int64_t seek_ts = target_ts; + bool still_seeking = false; + + cache_target_time_ = target_ts; + + // If the frame wasn't in the frame cache, see if this frame cache is too old to use + if (cached_frames_.isEmpty() + || target_ts < cached_frames_.first()->pts + || target_ts > cached_frames_.last()->pts + 2*second_ts_) { + if (!cache_is_locked) { + cache_lock_.lock(); + } + + ClearFrameCache(); + + Seek(seek_ts); + if (seek_ts == 0) { + cache_at_zero_ = true; + } + + still_seeking = true; + + if (!cache_is_locked) { + cache_lock_.unlock(); + } + } + + int ret; + AVPacket* pkt = av_packet_alloc(); + AVFrame* return_frame = nullptr; + + while (true) { + // Allocate a new frame + AVFrame* working_frame = av_frame_alloc(); + + // Pull from the decoder + ret = GetFrame(pkt, working_frame); + + // Handle any errors that aren't EOF (EOF is handled later on) + if (ret < 0 && ret != AVERROR_EOF) { + av_frame_free(&working_frame); + qCritical() << "Failed to retrieve frame:" << ret; + break; + } + + if (still_seeking) { + // Handle a failure to seek (occurs on some media) + // We'll only be here if the frame cache was emptied earlier + if (!cache_at_zero_ && (ret == AVERROR_EOF || working_frame->pts > target_ts)) { + + seek_ts = qMax(static_cast(0), seek_ts - second_ts_); + Seek(seek_ts); + if (seek_ts == 0) { + cache_at_zero_ = true; + } + av_frame_free(&working_frame); + continue; + + } else { + + still_seeking = false; + + } + } + + if (ret == AVERROR_EOF) { + + // Handle an "expected" EOF by using the last frame of our cache + cache_at_eof_ = true; + return_frame = cached_frames_.last(); + av_frame_free(&working_frame); + break; + + } else { + + // Whatever it is, keep this frame in memory for the time being just in case + if (cache_is_locked) { + cache_is_locked = false; + } else { + cache_lock_.lock(); + } + + cached_frames_.append(working_frame); + cache_wait_cond_.wakeAll(); + + cache_lock_.unlock(); + + // If this is a valid frame, see if this or the frame before it are the one we need + if (working_frame->pts == target_ts) { + return_frame = working_frame; + break; + } else if (working_frame->pts > target_ts) { + if (cached_frames_.isEmpty() && cache_at_zero_) { + return_frame = working_frame; + break; + } else { + return_frame = cached_frames_.at(cached_frames_.size() - 2); + break; + } + } + } + } + + av_packet_free(&pkt); + + return return_frame; +} + void FFmpegDecoder::ClearResources() { - if (opts_) { - av_dict_free(&opts_); - opts_ = nullptr; + if (instance_) { + delete instance_; + instance_ = nullptr; } - ClearFrameCache(); - FreeScaler(); - if (codec_ctx_) { - avcodec_free_context(&codec_ctx_); - codec_ctx_ = nullptr; - } - - if (fmt_ctx_) { - avformat_close_input(&fmt_ctx_); - fmt_ctx_ = nullptr; - } - open_ = false; } void FFmpegDecoder::SetupScaler(const int ÷r) { - scale_ctx_ = sws_getContext(avstream_->codecpar->width, - avstream_->codecpar->height, - static_cast(avstream_->codecpar->format), - avstream_->codecpar->width / divider, - avstream_->codecpar->height / divider, + VideoStream* vs = static_cast(stream().get()); + + scale_ctx_ = sws_getContext(vs->width(), + vs->height(), + src_pix_fmt_, + vs->width() / divider, + vs->height() / divider, ideal_pix_fmt_, SWS_FAST_BILINEAR, nullptr, @@ -1019,7 +1035,7 @@ void FFmpegDecoder::FreeScaler() } } -int64_t FFmpegDecoder::RangeStart() const +int64_t FFmpegDecoderInstance::RangeStart() const { if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; @@ -1027,7 +1043,7 @@ int64_t FFmpegDecoder::RangeStart() const return cached_frames_.first()->pts; } -int64_t FFmpegDecoder::RangeEnd() const +int64_t FFmpegDecoderInstance::RangeEnd() const { if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; @@ -1035,22 +1051,22 @@ int64_t FFmpegDecoder::RangeEnd() const return cached_frames_.last()->pts; } -bool FFmpegDecoder::CacheContainsTime(const int64_t &t) const +bool FFmpegDecoderInstance::CacheContainsTime(const int64_t &t) const { return (RangeStart() <= t && RangeEnd() >= t); } -bool FFmpegDecoder::CacheWillContainTime(const rational &t) const +bool FFmpegDecoderInstance::CacheWillContainTime(const int64_t &t) const { - return !cached_frames_.isEmpty() && t <= cache_target_time_; + return !cached_frames_.isEmpty() && t >= cached_frames_.first()->pts && t <= cache_target_time_; } -bool FFmpegDecoder::CacheCouldContainTime(const rational &t) const +bool FFmpegDecoderInstance::CacheCouldContainTime(const int64_t &t) const { - return !cached_frames_.isEmpty() && t <= (cache_target_time_ + 2); + return !cached_frames_.isEmpty() && t >= cached_frames_.first()->pts && t <= (cache_target_time_ + 2*second_ts_); } -AVFrame *FFmpegDecoder::GetFrameFromCache(const int64_t &t) const +AVFrame *FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const { if (t < cached_frames_.first()->pts) { @@ -1082,6 +1098,123 @@ AVFrame *FFmpegDecoder::GetFrameFromCache(const int64_t &t) const return nullptr; } +rational FFmpegDecoderInstance::sample_aspect_ratio() const +{ + return av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr); +} + +AVStream *FFmpegDecoderInstance::stream() const +{ + return avstream_; +} + +FFmpegDecoderInstance::FFmpegDecoderInstance(const char *filename, int stream_index) : + fmt_ctx_(nullptr), + opts_(nullptr), + cache_at_zero_(false), + cache_at_eof_(false) +{ + // Open file in a format context + int error_code = avformat_open_input(&fmt_ctx_, filename, nullptr, nullptr); + + // Handle format context error + if (error_code != 0) { + qDebug() << "Failed to open input:" << filename << error_code; + ClearResources(); + return; + } + + // Get stream information from format + error_code = avformat_find_stream_info(fmt_ctx_, nullptr); + + // Handle get stream information error + if (error_code < 0) { + qDebug() << "Failed to find stream info:" << error_code; + ClearResources(); + return; + } + + // Get reference to correct AVStream + avstream_ = fmt_ctx_->streams[stream_index]; + + // Find decoder + AVCodec* codec = avcodec_find_decoder(avstream_->codecpar->codec_id); + + // Handle failure to find decoder + if (codec == nullptr) { + qCritical() << "Failed to find appropriate decoder for this codec:" << filename << stream_index << avstream_->codecpar->codec_id; + ClearResources(); + return; + } + + // Allocate context for the decoder + codec_ctx_ = avcodec_alloc_context3(codec); + if (codec_ctx_ == nullptr) { + qCritical() << "Failed to allocate codec context"; + ClearResources(); + return; + } + + // Copy parameters from the AVStream to the AVCodecContext + error_code = avcodec_parameters_to_context(codec_ctx_, avstream_->codecpar); + + // Handle failure to copy parameters + if (error_code < 0) { + qCritical() << "Failed to copy parameters from AVStream to AVCodecContext"; + ClearResources(); + return; + } + + // Set multithreading setting + error_code = av_dict_set(&opts_, "threads", "auto", 0); + + // Handle failure to set multithreaded decoding + if (error_code < 0) { + qCritical() << "Failed to set codec options, performance may suffer"; + } + + // Open codec + error_code = avcodec_open2(codec_ctx_, codec, &opts_); + if (error_code < 0) { + qDebug() << "Failed to open codec" << codec->id << error_code; + ClearResources(); + return; + } + + // Store one second in the source's timebase + second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base))); +} + +FFmpegDecoderInstance::~FFmpegDecoderInstance() +{ + ClearResources(); +} + +bool FFmpegDecoderInstance::IsValid() const +{ + return codec_ctx_; +} + +void FFmpegDecoderInstance::ClearResources() +{ + ClearFrameCache(); + + if (opts_) { + av_dict_free(&opts_); + opts_ = nullptr; + } + + if (codec_ctx_) { + avcodec_free_context(&codec_ctx_); + codec_ctx_ = nullptr; + } + + if (fmt_ctx_) { + avformat_close_input(&fmt_ctx_); + fmt_ctx_ = nullptr; + } +} + /* void FFmpegDecoder::ClearTimerEvent() { diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index dc0bff919..3fcd817d1 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -40,6 +40,62 @@ extern "C" { OLIVE_NAMESPACE_ENTER +class FFmpegDecoderInstance { +public: + FFmpegDecoderInstance(const char* filename, int stream_index); + virtual ~FFmpegDecoderInstance(); + + bool IsValid() const; + + int64_t RangeStart() const; + int64_t RangeEnd() const; + bool CacheContainsTime(const int64_t& t) const; + bool CacheWillContainTime(const int64_t& t) const; + bool CacheCouldContainTime(const int64_t& t) const; + AVFrame* GetFrameFromCache(const int64_t& t) const; + + rational sample_aspect_ratio() const; + AVStream* stream() const; + + void ClearFrameCache(); + + AVFrame* RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); + + /** + * @brief Uses the FFmpeg API to retrieve a packet (stored in pkt_) and decode it (stored in frame_) + * + * @return + * + * An FFmpeg error code, or >= 0 on success + */ + int GetFrame(AVPacket* pkt, AVFrame* frame); + + QMutex* cache_lock(); + QWaitCondition* cache_wait_cond(); + + int64_t cache_target_time_; + +private: + void ClearResources(); + + void Seek(int64_t timestamp); + + AVFormatContext* fmt_ctx_; + AVCodecContext* codec_ctx_; + AVStream* avstream_; + AVDictionary* opts_; + + int64_t second_ts_; + + QWaitCondition cache_wait_cond_; + QMutex cache_lock_; + QList cached_frames_; + + bool cache_at_zero_; + bool cache_at_eof_; + +}; + /** * @brief A Decoder derivative that wraps FFmpeg functions as on Olive decoder */ @@ -88,59 +144,29 @@ private: */ void FFmpegError(int error_code); - /** - * @brief Uses the FFmpeg API to retrieve a packet (stored in pkt_) and decode it (stored in frame_) - * - * @return - * - * An FFmpeg error code, or >= 0 on success - */ - int GetFrame(AVPacket* pkt, AVFrame* frame); - virtual QString GetIndexFilename() override; void UnconditionalAudioIndex(const QAtomicInt* cancelled); - void Seek(int64_t timestamp); - void CacheFrameToDisk(AVFrame* f); - void ClearFrameCache(); - void ClearResources(); void SetupScaler(const int& divider); void FreeScaler(); - int64_t RangeStart() const; - int64_t RangeEnd() const; - bool CacheContainsTime(const int64_t& t) const; - bool CacheWillContainTime(const rational& t) const; - bool CacheCouldContainTime(const rational& t) const; - AVFrame* GetFrameFromCache(const int64_t& t) const; - - AVFormatContext* fmt_ctx_; - AVCodecContext* codec_ctx_; - AVStream* avstream_; - + AVPixelFormat src_pix_fmt_; AVPixelFormat ideal_pix_fmt_; PixelFormat::Format native_pix_fmt_; + rational time_base_; + rational aspect_ratio_; + int64_t start_time_; + SwsContext* scale_ctx_; int scale_divider_; - QWaitCondition cache_wait_cond_; - QWaitCondition cache_retrieve_cond_; - QMutex cache_lock_; - QMutex retrieve_lock_; - QList cached_frames_; - rational cache_target_time_; - bool cache_at_zero_; - bool cache_at_eof_; - - int64_t second_ts_; - - AVDictionary* opts_; + FFmpegDecoderInstance* instance_; //QTimer clear_timer_; From 4d8af6855fcb06e1737cea69f96774a6fb1db1b9 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sun, 12 Apr 2020 01:49:10 +1000 Subject: [PATCH 06/14] ffmpegdecoder: made all instances shared Decoders now share instances between them and can interact with other threads to share resources and continue the work of others. --- app/codec/ffmpeg/ffmpegdecoder.cpp | 279 +++++++++++++++-------------- app/codec/ffmpeg/ffmpegdecoder.h | 10 +- 2 files changed, 147 insertions(+), 142 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 8edd2bfa1..580dfaa4f 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -44,17 +44,16 @@ extern "C" { OLIVE_NAMESPACE_ENTER -QHash< Stream*, QList > FFmpegDecoder::instances_; +QHash< Stream*, QList > FFmpegDecoder::instances_; QMutex FFmpegDecoder::instance_lock_; FFmpegDecoder::FFmpegDecoder() : scale_ctx_(nullptr), - scale_divider_(-1), - instance_(nullptr) + scale_divider_(-1) { /* // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints - clear_timer_.setInterval(250); + clear_timer_.setInterval(2000); clear_timer_.moveToThread(qApp->thread()); connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent); */ @@ -78,16 +77,16 @@ bool FFmpegDecoder::Open() // Convert QString to a C string QByteArray fn_bytes = stream()->footage()->filename().toUtf8(); - instance_ = new FFmpegDecoderInstance(fn_bytes.constData(), stream()->index()); + FFmpegDecoderInstance* instance = new FFmpegDecoderInstance(fn_bytes.constData(), stream()->index()); - if (!instance_->IsValid()) { - delete instance_; + if (!instance->IsValid()) { + delete instance; return false; } if (stream()->type() == Stream::kVideo) { // Get an Olive compatible AVPixelFormat - src_pix_fmt_ = static_cast(instance_->stream()->codecpar->format); + src_pix_fmt_ = static_cast(instance->stream()->codecpar->format); ideal_pix_fmt_ = FFmpegCommon::GetCompatiblePixelFormat(src_pix_fmt_); // Determine which Olive native pixel format we retrieved @@ -110,13 +109,13 @@ bool FFmpegDecoder::Open() qFatal("Invalid output format"); } - aspect_ratio_ = instance_->sample_aspect_ratio(); + aspect_ratio_ = instance->sample_aspect_ratio(); //QMetaObject::invokeMethod(&clear_timer_, "start"); } - time_base_ = instance_->stream()->time_base; - start_time_ = instance_->stream()->start_time; + time_base_ = instance->stream()->time_base; + start_time_ = instance->stream()->start_time; // All allocation succeeded so we set the state to open open_ = true; @@ -124,8 +123,8 @@ bool FFmpegDecoder::Open() { QMutexLocker l(&instance_lock_); - QList list = instances_.value(stream().get()); - list.append(this); + QList list = instances_.value(stream().get()); + list.append(instance); instances_.insert(stream().get(), list); } @@ -168,129 +167,112 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid return nullptr; } - int decoder_index = instances_.value(stream().get()).indexOf(this); - int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_; - qDebug() << decoder_index << "is retrieving time" << target_ts << "!"; - + FFmpegDecoderInstance* working_instance = nullptr; AVFrame* return_frame = nullptr; - // See if our RAM cache already has a frame that matches this timestamp - if (instance_->CacheContainsTime(target_ts)) { - qDebug() << decoder_index << "found this frame in its cache"; - return_frame = instance_->GetFrameFromCache(target_ts); - } else { - // Check siblings for any others rendering at this time - qDebug() << decoder_index << "is waiting for list locker"; + // Find instance + do { QMutexLocker list_locker(&instance_lock_); - qDebug() << decoder_index << "acquired list locker"; - QList siblings = instances_.value(stream().get()); - for (int i=0;i non_ideal_contenders; - qDebug() << decoder_index << "waiting for sibling" << i << "'s cache"; - FFmpegDecoderInstance* sibling_instance = sibling->instance_; - QMutexLocker sibling_cache_locker(sibling_instance->cache_lock()); + QList instances = instances_.value(stream().get()); - qDebug() << decoder_index << "locked cache with range" << sibling_instance->RangeStart() << "-" << sibling_instance->RangeEnd(); + foreach (FFmpegDecoderInstance* i, instances) { + i->cache_lock()->lock(); - if (sibling_instance->CacheContainsTime(target_ts)) { + if (i->CacheContainsTime(target_ts)) { - qDebug() << decoder_index << "found the frame it needed in" << i << "'s cache"; + // Found our instance, allow others to enter the list - // We don't need the list anymore - qDebug() << decoder_index << "is unlocking the list locker"; list_locker.unlock(); - // Retrieve the frame we need from our sibling - return_frame = sibling_instance->GetFrameFromCache(target_ts); + // Get the frame from this cache + return_frame = i->GetFrameFromCache(target_ts); + + // Got our frame, allow cache to continue + i->cache_lock()->unlock(); break; + } else if (i->CacheWillContainTime(target_ts)) { + + // Found our instance, allow others to enter the list + list_locker.unlock(); + + do { + // Allow instance to continue to the next frame + i->cache_wait_cond()->wait(i->cache_lock()); + + // See if the cache now contains this frame, if so we'll exit this loop + if (i->CacheContainsTime(target_ts)) { + return_frame = i->GetFrameFromCache(target_ts); + } + } while (!return_frame); + + // Got our frame, allow cache to continue + i->cache_lock()->unlock(); + break; + + } else if (i->CacheCouldContainTime(target_ts)) { + + // Found our instance, allow others to enter the list + list_locker.unlock(); + + // Wait for this instance to finish working + while (i->IsWorking()) { + i->cache_wait_cond()->wait(i->cache_lock()); + } + + // Grab this instance + working_instance = i; + + // We DON'T unlock here, since we'll be starting our own retrieve + break; + + } else if (i->IsWorking()) { + + // Ignore currently working instances + i->cache_lock()->unlock(); + + } else if (i->CacheIsEmpty()) { + + // Prioritize this cache over others (leaves this instance LOCKED in case we end up using it later) + non_ideal_contenders.prepend(i); + } else { - bool cache_will_contain = sibling_instance->CacheWillContainTime(target_ts); - bool cache_could_contain = sibling_instance->CacheCouldContainTime(target_ts); + // De-prioritize this cache (leaves this instance LOCKED in case we end up using it later) + non_ideal_contenders.append(i); - if (cache_will_contain || cache_could_contain) { - // We found an instance that we can use, we no longer need the list - if (cache_will_contain) { - qDebug() << decoder_index << "found that cache" << i << "WILL contain our time (Target:" << target_ts << "Cache Target:" << sibling_instance->cache_target_time_ << "End:" << sibling_instance->RangeEnd() << ")"; - } else { - qDebug() << decoder_index << "found that cache" << i << "COULD contain our time"; - } - - qDebug() << decoder_index << "is unlocking the list locker since it found an instance to use"; - list_locker.unlock(); - - do { - if (cache_will_contain) { - - // Wait for next frame to arrive - qDebug() << decoder_index << "is WAITING for" << i << "to retrieve another frame"; - sibling_instance->cache_wait_cond()->wait(sibling_instance->cache_lock()); - qDebug() << decoder_index << "FINISHED waiting for" << i; - - // Attempt to pull frame from other thread - return_frame = sibling_instance->GetFrameFromCache(target_ts); - - } else if (cache_could_contain) { - - // Allow sibling instance to continue - qDebug() << decoder_index << "is unlocking" << i << "'s cache lock preparing for swap"; - sibling_cache_locker.unlock(); - - // Wait for sibling to finish its current job - qDebug() << decoder_index << "is waiting for sibling" << i << "to finish so we can swap"; - QMutexLocker sibling_locker(&sibling->mutex_); - - // Re-lock list - qDebug() << decoder_index << "is re-locking list locker for swap"; - list_locker.relock(); - - // Check if another thread has already swapped this isntance - if (sibling->instance_ == sibling_instance) { - - // Swap this instance with ours - qDebug() << decoder_index << "SWAPPING instance with" << i; - std::swap(sibling->instance_, instance_); - - // Continue rendering ourselves - qDebug() << decoder_index << "is continuing on its own"; - goto exit_sibling_loop; - - } else { - - // This instance has already swapped, see if they'll do anything about it - qDebug() << decoder_index << "waiting instance has already swapped, redoing loop"; - cache_will_contain = sibling_instance->CacheWillContainTime(target_ts); - continue; - - } - - } - } while (!return_frame); - } else { - qDebug() << decoder_index << "is ignoring" << i; - } } } - if (!return_frame) { -exit_sibling_loop: - // Before we unlock `list_locker`, we lock our own before we start encoding - instance_->cache_lock()->lock(); - - list_locker.unlock(); - - // If we have no RAM cache, we'll need to find this frame ourselves - return_frame = instance_->RetrieveFrame(target_ts, true); + // If we didn't find a suitable contender, grab the first non-suitable and roll with that + if (!return_frame && !working_instance && !non_ideal_contenders.isEmpty()) { + working_instance = non_ideal_contenders.takeFirst(); } + + // For all instances we left locked but didn't end up using, lock them now + foreach (FFmpegDecoderInstance* unsuitable_instance, non_ideal_contenders) { + unsuitable_instance->cache_lock()->unlock(); + } + } while (!return_frame && !working_instance); + + if (!return_frame && working_instance) { + + // This instance SHOULD remain locked from our earlier loop, making this operation safe + working_instance->SetWorking(true); + + // Retrieve frame + return_frame = working_instance->RetrieveFrame(target_ts, true); + + // Set working to false and wake any threads waiting + working_instance->cache_lock()->lock(); + working_instance->SetWorking(false); + working_instance->cache_wait_cond()->wakeAll(); + working_instance->cache_lock()->unlock(); } // We found the frame, we'll return a copy @@ -363,6 +345,7 @@ void FFmpegDecoder::Close() { QMutexLocker locker(&mutex_); + /* FIXME: Consider methods of clearing an instance (whichever is the least useful) { QMutexLocker l(&instance_lock_); @@ -370,6 +353,7 @@ void FFmpegDecoder::Close() list.removeOne(this); instances_.insert(stream().get(), list); } + */ ClearResources(); @@ -739,6 +723,8 @@ void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) int FFmpegDecoderInstance::GetFrame(AVPacket *pkt, AVFrame *frame) { + TIME_THIS_FUNCTION; + bool eof = false; int ret; @@ -792,8 +778,20 @@ QWaitCondition *FFmpegDecoderInstance::cache_wait_cond() return &cache_wait_cond_; } +bool FFmpegDecoderInstance::IsWorking() const +{ + return is_working_; +} + +void FFmpegDecoderInstance::SetWorking(bool working) +{ + is_working_ = working; +} + void FFmpegDecoderInstance::Seek(int64_t timestamp) { + TIME_THIS_FUNCTION; + avcodec_flush_buffers(codec_ctx_); av_seek_frame(fmt_ctx_, avstream_->index, timestamp, AVSEEK_FLAG_BACKWARD); } @@ -884,6 +882,10 @@ void FFmpegDecoderInstance::ClearFrameCache() AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) { + if (!cache_is_locked) { + cache_lock_.lock(); + } + int64_t seek_ts = target_ts; bool still_seeking = false; @@ -893,10 +895,6 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac if (cached_frames_.isEmpty() || target_ts < cached_frames_.first()->pts || target_ts > cached_frames_.last()->pts + 2*second_ts_) { - if (!cache_is_locked) { - cache_lock_.lock(); - } - ClearFrameCache(); Seek(seek_ts); @@ -905,10 +903,6 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac } still_seeking = true; - - if (!cache_is_locked) { - cache_lock_.unlock(); - } } int ret; @@ -925,6 +919,7 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac // Handle any errors that aren't EOF (EOF is handled later on) if (ret < 0 && ret != AVERROR_EOF) { av_frame_free(&working_frame); + cache_lock_.unlock(); qCritical() << "Failed to retrieve frame:" << ret; break; } @@ -949,10 +944,20 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac } } + if (cache_is_locked) { + cache_is_locked = false; + } else { + cache_lock_.lock(); + } + if (ret == AVERROR_EOF) { // Handle an "expected" EOF by using the last frame of our cache cache_at_eof_ = true; + + cache_wait_cond_.wakeAll(); + cache_lock_.unlock(); + return_frame = cached_frames_.last(); av_frame_free(&working_frame); break; @@ -960,15 +965,9 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac } else { // Whatever it is, keep this frame in memory for the time being just in case - if (cache_is_locked) { - cache_is_locked = false; - } else { - cache_lock_.lock(); - } - cached_frames_.append(working_frame); - cache_wait_cond_.wakeAll(); + cache_wait_cond_.wakeAll(); cache_lock_.unlock(); // If this is a valid frame, see if this or the frame before it are the one we need @@ -994,11 +993,6 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac void FFmpegDecoder::ClearResources() { - if (instance_) { - delete instance_; - instance_ = nullptr; - } - FreeScaler(); open_ = false; @@ -1053,7 +1047,10 @@ int64_t FFmpegDecoderInstance::RangeEnd() const bool FFmpegDecoderInstance::CacheContainsTime(const int64_t &t) const { - return (RangeStart() <= t && RangeEnd() >= t); + return !cached_frames_.isEmpty() + && ((RangeStart() <= t && RangeEnd() >= t) + || (cache_at_zero_ && t < cached_frames_.first()->pts) + || (cache_at_eof_ && t > cached_frames_.last()->pts)); } bool FFmpegDecoderInstance::CacheWillContainTime(const int64_t &t) const @@ -1066,6 +1063,11 @@ bool FFmpegDecoderInstance::CacheCouldContainTime(const int64_t &t) const return !cached_frames_.isEmpty() && t >= cached_frames_.first()->pts && t <= (cache_target_time_ + 2*second_ts_); } +bool FFmpegDecoderInstance::CacheIsEmpty() const +{ + return cached_frames_.isEmpty(); +} + AVFrame *FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const { if (t < cached_frames_.first()->pts) { @@ -1111,6 +1113,7 @@ AVStream *FFmpegDecoderInstance::stream() const FFmpegDecoderInstance::FFmpegDecoderInstance(const char *filename, int stream_index) : fmt_ctx_(nullptr), opts_(nullptr), + is_working_(false), cache_at_zero_(false), cache_at_eof_(false) { @@ -1215,14 +1218,12 @@ void FFmpegDecoderInstance::ClearResources() } } -/* -void FFmpegDecoder::ClearTimerEvent() +/*void FFmpegDecoder::ClearTimerEvent() { QMutexLocker locker(&mutex_); cache_at_zero_ = false; cached_frames_.remove_old_frames(QDateTime::currentMSecsSinceEpoch() - clear_timer_.interval()); -} -*/ +}*/ OLIVE_NAMESPACE_EXIT diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 3fcd817d1..4bf0509f6 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -52,6 +52,7 @@ public: bool CacheContainsTime(const int64_t& t) const; bool CacheWillContainTime(const int64_t& t) const; bool CacheCouldContainTime(const int64_t& t) const; + bool CacheIsEmpty() const; AVFrame* GetFrameFromCache(const int64_t& t) const; rational sample_aspect_ratio() const; @@ -73,6 +74,9 @@ public: QMutex* cache_lock(); QWaitCondition* cache_wait_cond(); + bool IsWorking() const; + void SetWorking(bool working); + int64_t cache_target_time_; private: @@ -91,6 +95,8 @@ private: QMutex cache_lock_; QList cached_frames_; + bool is_working_; + bool cache_at_zero_; bool cache_at_eof_; @@ -166,11 +172,9 @@ private: SwsContext* scale_ctx_; int scale_divider_; - FFmpegDecoderInstance* instance_; - //QTimer clear_timer_; - static QHash< Stream*, QList > instances_; + static QHash< Stream*, QList > instances_; static QMutex instance_lock_; private slots: From fa16d2169b263b081ba2f21a34da821075b9ed66 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sun, 12 Apr 2020 02:11:55 +1000 Subject: [PATCH 07/14] videorenderbackend: render sequentially rather than outwards --- app/render/backend/videorenderbackend.cpp | 34 +++++++++-------------- app/render/backend/videorenderbackend.h | 2 ++ 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/app/render/backend/videorenderbackend.cpp b/app/render/backend/videorenderbackend.cpp index a1f6485be..7b97ea1f5 100644 --- a/app/render/backend/videorenderbackend.cpp +++ b/app/render/backend/videorenderbackend.cpp @@ -40,7 +40,8 @@ VideoRenderBackend::VideoRenderBackend(QObject *parent) : RenderBackend(parent), operating_mode_(VideoRenderWorker::kHashRenderCache), only_signal_last_frame_requested_(true), - limit_caching_(true) + limit_caching_(true), + pop_toggle_(false) { connect(DiskManager::instance(), &DiskManager::DeletedFrame, this, &VideoRenderBackend::FrameRemovedFromDiskCache); } @@ -208,44 +209,35 @@ bool VideoRenderBackend::CanRender() TimeRange VideoRenderBackend::PopNextFrameFromQueue() { // Try to find the frame that's closest to the last time requested (the playhead) + rational earliest_allowed_time = (pop_toggle_) ? 0 : last_time_requested_; + pop_toggle_ = !pop_toggle_; // Set up playhead frame range to see if the queue contains this frame precisely - TimeRange test_range(last_time_requested_, last_time_requested_ + params_.time_base()); + TimeRange test_range(earliest_allowed_time, earliest_allowed_time + params_.time_base()); // Use this variable to find the closest frame in the range - rational closest_time = -1; + rational closest_time = RATIONAL_MAX; foreach (const TimeRange& range_here, cache_queue_) { if (range_here.OverlapsWith(test_range, false, false)) { - closest_time = -1; + closest_time = RATIONAL_MAX; break; } - for (int j=0;j<2;j++) { - rational compare; + if (range_here.in() >= earliest_allowed_time) { + rational frame_here = Timecode::snap_time_to_timebase(range_here.in(), params_.time_base()); - if (j == 0) { - compare = Timecode::snap_time_to_timebase(range_here.in(), params_.time_base()); - if (compare > range_here.in()) { - compare -= params_.time_base(); - } - } else { - compare = Timecode::snap_time_to_timebase(range_here.out(), params_.time_base()); - if (compare >= range_here.out()) { - compare -= params_.time_base(); - } + if (frame_here > range_here.in()) { + frame_here = qMax(rational(), frame_here - params_.time_base()); } - if (closest_time < 0 - || qAbs(compare - last_time_requested_) < qAbs(closest_time - last_time_requested_)) { - closest_time = compare; - } + closest_time = qMin(closest_time, frame_here); } } TimeRange frame_range; - if (closest_time == -1) { + if (closest_time == RATIONAL_MAX) { frame_range = test_range; } else { frame_range = TimeRange(closest_time, closest_time + params_.time_base()); diff --git a/app/render/backend/videorenderbackend.h b/app/render/backend/videorenderbackend.h index 522e61ced..885860537 100644 --- a/app/render/backend/videorenderbackend.h +++ b/app/render/backend/videorenderbackend.h @@ -129,6 +129,8 @@ private: bool limit_caching_; + bool pop_toggle_; + private slots: void ThreadCompletedDownload(NodeDependency dep, qint64 job_time, QByteArray hash, bool texture_existed); void ThreadSkippedFrame(NodeDependency dep, qint64 job_time, QByteArray hash); From cdf5a1058dbe3bc66e114d5c862b0fe0937bb353 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sun, 12 Apr 2020 14:12:08 +1000 Subject: [PATCH 08/14] ffmpegdecoder: re-use existing frame pool for memory management --- app/codec/ffmpeg/CMakeLists.txt | 1 + app/codec/ffmpeg/avframeptr.h | 59 +++++++++++++ app/codec/ffmpeg/ffmpegdecoder.cpp | 114 +++++++++++++------------- app/codec/ffmpeg/ffmpegdecoder.h | 24 ++++-- app/codec/ffmpeg/ffmpegframecache.cpp | 70 ++++++++++------ app/codec/ffmpeg/ffmpegframecache.h | 26 +++--- 6 files changed, 189 insertions(+), 105 deletions(-) create mode 100644 app/codec/ffmpeg/avframeptr.h diff --git a/app/codec/ffmpeg/CMakeLists.txt b/app/codec/ffmpeg/CMakeLists.txt index 9f3424e38..bee2d0e9e 100644 --- a/app/codec/ffmpeg/CMakeLists.txt +++ b/app/codec/ffmpeg/CMakeLists.txt @@ -16,6 +16,7 @@ set(OLIVE_SOURCES ${OLIVE_SOURCES} + codec/ffmpeg/avframeptr.h codec/ffmpeg/ffmpegcommon.h codec/ffmpeg/ffmpegcommon.cpp codec/ffmpeg/ffmpegdecoder.h diff --git a/app/codec/ffmpeg/avframeptr.h b/app/codec/ffmpeg/avframeptr.h new file mode 100644 index 000000000..b0f4bda95 --- /dev/null +++ b/app/codec/ffmpeg/avframeptr.h @@ -0,0 +1,59 @@ +#ifndef AVFRAMEPTR_H +#define AVFRAMEPTR_H + +extern "C" { +#include +} + +#include +#include + +#include "common/constructors.h" +#include "common/define.h" + +OLIVE_NAMESPACE_ENTER + +class AVFrameWrapper { +public: + AVFrameWrapper() { + frame_ = av_frame_alloc(); + birthtime_ = QDateTime::currentMSecsSinceEpoch(); + accessed_ = birthtime_; + } + + virtual ~AVFrameWrapper() { + av_frame_free(&frame_); + } + + DISABLE_COPY_MOVE(AVFrameWrapper) + + inline AVFrame* frame() const { + return frame_; + } + + inline const qint64& birthtime() const { + return birthtime_; + } + + inline const qint64& last_accessed() const { + return accessed_; + } + + void access() { + accessed_ = QDateTime::currentMSecsSinceEpoch(); + } + +private: + AVFrame* frame_; + + qint64 birthtime_; + + qint64 accessed_; + +}; + +using AVFramePtr = std::shared_ptr; + +OLIVE_NAMESPACE_EXIT + +#endif // AVFRAMEPTR_H diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 580dfaa4f..ecccf03b3 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -47,16 +47,16 @@ OLIVE_NAMESPACE_ENTER QHash< Stream*, QList > FFmpegDecoder::instances_; QMutex FFmpegDecoder::instance_lock_; +const int FFmpegDecoder::kMaxFrameLife = 5000; + FFmpegDecoder::FFmpegDecoder() : scale_ctx_(nullptr), scale_divider_(-1) { - /* // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints - clear_timer_.setInterval(2000); + clear_timer_.setInterval(kMaxFrameLife); clear_timer_.moveToThread(qApp->thread()); connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent); - */ } FFmpegDecoder::~FFmpegDecoder() @@ -77,16 +77,16 @@ bool FFmpegDecoder::Open() // Convert QString to a C string QByteArray fn_bytes = stream()->footage()->filename().toUtf8(); - FFmpegDecoderInstance* instance = new FFmpegDecoderInstance(fn_bytes.constData(), stream()->index()); + our_instance_ = new FFmpegDecoderInstance(fn_bytes.constData(), stream()->index()); - if (!instance->IsValid()) { - delete instance; + if (!our_instance_->IsValid()) { + delete our_instance_; return false; } if (stream()->type() == Stream::kVideo) { // Get an Olive compatible AVPixelFormat - src_pix_fmt_ = static_cast(instance->stream()->codecpar->format); + src_pix_fmt_ = static_cast(our_instance_->stream()->codecpar->format); ideal_pix_fmt_ = FFmpegCommon::GetCompatiblePixelFormat(src_pix_fmt_); // Determine which Olive native pixel format we retrieved @@ -109,13 +109,13 @@ bool FFmpegDecoder::Open() qFatal("Invalid output format"); } - aspect_ratio_ = instance->sample_aspect_ratio(); + aspect_ratio_ = our_instance_->sample_aspect_ratio(); - //QMetaObject::invokeMethod(&clear_timer_, "start"); + QMetaObject::invokeMethod(&clear_timer_, "start"); } - time_base_ = instance->stream()->time_base; - start_time_ = instance->stream()->start_time; + time_base_ = our_instance_->stream()->time_base; + start_time_ = our_instance_->stream()->start_time; // All allocation succeeded so we set the state to open open_ = true; @@ -124,7 +124,7 @@ bool FFmpegDecoder::Open() QMutexLocker l(&instance_lock_); QList list = instances_.value(stream().get()); - list.append(instance); + list.append(our_instance_); instances_.insert(stream().get(), list); } @@ -170,7 +170,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_; FFmpegDecoderInstance* working_instance = nullptr; - AVFrame* return_frame = nullptr; + AVFramePtr return_frame = nullptr; // Find instance do { @@ -181,7 +181,11 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid QList instances = instances_.value(stream().get()); foreach (FFmpegDecoderInstance* i, instances) { - i->cache_lock()->lock(); + + { + TIME_THIS_FUNCTION; + i->cache_lock()->lock(); + } if (i->CacheContainsTime(target_ts)) { @@ -297,10 +301,10 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid int output_linesize = copy->width() * PixelFormat::ChannelCount(native_pix_fmt_) * PixelFormat::BytesPerChannel(native_pix_fmt_); sws_scale(scale_ctx_, - return_frame->data, - return_frame->linesize, + return_frame->frame()->data, + return_frame->frame()->linesize, 0, - return_frame->height, + return_frame->frame()->height, &output_data, &output_linesize); @@ -357,7 +361,7 @@ void FFmpegDecoder::Close() ClearResources(); - //clear_timer_.stop(); + clear_timer_.stop(); } QString FFmpegDecoder::id() @@ -723,8 +727,6 @@ void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) int FFmpegDecoderInstance::GetFrame(AVPacket *pkt, AVFrame *frame) { - TIME_THIS_FUNCTION; - bool eof = false; int ret; @@ -790,8 +792,6 @@ void FFmpegDecoderInstance::SetWorking(bool working) void FFmpegDecoderInstance::Seek(int64_t timestamp) { - TIME_THIS_FUNCTION; - avcodec_flush_buffers(codec_ctx_); av_seek_frame(fmt_ctx_, avstream_->index, timestamp, AVSEEK_FLAG_BACKWARD); } @@ -880,7 +880,7 @@ void FFmpegDecoderInstance::ClearFrameCache() cache_at_zero_ = false; } -AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) +AVFramePtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) { if (!cache_is_locked) { cache_lock_.lock(); @@ -893,8 +893,8 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac // If the frame wasn't in the frame cache, see if this frame cache is too old to use if (cached_frames_.isEmpty() - || target_ts < cached_frames_.first()->pts - || target_ts > cached_frames_.last()->pts + 2*second_ts_) { + || target_ts < cached_frames_.first()->frame()->pts + || target_ts > cached_frames_.last()->frame()->pts + 2*second_ts_) { ClearFrameCache(); Seek(seek_ts); @@ -907,18 +907,18 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac int ret; AVPacket* pkt = av_packet_alloc(); - AVFrame* return_frame = nullptr; + AVFramePtr return_frame = nullptr; + + // Allocate a new frame + AVFrameWrapper working_frame; while (true) { - // Allocate a new frame - AVFrame* working_frame = av_frame_alloc(); // Pull from the decoder - ret = GetFrame(pkt, working_frame); + ret = GetFrame(pkt, working_frame.frame()); // Handle any errors that aren't EOF (EOF is handled later on) if (ret < 0 && ret != AVERROR_EOF) { - av_frame_free(&working_frame); cache_lock_.unlock(); qCritical() << "Failed to retrieve frame:" << ret; break; @@ -927,14 +927,13 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac if (still_seeking) { // Handle a failure to seek (occurs on some media) // We'll only be here if the frame cache was emptied earlier - if (!cache_at_zero_ && (ret == AVERROR_EOF || working_frame->pts > target_ts)) { + if (!cache_at_zero_ && (ret == AVERROR_EOF || working_frame.frame()->pts > target_ts)) { seek_ts = qMax(static_cast(0), seek_ts - second_ts_); Seek(seek_ts); if (seek_ts == 0) { cache_at_zero_ = true; } - av_frame_free(&working_frame); continue; } else { @@ -959,24 +958,23 @@ AVFrame *FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cac cache_lock_.unlock(); return_frame = cached_frames_.last(); - av_frame_free(&working_frame); break; } else { // Whatever it is, keep this frame in memory for the time being just in case - cached_frames_.append(working_frame); + AVFramePtr cached = cached_frames_.append(working_frame.frame()); cache_wait_cond_.wakeAll(); cache_lock_.unlock(); // If this is a valid frame, see if this or the frame before it are the one we need - if (working_frame->pts == target_ts) { - return_frame = working_frame; + if (cached->frame()->pts == target_ts) { + return_frame = cached; break; - } else if (working_frame->pts > target_ts) { + } else if (cached->frame()->pts > target_ts) { if (cached_frames_.isEmpty() && cache_at_zero_) { - return_frame = working_frame; + return_frame = cached; break; } else { return_frame = cached_frames_.at(cached_frames_.size() - 2); @@ -1034,7 +1032,7 @@ int64_t FFmpegDecoderInstance::RangeStart() const if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; } - return cached_frames_.first()->pts; + return cached_frames_.first()->frame()->pts; } int64_t FFmpegDecoderInstance::RangeEnd() const @@ -1042,25 +1040,25 @@ int64_t FFmpegDecoderInstance::RangeEnd() const if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; } - return cached_frames_.last()->pts; + return cached_frames_.last()->frame()->pts; } bool FFmpegDecoderInstance::CacheContainsTime(const int64_t &t) const { return !cached_frames_.isEmpty() && ((RangeStart() <= t && RangeEnd() >= t) - || (cache_at_zero_ && t < cached_frames_.first()->pts) - || (cache_at_eof_ && t > cached_frames_.last()->pts)); + || (cache_at_zero_ && t < cached_frames_.first()->frame()->pts) + || (cache_at_eof_ && t > cached_frames_.last()->frame()->pts)); } bool FFmpegDecoderInstance::CacheWillContainTime(const int64_t &t) const { - return !cached_frames_.isEmpty() && t >= cached_frames_.first()->pts && t <= cache_target_time_; + return !cached_frames_.isEmpty() && t >= cached_frames_.first()->frame()->pts && t <= cache_target_time_; } bool FFmpegDecoderInstance::CacheCouldContainTime(const int64_t &t) const { - return !cached_frames_.isEmpty() && t >= cached_frames_.first()->pts && t <= (cache_target_time_ + 2*second_ts_); + return !cached_frames_.isEmpty() && t >= cached_frames_.first()->frame()->pts && t <= (cache_target_time_ + 2*second_ts_); } bool FFmpegDecoderInstance::CacheIsEmpty() const @@ -1068,15 +1066,15 @@ bool FFmpegDecoderInstance::CacheIsEmpty() const return cached_frames_.isEmpty(); } -AVFrame *FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const +AVFramePtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const { - if (t < cached_frames_.first()->pts) { + if (t < cached_frames_.first()->frame()->pts) { if (cache_at_zero_) { return cached_frames_.first(); } - } else if (t > cached_frames_.last()->pts) { + } else if (t > cached_frames_.last()->frame()->pts) { if (cache_at_eof_) { return cached_frames_.last(); @@ -1086,10 +1084,10 @@ AVFrame *FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const // We already have this frame in the cache, find it for (int i=0;ipts == t // Test for an exact match - || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->pts > t)) { // Or for this frame to be the "closest" + if (this_frame->frame()->pts == t // Test for an exact match + || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->frame()->pts > t)) { // Or for this frame to be the "closest" return this_frame; @@ -1100,6 +1098,11 @@ AVFrame *FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const return nullptr; } +void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t) +{ + cached_frames_.remove_old_frames(t); +} + rational FFmpegDecoderInstance::sample_aspect_ratio() const { return av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr); @@ -1218,12 +1221,11 @@ void FFmpegDecoderInstance::ClearResources() } } -/*void FFmpegDecoder::ClearTimerEvent() +void FFmpegDecoder::ClearTimerEvent() { - QMutexLocker locker(&mutex_); - - cache_at_zero_ = false; - cached_frames_.remove_old_frames(QDateTime::currentMSecsSinceEpoch() - clear_timer_.interval()); -}*/ + our_instance_->cache_lock()->lock(); + our_instance_->RemoveFramesBefore(QDateTime::currentMSecsSinceEpoch() - kMaxFrameLife); + our_instance_->cache_lock()->unlock(); +} OLIVE_NAMESPACE_EXIT diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 4bf0509f6..40643c7a4 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -33,6 +33,7 @@ extern "C" { #include #include "audio/sampleformat.h" +#include "avframeptr.h" #include "codec/decoder.h" #include "codec/waveoutput.h" #include "ffmpegframecache.h" @@ -45,6 +46,8 @@ public: FFmpegDecoderInstance(const char* filename, int stream_index); virtual ~FFmpegDecoderInstance(); + DISABLE_COPY_MOVE(FFmpegDecoderInstance) + bool IsValid() const; int64_t RangeStart() const; @@ -53,14 +56,16 @@ public: bool CacheWillContainTime(const int64_t& t) const; bool CacheCouldContainTime(const int64_t& t) const; bool CacheIsEmpty() const; - AVFrame* GetFrameFromCache(const int64_t& t) const; + AVFramePtr GetFrameFromCache(const int64_t& t) const; + + void RemoveFramesBefore(const qint64& t); rational sample_aspect_ratio() const; AVStream* stream() const; void ClearFrameCache(); - AVFrame* RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); + AVFramePtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); /** * @brief Uses the FFmpeg API to retrieve a packet (stored in pkt_) and decode it (stored in frame_) @@ -77,8 +82,6 @@ public: bool IsWorking() const; void SetWorking(bool working); - int64_t cache_target_time_; - private: void ClearResources(); @@ -93,7 +96,10 @@ private: QWaitCondition cache_wait_cond_; QMutex cache_lock_; - QList cached_frames_; + //QList cached_frames_; + FFmpegFrameCache::Client cached_frames_; + + int64_t cache_target_time_; bool is_working_; @@ -172,13 +178,17 @@ private: SwsContext* scale_ctx_; int scale_divider_; - //QTimer clear_timer_; + QTimer clear_timer_; + + FFmpegDecoderInstance* our_instance_; static QHash< Stream*, QList > instances_; static QMutex instance_lock_; + static const int kMaxFrameLife; + private slots: - //void ClearTimerEvent(); + void ClearTimerEvent(); }; diff --git a/app/codec/ffmpeg/ffmpegframecache.cpp b/app/codec/ffmpeg/ffmpegframecache.cpp index 6f0a0313b..bd21411fb 100644 --- a/app/codec/ffmpeg/ffmpegframecache.cpp +++ b/app/codec/ffmpeg/ffmpegframecache.cpp @@ -26,21 +26,31 @@ OLIVE_NAMESPACE_ENTER QMutex FFmpegFrameCache::pool_lock_; -QList FFmpegFrameCache::frame_pool_; +QLinkedList FFmpegFrameCache::frame_pool_; -Frame *FFmpegFrameCache::Client::append(const VideoRenderingParams& params) +AVFramePtr FFmpegFrameCache::Client::append(int width, int height, int format) { - Frame* f = FFmpegFrameCache::Get(params); + AVFramePtr f = FFmpegFrameCache::Get(width, height, format); - frames_.append({f, QDateTime::currentMSecsSinceEpoch()}); + frames_.append(f); + + return f; +} + +AVFramePtr FFmpegFrameCache::Client::append(AVFrame *copy) +{ + AVFramePtr f = append(copy->width, copy->height, copy->format); + + av_frame_copy(f->frame(), copy); + f->frame()->pts = copy->pts; return f; } void FFmpegFrameCache::Client::clear() { - foreach (const CachedFrame& cf, frames_) { - FFmpegFrameCache::Release(cf.frame); + foreach (AVFramePtr cf, frames_) { + FFmpegFrameCache::Release(cf); } frames_.clear(); } @@ -50,19 +60,19 @@ bool FFmpegFrameCache::Client::isEmpty() const return frames_.isEmpty(); } -Frame *FFmpegFrameCache::Client::first() const +AVFramePtr FFmpegFrameCache::Client::first() const { - return frames_.first().frame; + return frames_.first(); } -Frame *FFmpegFrameCache::Client::at(int i) const +AVFramePtr FFmpegFrameCache::Client::at(int i) const { - return frames_.at(i).frame; + return frames_.at(i); } -Frame *FFmpegFrameCache::Client::last() const +AVFramePtr FFmpegFrameCache::Client::last() const { - return frames_.last().frame; + return frames_.last(); } int FFmpegFrameCache::Client::size() const @@ -72,48 +82,54 @@ int FFmpegFrameCache::Client::size() const void FFmpegFrameCache::Client::accessedFirst() { - frames_.first().accessed = QDateTime::currentMSecsSinceEpoch(); + frames_.first()->access(); } void FFmpegFrameCache::Client::accessedLast() { - frames_.last().accessed = QDateTime::currentMSecsSinceEpoch(); + frames_.last()->access(); } void FFmpegFrameCache::Client::accessed(int i) { - frames_[i].accessed = QDateTime::currentMSecsSinceEpoch(); + frames_[i]->access(); } void FFmpegFrameCache::Client::remove_old_frames(qint64 older_than) { - while (!frames_.isEmpty() && frames_.first().accessed < older_than) { - FFmpegFrameCache::Release(frames_.takeFirst().frame); + while (!frames_.isEmpty() && frames_.first()->last_accessed() < older_than) { + FFmpegFrameCache::Release(frames_.takeFirst()); } } -Frame* FFmpegFrameCache::Get(const VideoRenderingParams ¶ms) +AVFramePtr FFmpegFrameCache::Get(int width, int height, int format) { QMutexLocker locker(&pool_lock_); // See if we have a frame matching this description in the pool - for (int i=0;iwidth() == params.width() - && frame_pool_.at(i)->height() == params.height() - && frame_pool_.at(i)->format() == params.format()) { - return frame_pool_.takeAt(i); + QLinkedList::iterator i; + for (i=frame_pool_.begin();i!=frame_pool_.end();i++) { + AVFramePtr f = (*i); + + if (f->frame()->width == width + && f->frame()->height == height + && f->frame()->format == format) { + frame_pool_.erase(i); + return f; } } // Otherwise we'll need to create one - Frame* f = new Frame(); - f->set_video_params(params); - f->allocate(); + AVFramePtr f = std::make_shared(); + f->frame()->width = width; + f->frame()->height = height; + f->frame()->format = format; + av_frame_get_buffer(f->frame(), 1); return f; } -void FFmpegFrameCache::Release(Frame *f) +void FFmpegFrameCache::Release(AVFramePtr f) { QMutexLocker locker(&pool_lock_); diff --git a/app/codec/ffmpeg/ffmpegframecache.h b/app/codec/ffmpeg/ffmpegframecache.h index 20d75a5f9..bb38651be 100644 --- a/app/codec/ffmpeg/ffmpegframecache.h +++ b/app/codec/ffmpeg/ffmpegframecache.h @@ -21,10 +21,10 @@ #ifndef FFMPEGFRAMECACHE_H #define FFMPEGFRAMECACHE_H -#include +#include #include -#include "codec/frame.h" +#include "avframeptr.h" #include "render/videoparams.h" OLIVE_NAMESPACE_ENTER @@ -34,22 +34,23 @@ class FFmpegFrameCache public: FFmpegFrameCache() = default; - static Frame* Get(const VideoRenderingParams& params); + static AVFramePtr Get(int width, int height, int format); - static void Release(Frame* f); + static void Release(AVFramePtr f); class Client { public: Client() = default; - Frame* append(const VideoRenderingParams ¶ms); + AVFramePtr append(int width, int height, int format); + AVFramePtr append(AVFrame *copy); void clear(); bool isEmpty() const; - Frame* first() const; - Frame* at(int i) const; - Frame* last() const; + AVFramePtr first() const; + AVFramePtr at(int i) const; + AVFramePtr last() const; int size() const; void accessedFirst(); @@ -59,19 +60,14 @@ public: void remove_old_frames(qint64 older_than); private: - struct CachedFrame { - Frame* frame; - qint64 accessed; - }; - - QList frames_; + QList frames_; }; private: static QMutex pool_lock_; - static QList frame_pool_; + static QLinkedList frame_pool_; }; From aac47ba77603e593ca60ecc995f7c574d513b227 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 13 Apr 2020 12:58:04 +1000 Subject: [PATCH 09/14] viewer: re-queue video renderer even when beyond the length Ensures video renderer always renders where the playhead is. --- app/render/backend/videorenderbackend.cpp | 11 ++++++++--- app/render/backend/videorenderbackend.h | 2 ++ app/widget/viewer/viewer.cpp | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app/render/backend/videorenderbackend.cpp b/app/render/backend/videorenderbackend.cpp index 7b97ea1f5..c98649789 100644 --- a/app/render/backend/videorenderbackend.cpp +++ b/app/render/backend/videorenderbackend.cpp @@ -165,7 +165,7 @@ VideoRenderFrameCache *VideoRenderBackend::frame_cache() QString VideoRenderBackend::GetCachedFrame(const rational &time) { - last_time_requested_ = time; + UpdateLastRequestedTime(time); if (viewer_node() == nullptr) { // Nothing is connected - nothing to show or render @@ -182,8 +182,6 @@ QString VideoRenderBackend::GetCachedFrame(const rational &time) return nullptr; } - Requeue(); - // Find frame in map QByteArray frame_hash = frame_cache_.TimeToHash(time); @@ -196,6 +194,13 @@ QString VideoRenderBackend::GetCachedFrame(const rational &time) return QString(); } +void VideoRenderBackend::UpdateLastRequestedTime(const rational &time) +{ + last_time_requested_ = time; + + Requeue(); +} + NodeInput *VideoRenderBackend::GetDependentInput() { return viewer_node()->texture_input(); diff --git a/app/render/backend/videorenderbackend.h b/app/render/backend/videorenderbackend.h index 885860537..a73d46df2 100644 --- a/app/render/backend/videorenderbackend.h +++ b/app/render/backend/videorenderbackend.h @@ -66,6 +66,8 @@ public: QString GetCachedFrame(const rational& time); + void UpdateLastRequestedTime(const rational& time); + VideoRenderFrameCache* frame_cache(); const VideoRenderingParams& params() const; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index d8bde137b..442691e08 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -320,6 +320,7 @@ void ViewerWidget::UpdateTextureFromNode(const rational& time) { if (!GetConnectedNode() || time >= GetConnectedNode()->Length()) { main_gl_widget()->SetImage(QString()); + video_renderer_->UpdateLastRequestedTime(time); } else { QString frame_fn = video_renderer_->GetCachedFrame(time); From f80a82459ba64d99b4c6671720208318bcb0d89d Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 13 Apr 2020 13:24:30 +1000 Subject: [PATCH 10/14] ffmpegdecoder: clear cache_at_zero flag if frames are removed --- app/codec/ffmpeg/ffmpegdecoder.cpp | 11 +++++------ app/codec/ffmpeg/ffmpegframecache.cpp | 9 +++++++-- app/codec/ffmpeg/ffmpegframecache.h | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index ecccf03b3..fbbb869ee 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -47,13 +47,13 @@ OLIVE_NAMESPACE_ENTER QHash< Stream*, QList > FFmpegDecoder::instances_; QMutex FFmpegDecoder::instance_lock_; +// FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints const int FFmpegDecoder::kMaxFrameLife = 5000; FFmpegDecoder::FFmpegDecoder() : scale_ctx_(nullptr), scale_divider_(-1) { - // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints clear_timer_.setInterval(kMaxFrameLife); clear_timer_.moveToThread(qApp->thread()); connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent); @@ -182,10 +182,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid foreach (FFmpegDecoderInstance* i, instances) { - { - TIME_THIS_FUNCTION; - i->cache_lock()->lock(); - } + i->cache_lock()->lock(); if (i->CacheContainsTime(target_ts)) { @@ -1100,7 +1097,9 @@ AVFramePtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t) { - cached_frames_.remove_old_frames(t); + if (cached_frames_.remove_old_frames(t)) { + cache_at_zero_ = false; + } } rational FFmpegDecoderInstance::sample_aspect_ratio() const diff --git a/app/codec/ffmpeg/ffmpegframecache.cpp b/app/codec/ffmpeg/ffmpegframecache.cpp index bd21411fb..d64ac8c37 100644 --- a/app/codec/ffmpeg/ffmpegframecache.cpp +++ b/app/codec/ffmpeg/ffmpegframecache.cpp @@ -95,11 +95,16 @@ void FFmpegFrameCache::Client::accessed(int i) frames_[i]->access(); } -void FFmpegFrameCache::Client::remove_old_frames(qint64 older_than) +int FFmpegFrameCache::Client::remove_old_frames(qint64 older_than) { - while (!frames_.isEmpty() && frames_.first()->last_accessed() < older_than) { + int counter = 0; + + while (frames_.size() > 1 && frames_.first()->last_accessed() < older_than) { FFmpegFrameCache::Release(frames_.takeFirst()); + counter++; } + + return counter; } AVFramePtr FFmpegFrameCache::Get(int width, int height, int format) diff --git a/app/codec/ffmpeg/ffmpegframecache.h b/app/codec/ffmpeg/ffmpegframecache.h index bb38651be..084b5e869 100644 --- a/app/codec/ffmpeg/ffmpegframecache.h +++ b/app/codec/ffmpeg/ffmpegframecache.h @@ -57,7 +57,7 @@ public: void accessedLast(); void accessed(int i); - void remove_old_frames(qint64 older_than); + int remove_old_frames(qint64 older_than); private: QList frames_; From 5c927ce7a44645f39dd8ce246715164f7ceee989 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 13 Apr 2020 23:43:46 +1000 Subject: [PATCH 11/14] core: merged define and constructor headers --- app/common/CMakeLists.txt | 1 - app/common/constructors.h | 47 --------------------------------------- app/common/define.h | 17 ++++++++++++++ 3 files changed, 17 insertions(+), 48 deletions(-) delete mode 100644 app/common/constructors.h diff --git a/app/common/CMakeLists.txt b/app/common/CMakeLists.txt index e08078af0..3fd83b995 100644 --- a/app/common/CMakeLists.txt +++ b/app/common/CMakeLists.txt @@ -21,7 +21,6 @@ set(OLIVE_SOURCES common/cancelableobject.h common/channellayout.h common/clamp.h - common/constructors.h common/crashhandler.h common/crashhandler.cpp common/debug.h diff --git a/app/common/constructors.h b/app/common/constructors.h deleted file mode 100644 index 07743c41c..000000000 --- a/app/common/constructors.h +++ /dev/null @@ -1,47 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2019 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef CONSTRUCTORS_H -#define CONSTRUCTORS_H - -#include "common/define.h" - -OLIVE_NAMESPACE_ENTER - -/** - * Copy/move deleters. Similar to Q_DISABLE_COPY_MOVE, et al. but those functions are not present in Qt < 5.13 so we - * use our own functions for portability. - */ - -#define DISABLE_COPY(Class) \ - Class(const Class &) = delete;\ - Class &operator=(const Class &) = delete; - -#define DISABLE_MOVE(Class) \ - Class(Class &&) = delete; \ - Class &operator=(Class &&) = delete; - -#define DISABLE_COPY_MOVE(Class) \ - DISABLE_COPY(Class) \ - DISABLE_MOVE(Class) - -OLIVE_NAMESPACE_EXIT - -#endif // CONSTRUCTORS_H diff --git a/app/common/define.h b/app/common/define.h index 294388c17..405f34615 100644 --- a/app/common/define.h +++ b/app/common/define.h @@ -49,4 +49,21 @@ OLIVE_NAMESPACE_EXIT #define OLIVE_NS_ARG(x, y) QArgument(MACRO_VAL_AS_STR(OLIVE_NAMESPACE) "::" #x, y) +/** + * Copy/move deleters. Similar to Q_DISABLE_COPY_MOVE, et al. but those functions are not present in Qt < 5.13 so we + * use our own functions for portability. + */ + +#define DISABLE_COPY(Class) \ + Class(const Class &) = delete;\ + Class &operator=(const Class &) = delete; + +#define DISABLE_MOVE(Class) \ + Class(Class &&) = delete; \ + Class &operator=(Class &&) = delete; + +#define DISABLE_COPY_MOVE(Class) \ + DISABLE_COPY(Class) \ + DISABLE_MOVE(Class) + #endif // OLIVECOMMONDEFINE_H From 6bb972b4db9b55e447248ecc40fba22eccd01aef Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 13 Apr 2020 23:44:42 +1000 Subject: [PATCH 12/14] decoder: began implementation of a frame pool --- app/codec/CMakeLists.txt | 2 + app/codec/ffmpeg/CMakeLists.txt | 2 - app/codec/ffmpeg/avframeptr.h | 38 +++---- app/codec/ffmpeg/ffmpegdecoder.cpp | 100 +++++++++--------- app/codec/ffmpeg/ffmpegdecoder.h | 10 +- app/codec/ffmpeg/ffmpegframecache.cpp | 144 -------------------------- app/codec/ffmpeg/ffmpegframecache.h | 76 -------------- app/codec/framepool.cpp | 97 +++++++++++++++++ app/codec/framepool.h | 97 +++++++++++++++++ 9 files changed, 274 insertions(+), 292 deletions(-) delete mode 100644 app/codec/ffmpeg/ffmpegframecache.cpp delete mode 100644 app/codec/ffmpeg/ffmpegframecache.h create mode 100644 app/codec/framepool.cpp create mode 100644 app/codec/framepool.h diff --git a/app/codec/CMakeLists.txt b/app/codec/CMakeLists.txt index 791500cdd..606efc295 100644 --- a/app/codec/CMakeLists.txt +++ b/app/codec/CMakeLists.txt @@ -25,6 +25,8 @@ set(OLIVE_SOURCES codec/encoder.cpp codec/frame.h codec/frame.cpp + codec/framepool.h + codec/framepool.cpp codec/samplebuffer.h codec/samplebuffer.cpp codec/waveinput.h diff --git a/app/codec/ffmpeg/CMakeLists.txt b/app/codec/ffmpeg/CMakeLists.txt index bee2d0e9e..5cac2e216 100644 --- a/app/codec/ffmpeg/CMakeLists.txt +++ b/app/codec/ffmpeg/CMakeLists.txt @@ -23,7 +23,5 @@ set(OLIVE_SOURCES codec/ffmpeg/ffmpegdecoder.cpp codec/ffmpeg/ffmpegencoder.h codec/ffmpeg/ffmpegencoder.cpp - codec/ffmpeg/ffmpegframecache.h - codec/ffmpeg/ffmpegframecache.cpp PARENT_SCOPE ) diff --git a/app/codec/ffmpeg/avframeptr.h b/app/codec/ffmpeg/avframeptr.h index b0f4bda95..232470910 100644 --- a/app/codec/ffmpeg/avframeptr.h +++ b/app/codec/ffmpeg/avframeptr.h @@ -1,3 +1,23 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + #ifndef AVFRAMEPTR_H #define AVFRAMEPTR_H @@ -17,8 +37,6 @@ class AVFrameWrapper { public: AVFrameWrapper() { frame_ = av_frame_alloc(); - birthtime_ = QDateTime::currentMSecsSinceEpoch(); - accessed_ = birthtime_; } virtual ~AVFrameWrapper() { @@ -31,25 +49,9 @@ public: return frame_; } - inline const qint64& birthtime() const { - return birthtime_; - } - - inline const qint64& last_accessed() const { - return accessed_; - } - - void access() { - accessed_ = QDateTime::currentMSecsSinceEpoch(); - } - private: AVFrame* frame_; - qint64 birthtime_; - - qint64 accessed_; - }; using AVFramePtr = std::shared_ptr; diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index fbbb869ee..154b61877 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -57,6 +57,8 @@ FFmpegDecoder::FFmpegDecoder() : clear_timer_.setInterval(kMaxFrameLife); clear_timer_.moveToThread(qApp->thread()); connect(&clear_timer_, &QTimer::timeout, this, &FFmpegDecoder::ClearTimerEvent); + + av_buffer_pool_init(20, av_buffer_allocz); } FFmpegDecoder::~FFmpegDecoder() @@ -197,39 +199,33 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid i->cache_lock()->unlock(); break; - } else if (i->CacheWillContainTime(target_ts)) { + } else if (i->CacheWillContainTime(target_ts) || i->CacheCouldContainTime(target_ts)) { // Found our instance, allow others to enter the list list_locker.unlock(); - do { - // Allow instance to continue to the next frame - i->cache_wait_cond()->wait(i->cache_lock()); + if (i->IsWorking()) { + do { + // Allow instance to continue to the next frame + i->cache_wait_cond()->wait(i->cache_lock()); - // See if the cache now contains this frame, if so we'll exit this loop - if (i->CacheContainsTime(target_ts)) { - return_frame = i->GetFrameFromCache(target_ts); + // See if the cache now contains this frame, if so we'll exit this loop + if (i->CacheContainsTime(target_ts)) { + return_frame = i->GetFrameFromCache(target_ts); + } else if (!i->IsWorking()) { + // Grab this instance and continue it + working_instance = i; + break; + } + } while (!return_frame); + + if (working_instance != i) { + // We don't unlock if we're continuing this instance ourselves + i->cache_lock()->unlock(); } - } while (!return_frame); - - // Got our frame, allow cache to continue - i->cache_lock()->unlock(); - break; - - } else if (i->CacheCouldContainTime(target_ts)) { - - // Found our instance, allow others to enter the list - list_locker.unlock(); - - // Wait for this instance to finish working - while (i->IsWorking()) { - i->cache_wait_cond()->wait(i->cache_lock()); + } else { + working_instance = i; } - - // Grab this instance - working_instance = i; - - // We DON'T unlock here, since we'll be starting our own retrieve break; } else if (i->IsWorking()) { @@ -877,7 +873,7 @@ void FFmpegDecoderInstance::ClearFrameCache() cache_at_zero_ = false; } -AVFramePtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) +FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) { if (!cache_is_locked) { cache_lock_.lock(); @@ -889,9 +885,7 @@ AVFramePtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool c cache_target_time_ = target_ts; // If the frame wasn't in the frame cache, see if this frame cache is too old to use - if (cached_frames_.isEmpty() - || target_ts < cached_frames_.first()->frame()->pts - || target_ts > cached_frames_.last()->frame()->pts + 2*second_ts_) { + if (!CacheCouldContainTime(target_ts)) { ClearFrameCache(); Seek(seek_ts); @@ -904,7 +898,7 @@ AVFramePtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool c int ret; AVPacket* pkt = av_packet_alloc(); - AVFramePtr return_frame = nullptr; + FramePool::ElementPtr return_frame = nullptr; // Allocate a new frame AVFrameWrapper working_frame; @@ -960,16 +954,17 @@ AVFramePtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool c } else { // Whatever it is, keep this frame in memory for the time being just in case - AVFramePtr cached = cached_frames_.append(working_frame.frame()); + FramePool::ElementPtr cached = frame_pool_.Get(); + cached->set_timestamp(working_frame.frame()->pts); cache_wait_cond_.wakeAll(); cache_lock_.unlock(); // If this is a valid frame, see if this or the frame before it are the one we need - if (cached->frame()->pts == target_ts) { + if (cached->timestamp() == target_ts) { return_frame = cached; break; - } else if (cached->frame()->pts > target_ts) { + } else if (cached->timestamp() > target_ts) { if (cached_frames_.isEmpty() && cache_at_zero_) { return_frame = cached; break; @@ -1029,7 +1024,7 @@ int64_t FFmpegDecoderInstance::RangeStart() const if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; } - return cached_frames_.first()->frame()->pts; + return cached_frames_.first()->timestamp(); } int64_t FFmpegDecoderInstance::RangeEnd() const @@ -1037,25 +1032,25 @@ int64_t FFmpegDecoderInstance::RangeEnd() const if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; } - return cached_frames_.last()->frame()->pts; + return cached_frames_.last()->timestamp(); } bool FFmpegDecoderInstance::CacheContainsTime(const int64_t &t) const { return !cached_frames_.isEmpty() && ((RangeStart() <= t && RangeEnd() >= t) - || (cache_at_zero_ && t < cached_frames_.first()->frame()->pts) - || (cache_at_eof_ && t > cached_frames_.last()->frame()->pts)); + || (cache_at_zero_ && t < cached_frames_.first()->timestamp()) + || (cache_at_eof_ && t > cached_frames_.last()->timestamp())); } bool FFmpegDecoderInstance::CacheWillContainTime(const int64_t &t) const { - return !cached_frames_.isEmpty() && t >= cached_frames_.first()->frame()->pts && t <= cache_target_time_; + return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= cache_target_time_; } bool FFmpegDecoderInstance::CacheCouldContainTime(const int64_t &t) const { - return !cached_frames_.isEmpty() && t >= cached_frames_.first()->frame()->pts && t <= (cache_target_time_ + 2*second_ts_); + return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= (cache_target_time_ + 2*second_ts_); } bool FFmpegDecoderInstance::CacheIsEmpty() const @@ -1063,15 +1058,15 @@ bool FFmpegDecoderInstance::CacheIsEmpty() const return cached_frames_.isEmpty(); } -AVFramePtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const +FramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const { - if (t < cached_frames_.first()->frame()->pts) { + if (t < cached_frames_.first()->timestamp()) { if (cache_at_zero_) { return cached_frames_.first(); } - } else if (t > cached_frames_.last()->frame()->pts) { + } else if (t > cached_frames_.last()->timestamp()) { if (cache_at_eof_) { return cached_frames_.last(); @@ -1081,10 +1076,10 @@ AVFramePtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const // We already have this frame in the cache, find it for (int i=0;iframe()->pts == t // Test for an exact match - || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->frame()->pts > t)) { // Or for this frame to be the "closest" + if (this_frame->timestamp() == t // Test for an exact match + || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->timestamp() > t)) { // Or for this frame to be the "closest" return this_frame; @@ -1097,7 +1092,8 @@ AVFramePtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t) { - if (cached_frames_.remove_old_frames(t)) { + while (cached_frames_.size() > 1 && cached_frames_.first()->last_accessed() < t) { + cached_frames_.removeFirst(); cache_at_zero_ = false; } } @@ -1186,6 +1182,14 @@ FFmpegDecoderInstance::FFmpegDecoderInstance(const char *filename, int stream_in return; } + // Create buffer pool + if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO + && !frame_pool_.Allocate()) { + qDebug() << "Failed to allocate frame pool"; + ClearResources(); + return; + } + // Store one second in the source's timebase second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base))); } @@ -1218,6 +1222,8 @@ void FFmpegDecoderInstance::ClearResources() avformat_close_input(&fmt_ctx_); fmt_ctx_ = nullptr; } + + frame_pool_.Destroy(); } void FFmpegDecoder::ClearTimerEvent() diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 40643c7a4..746446f1e 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -35,8 +35,8 @@ extern "C" { #include "audio/sampleformat.h" #include "avframeptr.h" #include "codec/decoder.h" +#include "codec/framepool.h" #include "codec/waveoutput.h" -#include "ffmpegframecache.h" #include "project/item/footage/videostream.h" OLIVE_NAMESPACE_ENTER @@ -56,7 +56,7 @@ public: bool CacheWillContainTime(const int64_t& t) const; bool CacheCouldContainTime(const int64_t& t) const; bool CacheIsEmpty() const; - AVFramePtr GetFrameFromCache(const int64_t& t) const; + FramePool::ElementPtr GetFrameFromCache(const int64_t& t) const; void RemoveFramesBefore(const qint64& t); @@ -65,7 +65,7 @@ public: void ClearFrameCache(); - AVFramePtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); + FramePool::ElementPtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); /** * @brief Uses the FFmpeg API to retrieve a packet (stored in pkt_) and decode it (stored in frame_) @@ -96,8 +96,8 @@ private: QWaitCondition cache_wait_cond_; QMutex cache_lock_; - //QList cached_frames_; - FFmpegFrameCache::Client cached_frames_; + QList cached_frames_; + FramePool frame_pool_; int64_t cache_target_time_; diff --git a/app/codec/ffmpeg/ffmpegframecache.cpp b/app/codec/ffmpeg/ffmpegframecache.cpp deleted file mode 100644 index d64ac8c37..000000000 --- a/app/codec/ffmpeg/ffmpegframecache.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2019 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "ffmpegframecache.h" - -#include -#include - -OLIVE_NAMESPACE_ENTER - -QMutex FFmpegFrameCache::pool_lock_; -QLinkedList FFmpegFrameCache::frame_pool_; - -AVFramePtr FFmpegFrameCache::Client::append(int width, int height, int format) -{ - AVFramePtr f = FFmpegFrameCache::Get(width, height, format); - - frames_.append(f); - - return f; -} - -AVFramePtr FFmpegFrameCache::Client::append(AVFrame *copy) -{ - AVFramePtr f = append(copy->width, copy->height, copy->format); - - av_frame_copy(f->frame(), copy); - f->frame()->pts = copy->pts; - - return f; -} - -void FFmpegFrameCache::Client::clear() -{ - foreach (AVFramePtr cf, frames_) { - FFmpegFrameCache::Release(cf); - } - frames_.clear(); -} - -bool FFmpegFrameCache::Client::isEmpty() const -{ - return frames_.isEmpty(); -} - -AVFramePtr FFmpegFrameCache::Client::first() const -{ - return frames_.first(); -} - -AVFramePtr FFmpegFrameCache::Client::at(int i) const -{ - return frames_.at(i); -} - -AVFramePtr FFmpegFrameCache::Client::last() const -{ - return frames_.last(); -} - -int FFmpegFrameCache::Client::size() const -{ - return frames_.size(); -} - -void FFmpegFrameCache::Client::accessedFirst() -{ - frames_.first()->access(); -} - -void FFmpegFrameCache::Client::accessedLast() -{ - frames_.last()->access(); -} - -void FFmpegFrameCache::Client::accessed(int i) -{ - frames_[i]->access(); -} - -int FFmpegFrameCache::Client::remove_old_frames(qint64 older_than) -{ - int counter = 0; - - while (frames_.size() > 1 && frames_.first()->last_accessed() < older_than) { - FFmpegFrameCache::Release(frames_.takeFirst()); - counter++; - } - - return counter; -} - -AVFramePtr FFmpegFrameCache::Get(int width, int height, int format) -{ - QMutexLocker locker(&pool_lock_); - - // See if we have a frame matching this description in the pool - QLinkedList::iterator i; - for (i=frame_pool_.begin();i!=frame_pool_.end();i++) { - AVFramePtr f = (*i); - - if (f->frame()->width == width - && f->frame()->height == height - && f->frame()->format == format) { - frame_pool_.erase(i); - return f; - } - } - - // Otherwise we'll need to create one - AVFramePtr f = std::make_shared(); - f->frame()->width = width; - f->frame()->height = height; - f->frame()->format = format; - av_frame_get_buffer(f->frame(), 1); - - return f; -} - -void FFmpegFrameCache::Release(AVFramePtr f) -{ - QMutexLocker locker(&pool_lock_); - - frame_pool_.append(f); -} - -OLIVE_NAMESPACE_EXIT diff --git a/app/codec/ffmpeg/ffmpegframecache.h b/app/codec/ffmpeg/ffmpegframecache.h deleted file mode 100644 index 084b5e869..000000000 --- a/app/codec/ffmpeg/ffmpegframecache.h +++ /dev/null @@ -1,76 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2019 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef FFMPEGFRAMECACHE_H -#define FFMPEGFRAMECACHE_H - -#include -#include - -#include "avframeptr.h" -#include "render/videoparams.h" - -OLIVE_NAMESPACE_ENTER - -class FFmpegFrameCache -{ -public: - FFmpegFrameCache() = default; - - static AVFramePtr Get(int width, int height, int format); - - static void Release(AVFramePtr f); - - class Client - { - public: - Client() = default; - - AVFramePtr append(int width, int height, int format); - AVFramePtr append(AVFrame *copy); - void clear(); - - bool isEmpty() const; - AVFramePtr first() const; - AVFramePtr at(int i) const; - AVFramePtr last() const; - int size() const; - - void accessedFirst(); - void accessedLast(); - void accessed(int i); - - int remove_old_frames(qint64 older_than); - - private: - QList frames_; - - }; - -private: - static QMutex pool_lock_; - - static QLinkedList frame_pool_; - -}; - -OLIVE_NAMESPACE_EXIT - -#endif // FFMPEGFRAMECACHE_H diff --git a/app/codec/framepool.cpp b/app/codec/framepool.cpp new file mode 100644 index 000000000..d0fceaa14 --- /dev/null +++ b/app/codec/framepool.cpp @@ -0,0 +1,97 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "framepool.h" + +OLIVE_NAMESPACE_ENTER + +FramePool::FramePool() +{ + data_ = nullptr; +} + +bool FramePool::Allocate(int width, int height, PixelFormat::Format format, int nb_elements) +{ + delete [] data_; + + element_sz_ = width * height * PixelFormat::BytesPerPixel(format); + + if ((data_ = new char[element_sz_ * nb_elements])) { + // Only allocate available array if data allocation succeeded + available_.resize(nb_elements); + available_.fill(true); + + return true; + } else { + available_.clear(); + + return false; + } +} + +void FramePool::Destroy() +{ + delete [] data_; + available_.clear(); +} + +FramePool::~FramePool() +{ + delete [] data_; +} + +FramePool::ElementPtr FramePool::Get() +{ + for (int i=0;i(this, data_ + i * element_sz_); + } + } + + return nullptr; +} + +void FramePool::Release(FramePool::Element *e) +{ + quintptr offs = reinterpret_cast(e->data()); + quintptr start = reinterpret_cast(data_); + + quintptr diff = offs - start; + + int index = diff / element_sz_; + + available_.replace(index, true); +} + +FramePool::Element::Element(FramePool *parent, char *data) +{ + parent_ = parent; + data_ = data; + accessed_ = QDateTime::currentMSecsSinceEpoch(); +} + +FramePool::Element::~Element() +{ + parent_->Release(this); +} + +OLIVE_NAMESPACE_EXIT diff --git a/app/codec/framepool.h b/app/codec/framepool.h new file mode 100644 index 000000000..86da3bed2 --- /dev/null +++ b/app/codec/framepool.h @@ -0,0 +1,97 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef FRAMEPOOL_H +#define FRAMEPOOL_H + +#include +#include + +#include "render/pixelformat.h" + +OLIVE_NAMESPACE_ENTER + +class FramePool +{ +public: + FramePool(); + + ~FramePool(); + + bool Allocate(int width, int height, PixelFormat::Format format, int nb_elements); + + void Destroy(); + + DISABLE_COPY_MOVE(FramePool) + + class Element { + public: + Element(FramePool* parent, char* data); + ~Element(); + + inline char* data() const { + return data_; + } + + inline const int64_t& timestamp() const { + return timestamp_; + } + + inline void set_timestamp(const int64_t& timestamp) { + timestamp_ = timestamp; + } + + inline void access() { + accessed_ = QDateTime::currentMSecsSinceEpoch(); + } + + inline const int64_t& last_accessed() const { + return accessed_; + } + + private: + FramePool* parent_; + + char* data_; + + int64_t timestamp_; + + int64_t accessed_; + + }; + + using ElementPtr = std::shared_ptr; + + ElementPtr Get(); + + void Release(Element* e); + +private: + char* data_; + + int element_sz_; + + QVector available_; + +}; + +OLIVE_NAMESPACE_EXIT + +#endif // FRAMEPOOL_H From 2eb03bf386b2c36f4a1875c17c6f7fcc1c5e0754 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Tue, 14 Apr 2020 00:53:25 +1000 Subject: [PATCH 13/14] various: removed constructor.h include --- app/codec/decoder.h | 1 - app/codec/encoder.h | 1 - app/codec/ffmpeg/avframeptr.h | 1 - app/codec/samplebuffer.h | 1 - app/codec/waveinput.h | 1 - app/codec/waveoutput.h | 1 - app/project/item/footage/footage.h | 1 - app/project/item/item.h | 1 - app/render/backend/opengl/openglframebuffer.h | 1 - app/render/backend/opengl/opengltexture.h | 1 - app/render/backend/renderbackend.h | 1 - app/render/backend/renderworker.h | 1 - app/render/colorprocessor.h | 1 - app/task/taskmanager.h | 1 - 14 files changed, 14 deletions(-) diff --git a/app/codec/decoder.h b/app/codec/decoder.h index de617955f..07e028906 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -32,7 +32,6 @@ extern "C" { #include "codec/frame.h" #include "codec/samplebuffer.h" #include "codec/waveoutput.h" -#include "common/constructors.h" #include "common/rational.h" #include "project/item/footage/footage.h" diff --git a/app/codec/encoder.h b/app/codec/encoder.h index 7b897d15b..e5868335b 100644 --- a/app/codec/encoder.h +++ b/app/codec/encoder.h @@ -25,7 +25,6 @@ #include #include "codec/frame.h" -#include "common/constructors.h" #include "common/timerange.h" #include "render/audioparams.h" #include "render/videoparams.h" diff --git a/app/codec/ffmpeg/avframeptr.h b/app/codec/ffmpeg/avframeptr.h index 232470910..2e7716d52 100644 --- a/app/codec/ffmpeg/avframeptr.h +++ b/app/codec/ffmpeg/avframeptr.h @@ -28,7 +28,6 @@ extern "C" { #include #include -#include "common/constructors.h" #include "common/define.h" OLIVE_NAMESPACE_ENTER diff --git a/app/codec/samplebuffer.h b/app/codec/samplebuffer.h index 0a68c533f..83a2e4d9c 100644 --- a/app/codec/samplebuffer.h +++ b/app/codec/samplebuffer.h @@ -23,7 +23,6 @@ #include -#include "common/constructors.h" #include "render/audioparams.h" OLIVE_NAMESPACE_ENTER diff --git a/app/codec/waveinput.h b/app/codec/waveinput.h index bd4e7e5d8..606c1bfa7 100644 --- a/app/codec/waveinput.h +++ b/app/codec/waveinput.h @@ -23,7 +23,6 @@ #include -#include "common/constructors.h" #include "render/audioparams.h" OLIVE_NAMESPACE_ENTER diff --git a/app/codec/waveoutput.h b/app/codec/waveoutput.h index 71eb1cd8e..c3679079a 100644 --- a/app/codec/waveoutput.h +++ b/app/codec/waveoutput.h @@ -25,7 +25,6 @@ #include #include "audio/sampleformat.h" -#include "common/constructors.h" #include "render/audioparams.h" OLIVE_NAMESPACE_ENTER diff --git a/app/project/item/footage/footage.h b/app/project/item/footage/footage.h index db5fcade0..a3a026508 100644 --- a/app/project/item/footage/footage.h +++ b/app/project/item/footage/footage.h @@ -24,7 +24,6 @@ #include #include -#include "common/constructors.h" #include "common/rational.h" #include "project/item/item.h" #include "project/item/footage/audiostream.h" diff --git a/app/project/item/item.h b/app/project/item/item.h index ed2e7cbde..03739d0f8 100644 --- a/app/project/item/item.h +++ b/app/project/item/item.h @@ -28,7 +28,6 @@ #include #include -#include "common/constructors.h" #include "common/threadedobject.h" #include "common/xmlutils.h" #include "node/param.h" diff --git a/app/render/backend/opengl/openglframebuffer.h b/app/render/backend/opengl/openglframebuffer.h index 532dd8048..e7d8de9b4 100644 --- a/app/render/backend/opengl/openglframebuffer.h +++ b/app/render/backend/opengl/openglframebuffer.h @@ -23,7 +23,6 @@ #include -#include "common/constructors.h" #include "opengltexture.h" OLIVE_NAMESPACE_ENTER diff --git a/app/render/backend/opengl/opengltexture.h b/app/render/backend/opengl/opengltexture.h index 1b5562de2..b637b998c 100644 --- a/app/render/backend/opengl/opengltexture.h +++ b/app/render/backend/opengl/opengltexture.h @@ -25,7 +25,6 @@ #include #include "codec/frame.h" -#include "common/constructors.h" #include "render/pixelformat.h" OLIVE_NAMESPACE_ENTER diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index db53d503e..7b1b120fb 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -23,7 +23,6 @@ #include -#include "common/constructors.h" #include "dialog/rendercancel/rendercancel.h" #include "decodercache.h" #include "node/graph.h" diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index 224b8af08..9e85da223 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -23,7 +23,6 @@ #include -#include "common/constructors.h" #include "decodercache.h" #include "node/node.h" #include "node/output/track/track.h" diff --git a/app/render/colorprocessor.h b/app/render/colorprocessor.h index 3cbde81c8..abc7129d5 100644 --- a/app/render/colorprocessor.h +++ b/app/render/colorprocessor.h @@ -25,7 +25,6 @@ namespace OCIO = OCIO_NAMESPACE::v1; #include "codec/frame.h" -#include "common/constructors.h" #include "render/color.h" OLIVE_NAMESPACE_ENTER diff --git a/app/task/taskmanager.h b/app/task/taskmanager.h index 0095b81fc..5adcfc0a7 100644 --- a/app/task/taskmanager.h +++ b/app/task/taskmanager.h @@ -24,7 +24,6 @@ #include #include -#include "common/constructors.h" #include "task/task.h" OLIVE_NAMESPACE_ENTER From 7bfdf8af7c098f6698fadf57599079c99ec05257 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Tue, 14 Apr 2020 02:43:12 +1000 Subject: [PATCH 14/14] ffmpegframepool: implemented functional frame pool Includes splitting off core functionality to a base MemoryPool class that can be re-used later if necessary. --- app/codec/CMakeLists.txt | 2 - app/codec/ffmpeg/CMakeLists.txt | 2 + app/codec/ffmpeg/ffmpegdecoder.cpp | 125 +++++++++++--------- app/codec/ffmpeg/ffmpegdecoder.h | 19 ++- app/codec/ffmpeg/ffmpegframepool.cpp | 95 +++++++++++++++ app/codec/ffmpeg/ffmpegframepool.h | 53 +++++++++ app/codec/framepool.cpp | 97 ---------------- app/codec/framepool.h | 97 ---------------- app/common/CMakeLists.txt | 1 + app/common/memorypool.h | 165 +++++++++++++++++++++++++++ 10 files changed, 396 insertions(+), 260 deletions(-) create mode 100644 app/codec/ffmpeg/ffmpegframepool.cpp create mode 100644 app/codec/ffmpeg/ffmpegframepool.h delete mode 100644 app/codec/framepool.cpp delete mode 100644 app/codec/framepool.h create mode 100644 app/common/memorypool.h diff --git a/app/codec/CMakeLists.txt b/app/codec/CMakeLists.txt index 606efc295..791500cdd 100644 --- a/app/codec/CMakeLists.txt +++ b/app/codec/CMakeLists.txt @@ -25,8 +25,6 @@ set(OLIVE_SOURCES codec/encoder.cpp codec/frame.h codec/frame.cpp - codec/framepool.h - codec/framepool.cpp codec/samplebuffer.h codec/samplebuffer.cpp codec/waveinput.h diff --git a/app/codec/ffmpeg/CMakeLists.txt b/app/codec/ffmpeg/CMakeLists.txt index 5cac2e216..12267fe76 100644 --- a/app/codec/ffmpeg/CMakeLists.txt +++ b/app/codec/ffmpeg/CMakeLists.txt @@ -23,5 +23,7 @@ set(OLIVE_SOURCES codec/ffmpeg/ffmpegdecoder.cpp codec/ffmpeg/ffmpegencoder.h codec/ffmpeg/ffmpegencoder.cpp + codec/ffmpeg/ffmpegframepool.h + codec/ffmpeg/ffmpegframepool.cpp PARENT_SCOPE ) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 154b61877..2dcc97622 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -48,11 +48,11 @@ QHash< Stream*, QList > FFmpegDecoder::instances_; QMutex FFmpegDecoder::instance_lock_; // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints -const int FFmpegDecoder::kMaxFrameLife = 5000; +const int FFmpegDecoder::kMaxFrameLife = 2000; FFmpegDecoder::FFmpegDecoder() : scale_ctx_(nullptr), - scale_divider_(-1) + scale_divider_(0) { clear_timer_.setInterval(kMaxFrameLife); clear_timer_.moveToThread(qApp->thread()); @@ -172,7 +172,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_; FFmpegDecoderInstance* working_instance = nullptr; - AVFramePtr return_frame = nullptr; + FFmpegFramePool::ElementPtr return_frame = nullptr; // Find instance do { @@ -276,11 +276,12 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid if (return_frame) { if (divider != scale_divider_) { FreeScaler(); - SetupScaler(divider); + InitScaler(divider); } VideoStream* vs = static_cast(stream().get()); + // Create frame to return FramePtr copy = Frame::Create(); copy->set_video_params(VideoRenderingParams(vs->width() / divider, vs->height() / divider, @@ -289,15 +290,27 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid copy->set_sample_aspect_ratio(aspect_ratio_); copy->allocate(); - // Convert frame to RGBA for the rest of the pipeline + // Align buffer to data/linesize points that can be passed to sws_scale + uint8_t* input_data[4]; + int input_linesize[4]; + + av_image_fill_arrays(input_data, + input_linesize, + reinterpret_cast(return_frame->data()), + src_pix_fmt_, + vs->width(), + vs->height(), + 1); + + // Convert frame to RGB/A for the rest of the pipeline uint8_t* output_data = reinterpret_cast(copy->data()); - int output_linesize = copy->width() * PixelFormat::ChannelCount(native_pix_fmt_) * PixelFormat::BytesPerChannel(native_pix_fmt_); + int output_linesize = copy->width() * PixelFormat::BytesPerPixel(native_pix_fmt_); sws_scale(scale_ctx_, - return_frame->frame()->data, - return_frame->frame()->linesize, + input_data, + input_linesize, 0, - return_frame->frame()->height, + vs->height(), &output_data, &output_linesize); @@ -789,6 +802,8 @@ void FFmpegDecoderInstance::Seek(int64_t timestamp) av_seek_frame(fmt_ctx_, avstream_->index, timestamp, AVSEEK_FLAG_BACKWARD); } +/* OLD UNUSED CODE: Keeping this around in case the code proves useful + void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) { QFile save_frame(GetIndexFilename().append(QString::number(f->pts))); @@ -818,7 +833,7 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) } // See if we stored this frame in the disk cache - /* + QByteArray frame_loader; if (!got_frame) { QFile compressed_frame(GetIndexFilename().append(QString::number(target_ts))); @@ -841,30 +856,8 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) got_frame = true; } } - */ } - -/*void FFmpegDecoder::RemoveFirstFromFrameCache() -{ - if (cached_frames_.isEmpty()) { - return; - } - - AVFrame* first = cached_frames_.takeFirst(); - av_frame_free(&first); - cache_at_zero_ = false; -} - -void FFmpegDecoder::RemoveLastFromFrameCache() -{ - if (cached_frames_.isEmpty()) { - return; - } - - AVFrame* last = cached_frames_.takeLast(); - av_frame_free(&last); - cache_at_eof_ = false; -}*/ +*/ void FFmpegDecoderInstance::ClearFrameCache() { @@ -873,7 +866,7 @@ void FFmpegDecoderInstance::ClearFrameCache() cache_at_zero_ = false; } -FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) +FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) { if (!cache_is_locked) { cache_lock_.lock(); @@ -898,7 +891,7 @@ FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target int ret; AVPacket* pkt = av_packet_alloc(); - FramePool::ElementPtr return_frame = nullptr; + FFmpegFramePool::ElementPtr return_frame = nullptr; // Allocate a new frame AVFrameWrapper working_frame; @@ -954,9 +947,23 @@ FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target } else { // Whatever it is, keep this frame in memory for the time being just in case - FramePool::ElementPtr cached = frame_pool_.Get(); + FFmpegFramePool::ElementPtr cached = frame_pool_.Get(working_frame.frame()); + Q_ASSERT(cached); + + // Set timestamp so this frame can be identified later cached->set_timestamp(working_frame.frame()->pts); + // Store frame before just in case + FFmpegFramePool::ElementPtr previous; + if (cached_frames_.isEmpty()) { + previous = nullptr; + } else { + previous = cached_frames_.last(); + } + + // Append this frame and signal to other threads that a new frame has arrived + cached_frames_.append(cached); + cache_wait_cond_.wakeAll(); cache_lock_.unlock(); @@ -965,11 +972,11 @@ FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target return_frame = cached; break; } else if (cached->timestamp() > target_ts) { - if (cached_frames_.isEmpty() && cache_at_zero_) { + if (!previous && cache_at_zero_) { return_frame = cached; break; } else { - return_frame = cached_frames_.at(cached_frames_.size() - 2); + return_frame = previous; break; } } @@ -988,7 +995,7 @@ void FFmpegDecoder::ClearResources() open_ = false; } -void FFmpegDecoder::SetupScaler(const int ÷r) +void FFmpegDecoder::InitScaler(int divider) { VideoStream* vs = static_cast(stream().get()); @@ -1003,10 +1010,10 @@ void FFmpegDecoder::SetupScaler(const int ÷r) nullptr, nullptr); - if (!scale_ctx_) { - Error(QStringLiteral("Failed to allocate SwsContext")); - } else { + if (scale_ctx_) { scale_divider_ = divider; + } else { + scale_divider_ = 0; } } @@ -1015,7 +1022,8 @@ void FFmpegDecoder::FreeScaler() if (scale_ctx_) { sws_freeContext(scale_ctx_); scale_ctx_ = nullptr; - scale_divider_ = -1; + + scale_divider_ = 0; } } @@ -1058,17 +1066,19 @@ bool FFmpegDecoderInstance::CacheIsEmpty() const return cached_frames_.isEmpty(); } -FramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const +FFmpegFramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const { if (t < cached_frames_.first()->timestamp()) { if (cache_at_zero_) { + cached_frames_.first()->access(); return cached_frames_.first(); } } else if (t > cached_frames_.last()->timestamp()) { if (cache_at_eof_) { + cached_frames_.last()->access(); return cached_frames_.last(); } @@ -1076,11 +1086,12 @@ FramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) // We already have this frame in the cache, find it for (int i=0;itimestamp() == t // Test for an exact match || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->timestamp() > t)) { // Or for this frame to be the "closest" + this_frame->access(); return this_frame; } @@ -1092,6 +1103,7 @@ FramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t) { + // We keep one frame in memory as an identifier for what pts the decoder is up to while (cached_frames_.size() > 1 && cached_frames_.first()->last_accessed() < t) { cached_frames_.removeFirst(); cache_at_zero_ = false; @@ -1182,12 +1194,19 @@ FFmpegDecoderInstance::FFmpegDecoderInstance(const char *filename, int stream_in return; } - // Create buffer pool - if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO - && !frame_pool_.Allocate()) { - qDebug() << "Failed to allocate frame pool"; - ClearResources(); - return; + // Create frame pool + if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + + frame_pool_.SetParams(avstream_->codecpar->width, + avstream_->codecpar->height, + static_cast(avstream_->codecpar->format)); + + if (!frame_pool_.Allocate(64)) { + qDebug() << "Failed to allocate frame pool"; + ClearResources(); + return; + } + } // Store one second in the source's timebase @@ -1208,6 +1227,8 @@ void FFmpegDecoderInstance::ClearResources() { ClearFrameCache(); + frame_pool_.Destroy(); + if (opts_) { av_dict_free(&opts_); opts_ = nullptr; @@ -1222,8 +1243,6 @@ void FFmpegDecoderInstance::ClearResources() avformat_close_input(&fmt_ctx_); fmt_ctx_ = nullptr; } - - frame_pool_.Destroy(); } void FFmpegDecoder::ClearTimerEvent() diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index 746446f1e..a7939aa33 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -35,8 +35,8 @@ extern "C" { #include "audio/sampleformat.h" #include "avframeptr.h" #include "codec/decoder.h" -#include "codec/framepool.h" #include "codec/waveoutput.h" +#include "ffmpegframepool.h" #include "project/item/footage/videostream.h" OLIVE_NAMESPACE_ENTER @@ -56,7 +56,7 @@ public: bool CacheWillContainTime(const int64_t& t) const; bool CacheCouldContainTime(const int64_t& t) const; bool CacheIsEmpty() const; - FramePool::ElementPtr GetFrameFromCache(const int64_t& t) const; + FFmpegFramePool::ElementPtr GetFrameFromCache(const int64_t& t) const; void RemoveFramesBefore(const qint64& t); @@ -65,7 +65,7 @@ public: void ClearFrameCache(); - FramePool::ElementPtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); + FFmpegFramePool::ElementPtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked); /** * @brief Uses the FFmpeg API to retrieve a packet (stored in pkt_) and decode it (stored in frame_) @@ -96,8 +96,8 @@ private: QWaitCondition cache_wait_cond_; QMutex cache_lock_; - QList cached_frames_; - FramePool frame_pool_; + QList cached_frames_; + FFmpegFramePool frame_pool_; int64_t cache_target_time_; @@ -160,13 +160,13 @@ private: void UnconditionalAudioIndex(const QAtomicInt* cancelled); - void CacheFrameToDisk(AVFrame* f); - void ClearResources(); - void SetupScaler(const int& divider); + void InitScaler(int divider); void FreeScaler(); + SwsContext* scale_ctx_; + int scale_divider_; AVPixelFormat src_pix_fmt_; AVPixelFormat ideal_pix_fmt_; PixelFormat::Format native_pix_fmt_; @@ -175,9 +175,6 @@ private: rational aspect_ratio_; int64_t start_time_; - SwsContext* scale_ctx_; - int scale_divider_; - QTimer clear_timer_; FFmpegDecoderInstance* our_instance_; diff --git a/app/codec/ffmpeg/ffmpegframepool.cpp b/app/codec/ffmpeg/ffmpegframepool.cpp new file mode 100644 index 000000000..b3b99cb6f --- /dev/null +++ b/app/codec/ffmpeg/ffmpegframepool.cpp @@ -0,0 +1,95 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "ffmpegframepool.h" + +extern "C" { +#include +} + +OLIVE_NAMESPACE_ENTER + +FFmpegFramePool::FFmpegFramePool() : + width_(0), + height_(0), + format_(AV_PIX_FMT_NONE) +{ +} + +FFmpegFramePool::ElementPtr FFmpegFramePool::Get(AVFrame *copy) +{ + ElementPtr ele = MemoryPool::Get(); + + if (ele) { + av_image_copy_to_buffer(ele->data(), + GetElementSize(), + copy->data, + copy->linesize, + format_, + width_, + height_, + 1); + } + + return ele; +} + +void FFmpegFramePool::SetParams(int width, int height, AVPixelFormat format) +{ + int old_nb_elements; + + if (IsAllocated()) { + old_nb_elements = GetElementCount(); + + Destroy(); + } else { + old_nb_elements = 0; + } + + width_ = width; + height_ = height; + format_ = format; + + if (old_nb_elements) { + // Re-allocate automatically + Allocate(old_nb_elements); + } +} + +size_t FFmpegFramePool::GetElementSize() +{ + if (width_ == 0 || height_ == 0 || format_ == AV_PIX_FMT_NONE) { + return 0; + } + + int buf_sz = av_image_get_buffer_size(static_cast(format_), + width_, + height_, + 1); + + if (buf_sz < 0) { + qDebug() << "Failed to find buffer size:" << buf_sz; + return 0; + } + + return buf_sz; +} + +OLIVE_NAMESPACE_EXIT diff --git a/app/codec/ffmpeg/ffmpegframepool.h b/app/codec/ffmpeg/ffmpegframepool.h new file mode 100644 index 000000000..1239c20c1 --- /dev/null +++ b/app/codec/ffmpeg/ffmpegframepool.h @@ -0,0 +1,53 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef FFMPEGFRAMEPOOL_H +#define FFMPEGFRAMEPOOL_H + +#include "common/memorypool.h" +#include "render/pixelformat.h" +#include "render/videoparams.h" + +OLIVE_NAMESPACE_ENTER + +class FFmpegFramePool : public MemoryPool +{ +public: + FFmpegFramePool(); + + ElementPtr Get(AVFrame* copy); + + void SetParams(int width, int height, AVPixelFormat format); + +protected: + virtual size_t GetElementSize() override; + +private: + int width_; + + int height_; + + AVPixelFormat format_; + +}; + +OLIVE_NAMESPACE_EXIT + +#endif // FFMPEGFRAMEPOOL_H diff --git a/app/codec/framepool.cpp b/app/codec/framepool.cpp deleted file mode 100644 index d0fceaa14..000000000 --- a/app/codec/framepool.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2019 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#include "framepool.h" - -OLIVE_NAMESPACE_ENTER - -FramePool::FramePool() -{ - data_ = nullptr; -} - -bool FramePool::Allocate(int width, int height, PixelFormat::Format format, int nb_elements) -{ - delete [] data_; - - element_sz_ = width * height * PixelFormat::BytesPerPixel(format); - - if ((data_ = new char[element_sz_ * nb_elements])) { - // Only allocate available array if data allocation succeeded - available_.resize(nb_elements); - available_.fill(true); - - return true; - } else { - available_.clear(); - - return false; - } -} - -void FramePool::Destroy() -{ - delete [] data_; - available_.clear(); -} - -FramePool::~FramePool() -{ - delete [] data_; -} - -FramePool::ElementPtr FramePool::Get() -{ - for (int i=0;i(this, data_ + i * element_sz_); - } - } - - return nullptr; -} - -void FramePool::Release(FramePool::Element *e) -{ - quintptr offs = reinterpret_cast(e->data()); - quintptr start = reinterpret_cast(data_); - - quintptr diff = offs - start; - - int index = diff / element_sz_; - - available_.replace(index, true); -} - -FramePool::Element::Element(FramePool *parent, char *data) -{ - parent_ = parent; - data_ = data; - accessed_ = QDateTime::currentMSecsSinceEpoch(); -} - -FramePool::Element::~Element() -{ - parent_->Release(this); -} - -OLIVE_NAMESPACE_EXIT diff --git a/app/codec/framepool.h b/app/codec/framepool.h deleted file mode 100644 index 86da3bed2..000000000 --- a/app/codec/framepool.h +++ /dev/null @@ -1,97 +0,0 @@ -/*** - - Olive - Non-Linear Video Editor - Copyright (C) 2019 Olive Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -***/ - -#ifndef FRAMEPOOL_H -#define FRAMEPOOL_H - -#include -#include - -#include "render/pixelformat.h" - -OLIVE_NAMESPACE_ENTER - -class FramePool -{ -public: - FramePool(); - - ~FramePool(); - - bool Allocate(int width, int height, PixelFormat::Format format, int nb_elements); - - void Destroy(); - - DISABLE_COPY_MOVE(FramePool) - - class Element { - public: - Element(FramePool* parent, char* data); - ~Element(); - - inline char* data() const { - return data_; - } - - inline const int64_t& timestamp() const { - return timestamp_; - } - - inline void set_timestamp(const int64_t& timestamp) { - timestamp_ = timestamp; - } - - inline void access() { - accessed_ = QDateTime::currentMSecsSinceEpoch(); - } - - inline const int64_t& last_accessed() const { - return accessed_; - } - - private: - FramePool* parent_; - - char* data_; - - int64_t timestamp_; - - int64_t accessed_; - - }; - - using ElementPtr = std::shared_ptr; - - ElementPtr Get(); - - void Release(Element* e); - -private: - char* data_; - - int element_sz_; - - QVector available_; - -}; - -OLIVE_NAMESPACE_EXIT - -#endif // FRAMEPOOL_H diff --git a/app/common/CMakeLists.txt b/app/common/CMakeLists.txt index 3fd83b995..2b1d61727 100644 --- a/app/common/CMakeLists.txt +++ b/app/common/CMakeLists.txt @@ -32,6 +32,7 @@ set(OLIVE_SOURCES common/flipmodifiers.cpp common/functiontimer.h common/lerp.h + common/memorypool.h common/qtutils.h common/qtutils.cpp common/range.h diff --git a/app/common/memorypool.h b/app/common/memorypool.h new file mode 100644 index 000000000..7b2105f0b --- /dev/null +++ b/app/common/memorypool.h @@ -0,0 +1,165 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2019 Olive Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef MEMORYPOOL_H +#define MEMORYPOOL_H + +#include +#include +#include + +#include + +#include "common/define.h" + +OLIVE_NAMESPACE_ENTER + +template +class MemoryPool +{ +public: + MemoryPool() { + data_ = nullptr; + } + + ~MemoryPool() { + delete [] data_; + } + + DISABLE_COPY_MOVE(MemoryPool) + + bool Allocate(int nb_elements) { + delete [] data_; + + size_t ele_sz = GetElementSize(); + + if (!ele_sz) { + return false; + } + + if ((data_ = new char[ele_sz * nb_elements])) { + available_.resize(nb_elements); + available_.fill(true); + + return true; + } else { + available_.clear(); + + return false; + } + } + + void Destroy() { + delete [] data_; + data_ = nullptr; + + available_.clear(); + } + + inline bool IsAllocated() const { + return data_; + } + + inline int GetElementCount() const { + return available_.size(); + } + + class Element { + public: + Element(MemoryPool* parent, T* data) { + parent_ = parent; + data_ = data; + accessed_ = QDateTime::currentMSecsSinceEpoch(); + } + + ~Element() { + parent_->Release(this); + } + + inline T* data() const { + return data_; + } + + inline const int64_t& timestamp() const { + return timestamp_; + } + + inline void set_timestamp(const int64_t& timestamp) { + timestamp_ = timestamp; + } + + inline void access() { + accessed_ = QDateTime::currentMSecsSinceEpoch(); + } + + inline const int64_t& last_accessed() const { + return accessed_; + } + + private: + MemoryPool* parent_; + + T* data_; + + int64_t timestamp_; + + int64_t accessed_; + + }; + + using ElementPtr = std::shared_ptr; + + ElementPtr Get() { + for (int i=0;i(this, reinterpret_cast(data_ + i * GetElementSize())); + } + } + + // FIXME: Allocate a new "arena" + return nullptr; + } + + void Release(Element* e) { + quintptr diff = reinterpret_cast(e->data()) - reinterpret_cast(data_); + + int index = diff / GetElementSize(); + + available_.replace(index, true); + } + +protected: + virtual size_t GetElementSize() { + return sizeof(T); + } + +private: + char* data_; + + QVector available_; + +}; + +OLIVE_NAMESPACE_EXIT + +#endif // MEMORYPOOL_H