diff --git a/app/codec/decoder.h b/app/codec/decoder.h index 22635c0f6..86ffc2710 100644 --- a/app/codec/decoder.h +++ b/app/codec/decoder.h @@ -21,6 +21,7 @@ #ifndef DECODER_H #define DECODER_H +#include #include #include @@ -265,6 +266,8 @@ protected: bool open_; + QMutex mutex_; + private: void ConformInternal(SwrContext *resampler, WaveOutput *output, const char *in_data, int in_sample_count); diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index be49ce99f..3c12e118d 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -45,13 +45,10 @@ FFmpegDecoder::FFmpegDecoder() : fmt_ctx_(nullptr), codec_ctx_(nullptr), scale_ctx_(nullptr), - pkt_(nullptr), - last_retrieved_frame_(nullptr), opts_(nullptr), - multithreading_(false) +// multithreading_(false) + multithreading_(true) { - frames_[0] = nullptr; - frames_[1] = nullptr; } FFmpegDecoder::~FFmpegDecoder() @@ -61,6 +58,8 @@ FFmpegDecoder::~FFmpegDecoder() bool FFmpegDecoder::Open() { + QMutexLocker locker(&mutex_); + if (open_) { return true; } @@ -174,17 +173,10 @@ bool FFmpegDecoder::Open() Error(QStringLiteral("Failed to allocate SwsContext")); return false; } - } - pkt_ = av_packet_alloc(); - if (!pkt_) { - Error(QStringLiteral("Failed to allocate AVPacket")); - return false; + second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base))); } - frames_[0] = av_frame_alloc(); - frames_[1] = av_frame_alloc(); - // All allocation succeeded so we set the state to open open_ = true; @@ -193,7 +185,9 @@ bool FFmpegDecoder::Open() Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time) { - if (!open_ && !Open()) { + QMutexLocker locker(&mutex_); + + if (!open_) { return kFailedToOpen; } @@ -212,7 +206,10 @@ Decoder::RetrieveState FFmpegDecoder::GetRetrieveState(const rational& time) FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) { - if (!open_ && !Open()) { + QMutexLocker locker(&mutex_); + + if (!open_) { + qWarning() << "Tried to retrieve video on a decoder that's still closed"; return nullptr; } @@ -222,41 +219,42 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) int64_t target_ts = Timecode::time_to_timestamp(timecode, avstream_->time_base); - // Allocate frame that we'll return - FramePtr frame_container = Frame::Create(); - frame_container->set_width(avstream_->codecpar->width); - frame_container->set_height(avstream_->codecpar->height); - frame_container->set_format(native_pix_fmt_); - frame_container->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base)); - frame_container->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr)); - frame_container->allocate(); - bool got_frame = false; - QByteArray frame_loader; uint8_t* input_data[4]; int input_linesize[4]; - { - AVFrame* test_existing = nullptr; + // See if our RAM cache already has a frame that matches this timestamp + if (!cached_frames_.isEmpty()) { - if (frames_[0]->pts == target_ts) { - got_frame = frames_[0]; - } else if (frames_[1]->pts == target_ts) { - got_frame = frames_[1]; - } + if (target_ts >= cached_frames_.first()->pts + && target_ts <= cached_frames_.last()->pts) { - if (test_existing) { - for (int i=0;i<4;i++) { - input_data[i] = test_existing->data[i]; - input_linesize[i] = test_existing->linesize[i]; + // We already have this frame in the cache, find it + for (int i=0;ipts == target_ts // Test for an exact match + || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->pts > target_ts)) { // Or for this frame to be the "closest" + + // This frame is appropriate, return it + for (int i=0;i<4;i++) { + input_data[i] = this_frame->data[i]; + input_linesize[i] = this_frame->linesize[i]; + } + + got_frame = true; + + break; + + } } - - got_frame = true; } } // See if we stored this frame in the disk cache + /* + QByteArray frame_loader; if (!got_frame) { QFile compressed_frame(GetIndexFilename().append(QString::number(target_ts))); if (compressed_frame.exists() @@ -266,7 +264,6 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) // Read data frame_loader = qUncompress(compressed_frame.readAll()); - //frame_loader = compressed_frame.readAll(); av_image_fill_arrays(input_data, input_linesize, @@ -279,40 +276,45 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) got_frame = true; } } + */ // If we have no disk cache, we'll need to find this frame ourselves if (!got_frame) { - int64_t second_ts = qRound64(av_q2d(av_inv_q(avstream_->time_base))); - if (!last_retrieved_frame_ - || last_retrieved_frame_->pts == AV_NOPTS_VALUE - || last_retrieved_frame_->pts < target_ts - 2*second_ts - || last_retrieved_frame_->pts > target_ts) { + // If the frame wasn't in the frame cache, see if this frame cache is too old to use + if (cached_frames_.isEmpty() + || target_ts < cached_frames_.first()->pts + || target_ts > cached_frames_.last()->pts + 2*second_ts_) { + ClearFrameCache(); Seek(target_ts); } int64_t seek_ts = target_ts; int ret; + AVPacket* pkt = av_packet_alloc(); - bool frame_flipper = false; bool still_seeking = true; while (true) { - AVFrame* working_frame = frames_[frame_flipper]; + AVFrame* working_frame = av_frame_alloc(); - ret = GetFrame(pkt_, working_frame); + ret = GetFrame(pkt, working_frame); if (ret < 0) { FFmpegError(ret); + + av_frame_free(&working_frame); break; } if (still_seeking) { if (working_frame->pts > target_ts || working_frame->pts == AV_NOPTS_VALUE) { // Seek failed, try again - seek_ts -= second_ts; + seek_ts -= second_ts_; Seek(seek_ts); + + av_frame_free(&working_frame); continue; } else { still_seeking = false; @@ -323,10 +325,12 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) if (working_frame->pts == target_ts) { found_frame = working_frame; - } else if (working_frame->pts > target_ts && frames_[!frame_flipper]->pts < target_ts) { - found_frame = frames_[!frame_flipper]; + } else if (working_frame->pts > target_ts) { + found_frame = cached_frames_.last(); } + cached_frames_.append(working_frame); + if (found_frame) { // We found the frame we want got_frame = true; @@ -340,13 +344,23 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) //CacheFrameToDisk(found_frame); break; } - - frame_flipper = !frame_flipper; } + + av_packet_free(&pkt); } // If we're here and got the frame, we'll convert it and return it if (got_frame) { + + // Allocate frame that we'll return + FramePtr frame_container = Frame::Create(); + frame_container->set_width(avstream_->codecpar->width); + frame_container->set_height(avstream_->codecpar->height); + frame_container->set_format(native_pix_fmt_); + frame_container->set_timestamp(Timecode::timestamp_to_time(target_ts, avstream_->time_base)); + frame_container->set_sample_aspect_ratio(av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr)); + frame_container->allocate(); + // Convert frame to RGBA for the rest of the pipeline uint8_t* output_data = reinterpret_cast(frame_container->data()); int output_linesize = frame_container->width() * kRGBAChannels * PixelService::BytesPerChannel(native_pix_fmt_); @@ -360,6 +374,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) &output_linesize); return frame_container; + } return nullptr; @@ -367,7 +382,10 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode) FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioRenderingParams ¶ms) { - if (!open_ && !Open()) { + QMutexLocker locker(&mutex_); + + if (!open_) { + qWarning() << "Tried to retrieve audio on a decoder that's still closed"; return nullptr; } @@ -401,27 +419,14 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational & void FFmpegDecoder::Close() { + QMutexLocker locker(&mutex_); + if (opts_) { av_dict_free(&opts_); opts_ = nullptr; } - last_retrieved_frame_ = nullptr; - - if (frames_[1]) { - av_frame_free(&frames_[1]); - frames_[1] = nullptr; - } - - if (frames_[0]) { - av_frame_free(&frames_[0]); - frames_[0] = nullptr; - } - - if (pkt_) { - av_packet_free(&pkt_); - pkt_ = nullptr; - } + ClearFrameCache(); if (scale_ctx_) { sws_freeContext(scale_ctx_); @@ -638,10 +643,10 @@ void FFmpegDecoder::Index(const QAtomicInt* cancelled) input.close(); } + } else { + UnconditionalAudioIndex(cancelled); } - UnconditionalAudioIndex(pkt_, frames_[0], cancelled); - } } @@ -656,7 +661,7 @@ QString FFmpegDecoder::GetIndexFilename() .append(QString::number(avstream_->index)); } -void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const QAtomicInt* cancelled) +void FFmpegDecoder::UnconditionalAudioIndex(const QAtomicInt* cancelled) { // Iterate through each audio frame and extract the PCM data @@ -703,6 +708,8 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const FFmpegCommon::GetNativeSampleFormat(dst_sample_fmt)); WaveOutput wave_out(GetIndexFilename(), wave_params); + AVPacket* pkt = av_packet_alloc(); + AVFrame* frame = av_frame_alloc(); int ret; if (wave_out.open()) { @@ -777,10 +784,13 @@ void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame, const swr_free(&resampler); } + av_frame_free(&frame); + av_packet_free(&pkt); + Seek(0); } -void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const QAtomicInt* cancelled) +void FFmpegDecoder::UnconditionalVideoIndex(const QAtomicInt* cancelled) { VideoStreamPtr video_stream = std::static_pointer_cast(stream()); @@ -792,6 +802,8 @@ void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const // Iterate through every single frame and get each timestamp // NOTE: Expects no frames to have been read so far + AVPacket* pkt = av_packet_alloc(); + AVFrame* frame = av_frame_alloc(); int ret; while (true) { @@ -826,6 +838,9 @@ void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame, const } } + av_frame_free(&frame); + av_packet_free(&pkt); + Seek(0); } @@ -886,7 +901,7 @@ void FFmpegDecoder::ValidateVideoIndex(const QAtomicInt* cancelled) // Reset state Seek(0); - UnconditionalVideoIndex(pkt_, frames_[0], cancelled); + UnconditionalVideoIndex(cancelled); Seek(0); } @@ -926,3 +941,11 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) DiskManager::instance()->CreatedFile(save_frame.fileName(), QByteArray()); } } + +void FFmpegDecoder::ClearFrameCache() +{ + for (int i=0;i cached_frames_; - AVFrame* frames_[2]; - AVFrame* last_retrieved_frame_; + int64_t second_ts_; AVDictionary* opts_; diff --git a/app/codec/oiio/oiiodecoder.cpp b/app/codec/oiio/oiiodecoder.cpp index 898d6ea8d..38d210437 100644 --- a/app/codec/oiio/oiiodecoder.cpp +++ b/app/codec/oiio/oiiodecoder.cpp @@ -135,7 +135,7 @@ bool OIIODecoder::Open() Decoder::RetrieveState OIIODecoder::GetRetrieveState(const rational &time) { - if (!open_ && !Open()) { + if (!open_) { return kFailedToOpen; } diff --git a/app/render/backend/audio/audiobackend.cpp b/app/render/backend/audio/audiobackend.cpp index 11e859ed5..59d70245a 100644 --- a/app/render/backend/audio/audiobackend.cpp +++ b/app/render/backend/audio/audiobackend.cpp @@ -24,7 +24,7 @@ bool AudioBackend::InitInternal() // Initiate one thread per CPU core for (int i=0;iSetParameters(params()); processors_.append(processor); } diff --git a/app/render/backend/audio/audioworker.cpp b/app/render/backend/audio/audioworker.cpp index 608ca4e22..d7f445e5e 100644 --- a/app/render/backend/audio/audioworker.cpp +++ b/app/render/backend/audio/audioworker.cpp @@ -1,7 +1,7 @@ #include "audioworker.h" -AudioWorker::AudioWorker(QObject *parent) : - AudioRenderWorker(parent) +AudioWorker::AudioWorker(DecoderCache* decoder_cache, QObject *parent) : + AudioRenderWorker(decoder_cache, parent) { } diff --git a/app/render/backend/audio/audioworker.h b/app/render/backend/audio/audioworker.h index eff2bc5a5..5bc4db40e 100644 --- a/app/render/backend/audio/audioworker.h +++ b/app/render/backend/audio/audioworker.h @@ -6,7 +6,7 @@ class AudioWorker : public AudioRenderWorker { public: - AudioWorker(QObject* parent = nullptr); + AudioWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); protected: virtual void FrameToValue(StreamPtr stream, FramePtr frame, NodeValueTable* table) override; diff --git a/app/render/backend/audiorenderworker.cpp b/app/render/backend/audiorenderworker.cpp index c480b7930..ba3c3b203 100644 --- a/app/render/backend/audiorenderworker.cpp +++ b/app/render/backend/audiorenderworker.cpp @@ -2,8 +2,8 @@ #include "audio/audiomanager.h" -AudioRenderWorker::AudioRenderWorker(QObject *parent) : - RenderWorker(parent) +AudioRenderWorker::AudioRenderWorker(DecoderCache* decoder_cache, QObject *parent) : + RenderWorker(decoder_cache, parent) { } diff --git a/app/render/backend/audiorenderworker.h b/app/render/backend/audiorenderworker.h index 04aa7218e..9354959b9 100644 --- a/app/render/backend/audiorenderworker.h +++ b/app/render/backend/audiorenderworker.h @@ -7,7 +7,7 @@ class AudioRenderWorker : public RenderWorker { Q_OBJECT public: - AudioRenderWorker(QObject* parent = nullptr); + AudioRenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); void SetParameters(const AudioRenderingParams& audio_params); diff --git a/app/render/backend/opengl/openglbackend.cpp b/app/render/backend/opengl/openglbackend.cpp index d0542e0a7..5d40cba10 100644 --- a/app/render/backend/opengl/openglbackend.cpp +++ b/app/render/backend/opengl/openglbackend.cpp @@ -39,7 +39,7 @@ bool OpenGLBackend::InitInternal() // Initiate one thread per CPU core for (int i=0;iSetParameters(params()); processors_.append(processor); diff --git a/app/render/backend/opengl/openglworker.cpp b/app/render/backend/opengl/openglworker.cpp index c357eaaa5..e92005f4d 100644 --- a/app/render/backend/opengl/openglworker.cpp +++ b/app/render/backend/opengl/openglworker.cpp @@ -9,8 +9,8 @@ #include "render/colormanager.h" #include "render/pixelservice.h" -OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, QObject *parent) : - VideoRenderWorker(frame_cache, parent) +OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, DecoderCache* decoder_cache, QObject *parent) : + VideoRenderWorker(frame_cache, decoder_cache, parent) { } diff --git a/app/render/backend/opengl/openglworker.h b/app/render/backend/opengl/openglworker.h index 056441b0c..1d3d1bf64 100644 --- a/app/render/backend/opengl/openglworker.h +++ b/app/render/backend/opengl/openglworker.h @@ -12,7 +12,7 @@ class OpenGLWorker : public VideoRenderWorker { Q_OBJECT public: - OpenGLWorker(VideoRenderFrameCache* frame_cache, + OpenGLWorker(VideoRenderFrameCache* frame_cache, DecoderCache *decoder_cache, QObject* parent = nullptr); signals: diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index a5d3458b7..1113a3b3a 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -83,6 +83,8 @@ void RenderBackend::Close() threads_.clear(); processors_.clear(); + + decoder_cache_.Clear(); } const QString &RenderBackend::GetError() const @@ -375,6 +377,11 @@ void RenderBackend::SetWorkerBusyState(RenderWorker *worker, bool busy) processor_busy_state_.replace(processors_.indexOf(worker), busy); } +DecoderCache *RenderBackend::decoder_cache() +{ + return &decoder_cache_; +} + bool RenderBackend::AllProcessorsAreAvailable() const { foreach (bool busy, processor_busy_state_) { diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index 9b44c58b4..e48edbe81 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -96,10 +96,14 @@ protected: bool WorkerIsBusy(RenderWorker* worker) const; void SetWorkerBusyState(RenderWorker* worker, bool busy); + DecoderCache* decoder_cache(); + TimeRangeList cache_queue_; QVector processors_; + DecoderCache decoder_cache_; + bool compiled_; QHash render_job_info_; diff --git a/app/render/backend/rendercache.h b/app/render/backend/rendercache.h index e567de386..9194cb926 100644 --- a/app/render/backend/rendercache.h +++ b/app/render/backend/rendercache.h @@ -18,41 +18,11 @@ public: bool Has(K key) const {return values_.contains(key);} -private: - QHash values_; -}; + void Lock() { lock_.lock(); } -template -class ThreadSafeRenderCache -{ -public: - ThreadSafeRenderCache() = default; + void Unlock() { lock_.unlock(); } - void Clear() { - lock_.lock(); - values_.clear(); - lock_.unlock(); - } - - void Add(K key, V val) { - lock_.lock(); - values_.insert(key, val); - lock_.unlock(); - } - - V Get(K key) { - lock_.lock(); - V val = values_.value(key); - lock_.unlock(); - return val; - } - - bool Has(K key) { - lock_.lock(); - bool has = values_.contains(key); - lock_.unlock(); - return has; - } + QMutex* lock() { return &lock_; } private: QHash values_; diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index 0904d9804..f74569f50 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -4,9 +4,10 @@ #include "node/block/block.h" -RenderWorker::RenderWorker(QObject *parent) : +RenderWorker::RenderWorker(DecoderCache *decoder_cache, QObject *parent) : QObject(parent), - started_(false) + started_(false), + decoder_cache_(decoder_cache) { } @@ -27,8 +28,6 @@ void RenderWorker::Close() { CloseInternal(); - decoder_cache_.Clear(); - started_ = false; } @@ -61,14 +60,22 @@ StreamPtr RenderWorker::ResolveStreamFromInput(NodeInput *input) DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream) { + QMutexLocker locker(decoder_cache_->lock()); + // Access a map of Node inputs and decoder instances and retrieve a frame! - DecoderPtr decoder = decoder_cache_.Get(stream.get()); + DecoderPtr decoder = decoder_cache_->Get(stream.get()); if (decoder == nullptr && stream != nullptr) { // Create a new Decoder here decoder = Decoder::CreateFromID(stream->footage()->decoder()); decoder->set_stream(stream); - decoder_cache_.Add(stream.get(), decoder); + + if (decoder->Open()) { + decoder_cache_->Add(stream.get(), decoder); + } else { + decoder = nullptr; + qWarning() << "Failed to open decoder for" << stream->footage()->filename(); + } } return decoder; diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index f1b145481..95c040486 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -13,7 +13,7 @@ class RenderWorker : public QObject, public CancelableObject { Q_OBJECT public: - RenderWorker(QObject* parent = nullptr); + RenderWorker(DecoderCache* decoder_cache, QObject* parent = nullptr); DISABLE_COPY_MOVE(RenderWorker) @@ -62,7 +62,7 @@ private: bool started_; - DecoderCache decoder_cache_; + DecoderCache* decoder_cache_; NodeDependency path_; diff --git a/app/render/backend/videorenderworker.cpp b/app/render/backend/videorenderworker.cpp index 04bced33e..33abf0537 100644 --- a/app/render/backend/videorenderworker.cpp +++ b/app/render/backend/videorenderworker.cpp @@ -6,8 +6,8 @@ #include "project/project.h" #include "render/pixelservice.h" -VideoRenderWorker::VideoRenderWorker(VideoRenderFrameCache *frame_cache, QObject *parent) : - RenderWorker(parent), +VideoRenderWorker::VideoRenderWorker(VideoRenderFrameCache *frame_cache, DecoderCache* decoder_cache, QObject *parent) : + RenderWorker(decoder_cache, parent), frame_cache_(frame_cache), operating_mode_(kHashRenderCache) { diff --git a/app/render/backend/videorenderworker.h b/app/render/backend/videorenderworker.h index c30dbaa7d..c3de8fd16 100644 --- a/app/render/backend/videorenderworker.h +++ b/app/render/backend/videorenderworker.h @@ -41,7 +41,7 @@ public: kHashRenderCache = 0x7 }; - VideoRenderWorker(VideoRenderFrameCache* frame_cache, QObject* parent = nullptr); + VideoRenderWorker(VideoRenderFrameCache* frame_cache, DecoderCache *decoder_cache, QObject* parent = nullptr); void SetParameters(const VideoRenderingParams& video_params);