From f5de19dd6bc2d6f5c3ea24939a4705327eacd12c Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 15 Feb 2020 13:57:29 +1100 Subject: [PATCH] ffmpeg: allow decoders immediate access to frames even while indexing Previously, FFmpeg decoders needed to wait for the initial index to finish before they could retrieve frames. Now they can retrieve frames while the index is occurring, provided the appropriate frame has already been indexed (if not, the retrieve will need to wait still). --- app/codec/ffmpeg/ffmpegdecoder.cpp | 168 ++++++++++------------- app/codec/ffmpeg/ffmpegdecoder.h | 29 ++-- app/project/item/footage/stream.cpp | 5 + app/project/item/footage/stream.h | 7 +- app/project/item/footage/videostream.cpp | 91 ++++++++++++ app/project/item/footage/videostream.h | 13 ++ 6 files changed, 199 insertions(+), 114 deletions(-) diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index f5cd7e2f7..a7040cd19 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -380,8 +380,6 @@ FramePtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational & void FFmpegDecoder::Close() { - frame_index_.clear(); - if (opts_) { av_dict_free(&opts_); opts_ = nullptr; @@ -676,7 +674,7 @@ bool FFmpegDecoder::Probe(Footage *f) // Use last frame index as the duration // FIXME: Does this skip the last frame? - int64_t duration = frame_index_.last(); + int64_t duration = std::static_pointer_cast(stream())->last_frame_index_timestamp(); f->stream(0)->set_duration(duration); @@ -719,25 +717,17 @@ void FFmpegDecoder::Index() return; } - stream()->index_lock_.lock(); + QMutexLocker locker(stream()->index_process_lock()); - if (!LoadIndex()) { + if (stream()->type() == Stream::kVideo) { - // Reset state - Seek(0); + ValidateVideoIndex(); - if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - IndexVideo(pkt_, frame_); - } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - IndexAudio(pkt_, frame_); + } else if (stream()->type() == Stream::kAudio) { + if (!QFileInfo::exists(GetIndexFilename())) { + UnconditionalAudioIndex(pkt_, frame_); } - - // Reset state - Seek(0); - } - - stream()->index_lock_.unlock(); } QString FFmpegDecoder::GetIndexFilename() @@ -777,62 +767,12 @@ QString FFmpegDecoder::GetConformedFilename(const AudioRenderingParams ¶ms) return index_fn; } -bool FFmpegDecoder::LoadIndex() -{ - switch (avstream_->codecpar->codec_type) { - case AVMEDIA_TYPE_VIDEO: - { - // Load index from file - QFile index_file(GetIndexFilename()); - - if (!index_file.exists()) { - return false; - } - - if (index_file.open(QFile::ReadOnly)) { - // Resize based on filesize - frame_index_.resize(static_cast(static_cast(index_file.size()) / sizeof(int64_t))); - - // Read frame index into vector - index_file.read(reinterpret_cast(frame_index_.data()), - index_file.size()); - - index_file.close(); - - return true; - } - break; - } - case AVMEDIA_TYPE_AUDIO: - { - return QFileInfo::exists(GetIndexFilename()); - } - default: - break; - } - - return false; -} - -void FFmpegDecoder::SaveIndex() -{ - // Save index to file - QFile index_file(GetIndexFilename()); - if (index_file.open(QFile::WriteOnly)) { - // Write index in binary - index_file.write(reinterpret_cast(frame_index_.constData()), - frame_index_.size() * static_cast(sizeof(int64_t))); - - index_file.close(); - } else { - qWarning() << QStringLiteral("Failed to save index for %1").arg(stream()->footage()->filename()); - } -} - -void FFmpegDecoder::IndexAudio(AVPacket *pkt, AVFrame *frame) +void FFmpegDecoder::UnconditionalAudioIndex(AVPacket *pkt, AVFrame *frame) { // Iterate through each audio frame and extract the PCM data + Seek(0); + uint64_t channel_layout = avstream_->codecpar->channel_layout; if (!channel_layout) { if (!avstream_->codecpar->channels) { @@ -925,12 +865,18 @@ void FFmpegDecoder::IndexAudio(AVPacket *pkt, AVFrame *frame) if (resampler != nullptr) { swr_free(&resampler); } + + Seek(0); } -void FFmpegDecoder::IndexVideo(AVPacket* pkt, AVFrame* frame) +void FFmpegDecoder::UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame) { + VideoStreamPtr video_stream = std::static_pointer_cast(stream()); + + Seek(0); + // This should be unnecessary, but just in case... - frame_index_.clear(); + video_stream->clear_frame_index(); // Iterate through every single frame and get each timestamp // NOTE: Expects no frames to have been read so far @@ -941,7 +887,7 @@ void FFmpegDecoder::IndexVideo(AVPacket* pkt, AVFrame* frame) ret = GetFrame(pkt, frame); if (ret >= 0) { - frame_index_.append(frame->pts); + video_stream->append_frame_index(frame->pts); } else { // Assume we've reached the end of the file break; @@ -949,7 +895,11 @@ void FFmpegDecoder::IndexVideo(AVPacket* pkt, AVFrame* frame) } // Save index to file - SaveIndex(); + if (!video_stream->save_frame_index(GetIndexFilename())) { + qWarning() << QStringLiteral("Failed to save index for %1").arg(stream()->footage()->filename()); + } + + Seek(0); } int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame) @@ -999,31 +949,65 @@ int FFmpegDecoder::GetFrame(AVPacket *pkt, AVFrame *frame) int64_t FFmpegDecoder::GetClosestTimestampInIndex(const int64_t &ts) { - // Index now if we haven't already - if (frame_index_.isEmpty()) { - Index(); - } + VideoStreamPtr video_stream = std::static_pointer_cast(stream()); - if (frame_index_.isEmpty()) { - return -1; - } + bool index_is_being_created = false; - if (ts <= 0) { - return frame_index_.first(); - } + // Check if the frame index has been populated + if (video_stream->is_frame_index_empty()) { - // Use index to find closest frame in file - for (int i=1;iindex_process_lock()->tryLock()) { - if (this_ts == ts) { - return ts; - } else if (this_ts > ts) { - return frame_index_.at(i - 1); + // If not, make an index + ValidateVideoIndex(); + + video_stream->index_process_lock()->unlock(); + + // If the index is still empty, the video must just be empty + if (video_stream->is_frame_index_empty()) { + return -1; + } + } else { + // The index is being created in another thread, wait until we have more information + index_is_being_created = true; } } - return frame_index_.last(); + int64_t closest_ts = -1; + + do { + if (index_is_being_created && video_stream->index_process_lock()->tryLock()) { + index_is_being_created = false; + video_stream->index_process_lock()->unlock(); + } + + // FIXME: Wait for update from index + //WaitForUpdate(); + + closest_ts = video_stream->get_closest_timestamp_in_frame_index(ts); + + } while (index_is_being_created); + + return closest_ts; +} + +void FFmpegDecoder::ValidateVideoIndex() +{ + VideoStreamPtr video_stream = std::static_pointer_cast(stream()); + + if (video_stream->is_frame_index_empty()) { + video_stream->load_frame_index(GetIndexFilename()); + } + + if (video_stream->is_frame_index_empty()) { + // Reset state + Seek(0); + + UnconditionalVideoIndex(pkt_, frame_); + + Seek(0); + } } void FFmpegDecoder::Seek(int64_t timestamp) diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index e88010957..553518884 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -32,12 +32,14 @@ extern "C" { #include "audio/sampleformat.h" #include "codec/decoder.h" #include "codec/waveoutput.h" +#include "project/item/footage/videostream.h" /** * @brief A Decoder derivative that wraps FFmpeg functions as on Olive decoder */ class FFmpegDecoder : public Decoder { + Q_OBJECT public: // Constructor FFmpegDecoder(); @@ -63,6 +65,8 @@ public: void SetMultithreading(bool e); + virtual void Index() override; + private: void ConformInternal(SwrContext *resampler, WaveOutput *output, const char *in_data, int in_sample_count); @@ -94,8 +98,6 @@ private: */ int GetFrame(AVPacket* pkt, AVFrame* frame); - virtual void Index() override; - /** * @brief Returns the filename for the index * @@ -108,26 +110,13 @@ private: */ QString GetConformedFilename(const AudioRenderingParams ¶ms); - /** - * @brief Used internally to load a frame index into frame_index_ - * - * @return - * - * TRUE if a frame index was successfully loaded. FALSE usually means the file didn't exist and Index() should be - * run to create it. - */ - bool LoadIndex(); - - /** - * @brief Used in Index() to save the just created frame index to a file that can be loaded later - */ - void SaveIndex(); - - void IndexAudio(AVPacket* pkt, AVFrame* frame); - void IndexVideo(AVPacket* pkt, AVFrame* frame); + void UnconditionalAudioIndex(AVPacket* pkt, AVFrame* frame); + void UnconditionalVideoIndex(AVPacket* pkt, AVFrame* frame); int64_t GetClosestTimestampInIndex(const int64_t& ts); + void ValidateVideoIndex(); + void Seek(int64_t timestamp); AVFormatContext* fmt_ctx_; @@ -145,8 +134,6 @@ private: AVDictionary* opts_; - QVector frame_index_; - bool multithreading_; }; diff --git a/app/project/item/footage/stream.cpp b/app/project/item/footage/stream.cpp index 180467140..c7f273271 100644 --- a/app/project/item/footage/stream.cpp +++ b/app/project/item/footage/stream.cpp @@ -140,6 +140,11 @@ StreamID Stream::ToID() const return StreamID(footage_->filename(), index_); } +QMutex* Stream::index_process_lock() +{ + return &index_process_lock_; +} + void Stream::FootageSetEvent(Footage*) { } diff --git a/app/project/item/footage/stream.h b/app/project/item/footage/stream.h index e90b48e68..0a0a7a319 100644 --- a/app/project/item/footage/stream.h +++ b/app/project/item/footage/stream.h @@ -103,7 +103,7 @@ public: StreamID ToID() const; - QMutex index_lock_; + QMutex* index_process_lock(); protected: virtual void FootageSetEvent(Footage*); @@ -112,6 +112,9 @@ protected: virtual void SaveCustomParameters(QXmlStreamWriter* writer) const; +signals: + void IndexChanged(); + private: Footage* footage_; @@ -125,6 +128,8 @@ private: bool enabled_; + QMutex index_process_lock_; + }; using StreamPtr = std::shared_ptr; diff --git a/app/project/item/footage/videostream.cpp b/app/project/item/footage/videostream.cpp index 4f7ae7d1c..196dee576 100644 --- a/app/project/item/footage/videostream.cpp +++ b/app/project/item/footage/videostream.cpp @@ -20,6 +20,8 @@ #include "videostream.h" +#include + VideoStream::VideoStream() { set_type(kVideo); @@ -41,3 +43,92 @@ void VideoStream::set_frame_rate(const rational &frame_rate) { frame_rate_ = frame_rate; } + +int64_t VideoStream::get_closest_timestamp_in_frame_index(const int64_t &ts) +{ + QMutexLocker locker(&index_access_lock_); + + if (ts <= 0) { + return frame_index_.first(); + } + + // Use index to find closest frame in file + for (int i=0;i ts) { + return frame_index_.at(i - 1); + } + } + + return frame_index_.last(); +} + +void VideoStream::clear_frame_index() +{ + QMutexLocker locker(&index_access_lock_); + + frame_index_.clear(); +} + +void VideoStream::append_frame_index(const int64_t &ts) +{ + QMutexLocker locker(&index_access_lock_); + + frame_index_.append(ts); +} + +bool VideoStream::is_frame_index_empty() +{ + QMutexLocker locker(&index_access_lock_); + + return frame_index_.isEmpty(); +} + +int64_t VideoStream::last_frame_index_timestamp() +{ + QMutexLocker locker(&index_access_lock_); + + return frame_index_.last(); +} + +bool VideoStream::load_frame_index(const QString &s) +{ + // Load index from file + QFile index_file(s); + + if (index_file.exists() && index_file.open(QFile::ReadOnly)) { + QMutexLocker locker(&index_access_lock_); + + // Resize based on filesize + frame_index_.resize(static_cast(index_file.size()) / sizeof(int64_t)); + + // Read frame index into vector + index_file.read(reinterpret_cast(frame_index_.data()), + index_file.size()); + + index_file.close(); + } + + return false; +} + +bool VideoStream::save_frame_index(const QString &s) +{ + QFile index_file(s); + + if (index_file.open(QFile::WriteOnly)) { + // Write index in binary + QMutexLocker locker(&index_access_lock_); + + index_file.write(reinterpret_cast(frame_index_.constData()), + frame_index_.size() * static_cast(sizeof(int64_t))); + + index_file.close(); + return true; + } + + return false; +} diff --git a/app/project/item/footage/videostream.h b/app/project/item/footage/videostream.h index addd2f9cb..2940cfd26 100644 --- a/app/project/item/footage/videostream.h +++ b/app/project/item/footage/videostream.h @@ -38,9 +38,22 @@ public: const rational& frame_rate() const; void set_frame_rate(const rational& frame_rate); + int64_t get_closest_timestamp_in_frame_index(const int64_t& ts); + void clear_frame_index(); + void append_frame_index(const int64_t& ts); + bool is_frame_index_empty(); + int64_t last_frame_index_timestamp(); + + bool load_frame_index(const QString& s); + bool save_frame_index(const QString& s); + private: rational frame_rate_; + QVector frame_index_; + + QMutex index_access_lock_; + }; using VideoStreamPtr = std::shared_ptr;