From 2cf0b24e0fcb375351896b97b9bdf025a45cb380 Mon Sep 17 00:00:00 2001 From: itsmattkc <34096995+itsmattkc@users.noreply.github.com> Date: Mon, 9 May 2022 19:04:59 -0700 Subject: [PATCH] framehashcache: refactored to remove hashes --- app/render/framehashcache.cpp | 254 +++++++++------------------------- app/render/framehashcache.h | 43 ++---- app/render/playbackcache.cpp | 8 +- 3 files changed, 80 insertions(+), 225 deletions(-) diff --git a/app/render/framehashcache.cpp b/app/render/framehashcache.cpp index 269b5bf19..1c175a4d5 100644 --- a/app/render/framehashcache.cpp +++ b/app/render/framehashcache.cpp @@ -37,7 +37,7 @@ namespace olive { QMutex FrameHashCache::currently_saving_frames_mutex_; -QMap FrameHashCache::currently_saving_frames_; +QMap FrameHashCache::currently_saving_frames_; const QString FrameHashCache::kCacheFormatExtension = QStringLiteral(".exr"); #define super PlaybackCache @@ -51,127 +51,63 @@ FrameHashCache::FrameHashCache(QObject *parent) : } } -QByteArray FrameHashCache::GetHash(const int64_t &time) -{ - if (time < GetMapSize()) { - return time_hash_map_.at(time); - } else { - return QByteArray(); - } -} - -QByteArray FrameHashCache::GetHash(const rational &time) -{ - return GetHash(ToTimestamp(time)); -} - -void FrameHashCache::SetHash(const rational &time, const QByteArray &hash, bool frame_exists) -{ - int64_t ts = ToTimestamp(time); - if (ts >= GetMapSize()) { - // Disabled: bizarrely causes the whole app to hang indefinitely when used - // Reserve an extra minute to cut down on the amount of reallocations to make - //time_hash_map_.reserve(ts + timebase_.flipped().toDouble() * 60); - - // Add enough entries to insert this hash - time_hash_map_.resize(ts + 1); - } - time_hash_map_[ts] = hash; - hash_time_map_[hash].push_back(ts); - - TimeRange validated_range; - if (frame_exists) { - validated_range = TimeRange(time, time + timebase_); - Validate(validated_range); - } -} - void FrameHashCache::SetTimebase(const rational &tb) { timebase_ = tb; } -void FrameHashCache::ValidateFramesWithHash(const QByteArray &hash) +void FrameHashCache::ValidateTimestamp(const int64_t &ts) { - auto invalidated_ranges = GetInvalidatedRanges(ToTime(GetMapSize())); - - const std::vector × = hash_time_map_[hash]; - for (auto it=times.cbegin(); it!=times.cend(); it++) { - const int64_t &i = *it; - - TimeRange frame_range(ToTime(i), ToTime(i+1)); - - if (invalidated_ranges.contains(frame_range)) { - Validate(frame_range); - } - } + TimeRange frame_range(ToTime(ts), ToTime(ts+1)); + Validate(frame_range); } -bool FrameHashCache::SaveCacheFrame(const QByteArray& hash, - char* data, - const VideoParams& vparam, - int linesize_bytes) const +bool FrameHashCache::SaveCacheFrame(const int64_t &time, FramePtr frame) const { - return SaveCacheFrame(GetCacheDirectory(), hash, data, vparam, linesize_bytes); + return SaveCacheFrame(GetCacheDirectory(), uuid_, time, frame); } -bool FrameHashCache::SaveCacheFrame(const QByteArray &hash, FramePtr frame) const -{ - return SaveCacheFrame(GetCacheDirectory(), hash, frame); -} - -bool FrameHashCache::SaveCacheFrame(const QString &cache_path, const QByteArray &hash, char *data, const VideoParams &vparam, int linesize_bytes) +bool FrameHashCache::SaveCacheFrame(const QString &cache_path, const QUuid &uuid, const int64_t &time, FramePtr frame) { if (cache_path.isEmpty()) { qWarning() << "Failed to save cache frame with empty path"; return false; } - QString fn = CachePathName(cache_path, hash); + QString fn = CachePathName(cache_path, uuid, time); - if (SaveCacheFrame(fn, data, vparam, linesize_bytes)) { - // Register frame with the disk manager + QMutexLocker locker(¤tly_saving_frames_mutex_); + currently_saving_frames_.insert(fn, frame); + locker.unlock(); + + bool ret = SaveCacheFrame(fn, frame); + + locker.relock(); + currently_saving_frames_.remove(fn); + locker.unlock(); + + // Register frame with the disk manager + if (ret) { QMetaObject::invokeMethod(DiskManager::instance(), "CreatedFile", Qt::QueuedConnection, Q_ARG(QString, cache_path), - Q_ARG(QString, fn), - Q_ARG(QByteArray, hash)); - - return true; - } else { - return false; + Q_ARG(QString, fn)); } + + return ret; } -bool FrameHashCache::SaveCacheFrame(const QString &cache_path, const QByteArray &hash, FramePtr frame) -{ - if (frame) { - QMutexLocker locker(¤tly_saving_frames_mutex_); - currently_saving_frames_.insert(hash, frame); - locker.unlock(); - - bool ret = SaveCacheFrame(cache_path, hash, frame->data(), frame->video_params(), frame->linesize_bytes()); - - locker.relock(); - currently_saving_frames_.remove(hash); - locker.unlock(); - - return ret; - } else { - qWarning() << "Attempted to save a NULL frame to the cache. This may or may not be desirable."; - return false; - } -} - -FramePtr FrameHashCache::LoadCacheFrame(const QString &cache_path, const QByteArray &hash) +FramePtr FrameHashCache::LoadCacheFrame(const QString &cache_path, const QUuid &uuid, const int64_t &time) { // Minor optimization, we store frames currently being saved just in case something tries to load // while we're saving. This should *occasionally* optimize and also prevent scenarios where // we try to load a frame that's half way through being saved. + QString filename = CachePathName(cache_path, uuid, time); + QMutexLocker locker(¤tly_saving_frames_mutex_); - if (currently_saving_frames_.contains(hash)) { - return currently_saving_frames_.value(hash); + if (currently_saving_frames_.contains(filename)) { + return currently_saving_frames_.value(filename); } locker.unlock(); @@ -180,12 +116,12 @@ FramePtr FrameHashCache::LoadCacheFrame(const QString &cache_path, const QByteAr return nullptr; } - return LoadCacheFrame(CachePathName(cache_path, hash)); + return LoadCacheFrame(filename); } -FramePtr FrameHashCache::LoadCacheFrame(const QByteArray &hash) const +FramePtr FrameHashCache::LoadCacheFrame(const int64_t &hash) const { - return LoadCacheFrame(GetCacheDirectory(), hash); + return LoadCacheFrame(GetCacheDirectory(), uuid_, hash); } FramePtr FrameHashCache::LoadCacheFrame(const QString &fn) @@ -261,65 +197,17 @@ struct HashTimePair { void FrameHashCache::ShiftEvent(const rational &from, const rational &to) { - // POSITIVE if moving forward -> - // NEGATIVE if moving backward <- - rational diff = to - from; - bool diff_is_negative = (diff < 0); - - int64_t to_ts = ToTimestamp(to); - int64_t from_ts = ToTimestamp(from); - - if (from_ts >= GetMapSize()) { - return; - } - - int64_t ts_diff = to_ts - from_ts; - for (int64_t i=qMin(from_ts, to_ts); i × = hash_time_map_[hash]; - - for (auto jt=times.begin(); jt!=times.end(); jt++) { - int64_t &this_ts = *jt; - if (this_ts == i) { - this_ts += ts_diff; - break; - } - } - } - - if (diff_is_negative) { - // We're moving the frames starting at `from` backwards to where `to` is - if (to_ts < GetMapSize()) { - time_hash_map_.erase(time_hash_map_.begin() + to_ts, time_hash_map_.begin() + from_ts); - } - } else { - // We're moving the frames starting at `from` forwards to where `to` is - if (from_ts < GetMapSize()) { - time_hash_map_.insert(time_hash_map_.begin() + from_ts, ts_diff, QByteArray()); - } - } } void FrameHashCache::InvalidateEvent(const TimeRange &range) { - if (!timebase_.isNull()) { - int64_t start = ToTimestamp(range.in(), Timecode::kCeil); - int64_t end = ToTimestamp(range.out(), Timecode::kCeil); - for (int64_t i=start; i ×_for_hash = hash_time_map_[hash]; - for (auto it=times_for_hash.cbegin(); it!=times_for_hash.cend(); ) { - if ((*it) == i) { - times_for_hash.erase(it); - break; - } else { - it++; - } - } + rational t; + while (iterator.GetNext(&t)) { + int64_t ts = ToTimestamp(t, Timecode::kCeil); - hash.clear(); - } + QFile::remove(CachePathName(ts)); } } @@ -333,62 +221,51 @@ int64_t FrameHashCache::ToTimestamp(const rational &ts, Timecode::Rounding round return Timecode::time_to_timestamp(ts, timebase_, rounding); } -void FrameHashCache::HashDeleted(const QString& s, const QByteArray &hash) +void FrameHashCache::HashDeleted(const QString& path, const QString &filename) { QString cache_dir = GetCacheDirectory(); - if (cache_dir.isEmpty() || s != cache_dir) { + if (cache_dir.isEmpty() || path != cache_dir) { return; } - TimeRangeList ranges_to_invalidate; - const std::vector ×_for_hash = hash_time_map_[hash]; - - for (auto it=times_for_hash.begin(); it!=times_for_hash.end(); it++) { - const int64_t &i = *it; - ranges_to_invalidate.insert(TimeRange(ToTime(i), ToTime(i+1))); + QFileInfo info(filename); + if (uuid_.toString() != info.dir().dirName()) { + return; } - foreach (const TimeRange& range, ranges_to_invalidate) { - // We set job time to 0 because the nodes haven't changed and any render job should be up - // to date - Invalidate(range); - } + int64_t timestamp = info.fileName().toLongLong(); + Invalidate(TimeRange(ToTime(timestamp), ToTime(timestamp + 1))); } void FrameHashCache::ProjectInvalidated(Project *p) { if (GetProject() == p) { - time_hash_map_.clear(); - hash_time_map_.clear(); - InvalidateAll(); } } -QString FrameHashCache::CachePathName(const QByteArray& hash) const +QString FrameHashCache::CachePathName(const int64_t &time) const { - return CachePathName(GetCacheDirectory(), hash); + return CachePathName(GetCacheDirectory(), uuid_, time); } -QString FrameHashCache::CachePathName(const QString &cache_path, const QByteArray &hash) +QString FrameHashCache::CachePathName(const QString &cache_path, const QUuid &cache_id, const int64_t &time) { - QDir cache_dir(QDir(cache_path).filePath(QString(hash.left(1).toHex()))); - - QString filename = QStringLiteral("%1%2").arg(QString(hash.mid(1).toHex()), kCacheFormatExtension); + QString filename = QDir(QDir(cache_path).filePath(cache_id.toString())).filePath(QString::number(time)); // Register that in some way this hash has been accessed QMetaObject::invokeMethod(DiskManager::instance(), "Accessed", Qt::QueuedConnection, Q_ARG(QString, cache_path), - Q_ARG(QByteArray, hash)); + Q_ARG(QString, filename)); - return cache_dir.filePath(filename); + return filename; } -bool FrameHashCache::SaveCacheFrame(const QString &filename, char *data, const VideoParams &vparam, int linesize_bytes) +bool FrameHashCache::SaveCacheFrame(const QString &filename, const FramePtr frame) { - if (!VideoParams::FormatIsFloat(vparam.format())) { + if (!VideoParams::FormatIsFloat(frame->format())) { return false; } @@ -401,45 +278,44 @@ bool FrameHashCache::SaveCacheFrame(const QString &filename, char *data, const V // Floating point types are stored in EXR Imf::PixelType pix_type; - if (vparam.format() == VideoParams::kFormatFloat16) { + if (frame->format() == VideoParams::kFormatFloat16) { pix_type = Imf::HALF; } else { pix_type = Imf::FLOAT; } - Imf::Header header(vparam.effective_width(), - vparam.effective_height()); + Imf::Header header(frame->width(), frame->height()); header.channels().insert("R", Imf::Channel(pix_type)); header.channels().insert("G", Imf::Channel(pix_type)); header.channels().insert("B", Imf::Channel(pix_type)); - if (vparam.channel_count() == VideoParams::kRGBAChannelCount) { + if (frame->channel_count() == VideoParams::kRGBAChannelCount) { header.channels().insert("A", Imf::Channel(pix_type)); } header.compression() = Imf::DWAA_COMPRESSION; header.insert("dwaCompressionLevel", Imf::FloatAttribute(200.0f)); - header.pixelAspectRatio() = vparam.pixel_aspect_ratio().toDouble(); + header.pixelAspectRatio() = frame->video_params().pixel_aspect_ratio().toDouble(); - header.insert("oliveDivider", Imf::IntAttribute(vparam.divider())); + header.insert("oliveDivider", Imf::IntAttribute(frame->video_params().divider())); try { Imf::OutputFile out(filename.toUtf8(), header, 0); - int bpc = VideoParams::GetBytesPerChannel(vparam.format()); + int bpc = VideoParams::GetBytesPerChannel(frame->format()); - size_t xs = vparam.channel_count() * bpc; - size_t ys = linesize_bytes; + size_t xs = frame->channel_count() * bpc; + size_t ys = frame->linesize_bytes(); Imf::FrameBuffer framebuffer; - framebuffer.insert("R", Imf::Slice(pix_type, data, xs, ys)); - framebuffer.insert("G", Imf::Slice(pix_type, data + bpc, xs, ys)); - framebuffer.insert("B", Imf::Slice(pix_type, data + 2*bpc, xs, ys)); - if (vparam.channel_count() == VideoParams::kRGBAChannelCount) { - framebuffer.insert("A", Imf::Slice(pix_type, data + 3*bpc, xs, ys)); + framebuffer.insert("R", Imf::Slice(pix_type, frame->data(), xs, ys)); + framebuffer.insert("G", Imf::Slice(pix_type, frame->data() + bpc, xs, ys)); + framebuffer.insert("B", Imf::Slice(pix_type, frame->data() + 2*bpc, xs, ys)); + if (frame->channel_count() == VideoParams::kRGBAChannelCount) { + framebuffer.insert("A", Imf::Slice(pix_type, frame->data() + 3*bpc, xs, ys)); } out.setFrameBuffer(framebuffer); - out.writePixels(vparam.effective_height()); + out.writePixels(frame->height()); return true; } catch (const std::exception &e) { diff --git a/app/render/framehashcache.h b/app/render/framehashcache.h index cb75d64eb..f56d3cc7c 100644 --- a/app/render/framehashcache.h +++ b/app/render/framehashcache.h @@ -22,6 +22,7 @@ #define VIDEORENDERFRAMECACHE_H #include +#include #include "common/rational.h" #include "common/timecodefunctions.h" @@ -38,35 +39,25 @@ class FrameHashCache : public PlaybackCache public: FrameHashCache(QObject* parent = nullptr); - QByteArray GetHash(const int64_t& time); - QByteArray GetHash(const rational& time); - - const rational &GetTimebase() const - { - return timebase_; - } + const rational &GetTimebase() const { return timebase_; } void SetTimebase(const rational& tb); - void ValidateFramesWithHash(const QByteArray& hash); + void ValidateTimestamp(const int64_t &ts); /** * @brief Return the path of the cached image at this time */ - QString CachePathName(const QByteArray &hash) const; - static QString CachePathName(const QString& cache_path, const QByteArray &hash); + QString CachePathName(const int64_t &time) const; + static QString CachePathName(const QString& cache_path, const QUuid &cache_id, const int64_t &time); - static bool SaveCacheFrame(const QString& filename, char *data, const VideoParams &vparam, int linesize_bytes); - bool SaveCacheFrame(const QByteArray& hash, char *data, const VideoParams &vparam, int linesize_bytes) const; - bool SaveCacheFrame(const QByteArray& hash, FramePtr frame) const; - static bool SaveCacheFrame(const QString& cache_path, const QByteArray& hash, char *data, const VideoParams &vparam, int linesize_bytes); - static bool SaveCacheFrame(const QString& cache_path, const QByteArray& hash, FramePtr frame); - static FramePtr LoadCacheFrame(const QString& cache_path, const QByteArray& hash); - FramePtr LoadCacheFrame(const QByteArray& hash) const; + static bool SaveCacheFrame(const QString& filename, FramePtr frame); + bool SaveCacheFrame(const int64_t &time, FramePtr frame) const; + static bool SaveCacheFrame(const QString& cache_path, const QUuid &uuid, const int64_t &time, FramePtr frame); + static FramePtr LoadCacheFrame(const QString& cache_path, const QUuid &uuid, const int64_t &time); + FramePtr LoadCacheFrame(const int64_t &time) const; static FramePtr LoadCacheFrame(const QString& fn); - void SetHash(const olive::rational &time, const QByteArray& hash, bool frame_exists); - protected: virtual void ShiftEvent(const rational& from, const rational& to) override; @@ -76,22 +67,16 @@ private: rational ToTime(const int64_t &ts) const; int64_t ToTimestamp(const rational &ts, Timecode::Rounding rounding = Timecode::kRound) const; - int64_t GetMapSize() const - { - return int64_t(time_hash_map_.size()); - } - - std::vector time_hash_map_; - std::map > hash_time_map_; - rational timebase_; + QUuid uuid_; + static QMutex currently_saving_frames_mutex_; - static QMap currently_saving_frames_; + static QMap currently_saving_frames_; static const QString kCacheFormatExtension; private slots: - void HashDeleted(const QString &s, const QByteArray& hash); + void HashDeleted(const QString &path, const QString &filename); void ProjectInvalidated(Project* p); diff --git a/app/render/playbackcache.cpp b/app/render/playbackcache.cpp index 7a51597c0..2f225fd81 100644 --- a/app/render/playbackcache.cpp +++ b/app/render/playbackcache.cpp @@ -96,13 +96,7 @@ void PlaybackCache::ShiftEvent(const rational &, const rational &) Project *PlaybackCache::GetProject() const { - // NOTE: A lot of assumptions in this behavior - ViewerOutput* viewer = static_cast(parent()); - if (!viewer) { - return nullptr; - } - - return viewer->project(); + return Project::GetProjectFromObject(this); } TimeRangeList PlaybackCache::GetInvalidatedRanges(TimeRange intersecting)