cache: better implemented changing disk cache folders

Not complete yet, but getting there.
This commit is contained in:
itsmattkc
2020-08-07 02:31:46 +10:00
parent 00d84cfda0
commit df14c2aaa7
33 changed files with 701 additions and 431 deletions
+1 -1
View File
@@ -249,7 +249,7 @@ QList<TimeRange> AudioPlaybackCache::NoLockGetValidRanges(const TimeRange& range
void AudioPlaybackCache::UpdateFilename(const QString &s)
{
filename_ = QDir(FileFunctions::GetMediaCacheLocation()).filePath(s);
filename_ = QDir(GetCacheFilename()).filePath(s);
filename_.append(QStringLiteral(".pcm"));
}
+3 -1
View File
@@ -396,6 +396,7 @@ void RenderBackend::RunNextJob()
worker->SetRenderMode(render_mode_);
worker->SetPreviewGenerationEnabled(generate_audio_previews_);
worker->SetCopyMap(&copy_map_);
worker->SetViewerNode(viewer_node_);
// Move ticket from queue to running list
RenderTicketPtr ticket = render_queue_.front();
@@ -592,7 +593,8 @@ void RenderBackend::AutoCacheVideoRendered()
QFutureWatcher<bool>* w = new QFutureWatcher<bool>();
autocache_video_download_tasks_.insert(w, hash);
connect(w, &QFutureWatcher<bool>::finished, this, &RenderBackend::AutoCacheVideoDownloaded);
w->setFuture(QtConcurrent::run(FrameHashCache::SaveCacheFrame,
w->setFuture(QtConcurrent::run(viewer_node_->video_frame_cache(),
&FrameHashCache::SaveCacheFrame,
hash,
watcher->Get().value<FramePtr>()));
}
+9 -13
View File
@@ -274,21 +274,17 @@ QVariant RenderWorker::GetCachedFrame(const Node* node, const rational& time)
if (node->id() == QStringLiteral("org.olivevideoeditor.Olive.videoinput")) {
QByteArray hash = HashNode(node, video_params(), time);
QString fn = FrameHashCache::CachePathName(hash);
FramePtr f = viewer_->video_frame_cache()->LoadCacheFrame(hash);
if (QFileInfo::exists(fn)) {
FramePtr f = FrameHashCache::LoadCacheFrame(hash);
if (f) {
// The cached frame won't load with the correct divider by default, so we enforce it here
f->set_video_params(VideoParams(f->width() * video_params_.divider(),
f->height() * video_params_.divider(),
f->video_params().time_base(),
f->video_params().format(),
video_params_.divider()));
if (f) {
// The cached frame won't load with the correct divider by default, so we enforce it here
f->set_video_params(VideoParams(f->width() * video_params_.divider(),
f->height() * video_params_.divider(),
f->video_params().time_base(),
f->video_params().format(),
video_params_.divider()));
return CachedFrameToTexture(f);
}
return CachedFrameToTexture(f);
}
}
+7
View File
@@ -48,6 +48,11 @@ public:
available_ = a;
}
void SetViewerNode(ViewerOutput* viewer)
{
viewer_ = viewer;
}
void SetVideoParams(const VideoParams& params)
{
video_params_ = params;
@@ -171,6 +176,8 @@ private:
bool generate_audio_previews_;
ViewerOutput* viewer_;
QHash<Node*, Node*>* copy_map_;
RenderMode::Mode render_mode_;
+186 -103
View File
@@ -25,62 +25,56 @@
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QMessageBox>
#include <QStandardPaths>
#include "common/filefunctions.h"
#include "config/config.h"
#include "core.h"
OLIVE_NAMESPACE_ENTER
DiskManager* DiskManager::instance_ = nullptr;
DiskManager::DiskManager() :
consumption_(0)
DiskManager::DiskManager()
{
// Try to load any current cache index from file
QFile cache_index_file(GetCacheIndexFilename());
// Add default cache location
QFile default_disk_cache_file(QDir(FileFunctions::GetConfigurationLocation()).filePath(QStringLiteral("defaultdiskcache")));
if (default_disk_cache_file.open(QFile::ReadOnly)) {
QString default_dir = default_disk_cache_file.readAll();
if (cache_index_file.open(QFile::ReadOnly)) {
QDataStream ds(&cache_index_file);
while (!cache_index_file.atEnd()) {
HashTime h;
ds >> h.file_name;
ds >> h.hash;
ds >> h.access_time;
ds >> h.file_size;
if (QFileInfo::exists(h.file_name)) {
consumption_ += h.file_size;
disk_data_.append(h);
}
if (FileFunctions::DirectoryIsValid(default_dir, true)) {
GetOpenFolder(default_dir);
} else {
QMessageBox::warning(nullptr,
tr("Disk Cache Error"),
tr("Unable to set custom application disk cache. Using default instead."));
}
default_disk_cache_file.close();
}
// If no custom default was loaded, load default
if (open_folders_.isEmpty()) {
GetOpenFolder(QDir(QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation)).filePath("mediacache"));
}
QFile disk_cache_index(QDir(FileFunctions::GetConfigurationLocation()).filePath(QStringLiteral("diskcache")));
if (disk_cache_index.open(QFile::ReadOnly)) {
QTextStream stream(&disk_cache_index);
QString line;
while (stream.readLineInto(&line)) {
GetOpenFolder(line);
}
disk_cache_index.close();
}
}
DiskManager::~DiskManager()
{
if (Config::Current()["ClearDiskCacheOnClose"].toBool()) {
// Clear all cache data
ClearDiskCache(true);
} else {
// Save current cache index
QFile cache_index_file(GetCacheIndexFilename());
if (cache_index_file.open(QFile::WriteOnly)) {
QDataStream ds(&cache_index_file);
foreach (const HashTime& h, disk_data_) {
ds << h.file_name;
ds << h.hash;
ds << h.access_time;
ds << h.file_size;
}
} else {
qWarning() << "Failed to write cache index:" << GetCacheIndexFilename();
}
}
}
void DiskManager::CreateInstance()
@@ -99,107 +93,178 @@ DiskManager *DiskManager::instance()
return instance_;
}
void DiskManager::Accessed(const QByteArray &hash)
void DiskManager::Accessed(const QString &cache_folder, const QByteArray &hash)
{
lock_.lock();
DiskCacheFolder* f = GetOpenFolder(cache_folder);
for (int i=disk_data_.size()-1;i>=0;i--) {
const HashTime& h = disk_data_.at(i);
f->Accessed(hash);
}
if (h.hash == hash) {
HashTime moved_hash = h;
void DiskManager::CreatedFile(const QString &cache_folder, const QString &file_name, const QByteArray &hash)
{
DiskCacheFolder* f = GetOpenFolder(cache_folder);
moved_hash.access_time = QDateTime::currentMSecsSinceEpoch();
f->CreatedFile(file_name, hash);
}
disk_data_.removeAt(i);
disk_data_.append(moved_hash);
break;
bool DiskManager::ClearDiskCache(const QString &cache_folder)
{
DiskCacheFolder* f = GetOpenFolder(cache_folder);
return f->ClearCache();
}
DiskCacheFolder *DiskManager::GetOpenFolder(const QString &path)
{
// If path is empty, this must mean default
if (path.isEmpty()) {
return GetDefaultCacheFolder();
}
// See if we have an existing path with this name
foreach (DiskCacheFolder* f, open_folders_) {
if (f->GetPath() == path) {
return f;
}
}
lock_.unlock();
// We must have to open this folder
DiskCacheFolder* f = new DiskCacheFolder(path, this);
connect(f, &DiskCacheFolder::DeletedFrame, this, &DiskManager::DeletedFrame);
open_folders_.append(f);
return f;
}
void DiskManager::Accessed(const QString &filename)
DiskCacheFolder::DiskCacheFolder(const QString &path, QObject *parent) :
QObject(parent)
{
lock_.lock();
SetPath(path);
}
for (int i=disk_data_.size()-1;i>=0;i--) {
const HashTime& h = disk_data_.at(i);
DiskCacheFolder::~DiskCacheFolder()
{
CloseCacheFolder();
}
if (h.file_name == filename) {
HashTime moved_hash = h;
bool DiskCacheFolder::ClearCache()
{
bool deleted_files = true;
moved_hash.access_time = QDateTime::currentMSecsSinceEpoch();
std::list<HashTime>::iterator i = disk_data_.begin();
disk_data_.removeAt(i);
disk_data_.append(moved_hash);
break;
while (i != disk_data_.end()) {
// We return a false result if any of the files fail to delete, but still try to delete as many as we can
if (QFile::remove(i->file_name) || !QFileInfo::exists(i->file_name)) {
emit DeletedFrame(path_, i->hash);
i = disk_data_.erase(i);
} else {
qWarning() << "Failed to delete" << i->file_name;
deleted_files = false;
i++;
}
}
lock_.unlock();
return deleted_files;
}
void DiskManager::CreatedFile(const QString &file_name, const QByteArray &hash)
void DiskCacheFolder::Accessed(const QByteArray &hash)
{
lock_.lock();
std::list<HashTime>::iterator i = disk_data_.begin();
while (i != disk_data_.end()) {
if (i->hash == hash) {
// Copy access data and erase from list
HashTime accessed_hash = *i;
disk_data_.erase(i);
// Add it to the end
disk_data_.push_back(accessed_hash);
// End loop
break;
} else {
i++;
}
}
}
void DiskCacheFolder::CreatedFile(const QString &file_name, const QByteArray &hash)
{
qint64 file_size = QFile(file_name).size();
disk_data_.append({file_name, hash, QDateTime::currentMSecsSinceEpoch(), file_size});
disk_data_.push_back({file_name, hash, file_size});
consumption_ += file_size;
QList<QByteArray> deleted_hashes;
while (consumption_ > DiskLimit()) {
while (consumption_ > limit_) {
deleted_hashes.append(DeleteLeastRecent());
}
lock_.unlock();
foreach (const QByteArray& h, deleted_hashes) {
emit DeletedFrame(h);
emit DeletedFrame(path_, h);
}
}
bool DiskManager::ClearDiskCache(bool quick_delete)
void DiskCacheFolder::SetPath(const QString &path)
{
bool deleted_files;
lock_.lock();
if (quick_delete) {
deleted_files = QDir(FileFunctions::GetMediaCacheLocation()).removeRecursively();
// If this is currently set to a folder, close it out now
CloseCacheFolder();
// Signal that disk cache is gone
if (!disk_data_.empty()) {
foreach (const HashTime& h, disk_data_) {
emit DeletedFrame(path_, h.hash);
}
disk_data_.clear();
} else {
deleted_files = true;
}
for (int i=0;i<disk_data_.size();i++) {
const HashTime& ht = disk_data_.at(i);
// Set defaults
clear_on_close_ = false;
consumption_ = 0;
limit_ = 21474836480; // Default to 20 GB
// We return a false result if any of the files fail to delete, but still try to delete as many as we can
if (QFile::remove(ht.file_name) || !QFileInfo::exists(ht.file_name)) {
emit DeletedFrame(ht.hash);
disk_data_.removeAt(i);
i--;
} else {
qWarning() << "Failed to delete" << ht.file_name;
deleted_files = false;
// Set path
path_ = path;
// Attempt to load existing index file from path
QDir path_dir(path_);
path_dir.mkpath(".");
index_path_ = path_dir.filePath(QStringLiteral("index"));
// Try to load any current cache index from file
QFile cache_index_file(index_path_);
if (cache_index_file.open(QFile::ReadOnly)) {
QDataStream ds(&cache_index_file);
ds >> limit_;
ds >> clear_on_close_;
while (!cache_index_file.atEnd()) {
HashTime h;
ds >> h.file_name;
ds >> h.hash;
ds >> h.file_size;
if (QFileInfo::exists(h.file_name)) {
consumption_ += h.file_size;
disk_data_.push_back(h);
}
}
cache_index_file.close();
}
lock_.unlock();
return deleted_files;
}
QByteArray DiskManager::DeleteLeastRecent()
QByteArray DiskCacheFolder::DeleteLeastRecent()
{
HashTime h = disk_data_.takeFirst();
HashTime h = disk_data_.front();
disk_data_.pop_front();
QFile::remove(h.file_name);
@@ -208,19 +273,37 @@ QByteArray DiskManager::DeleteLeastRecent()
return h.hash;
}
qint64 DiskManager::DiskLimit()
void DiskCacheFolder::CloseCacheFolder()
{
double gigabytes = Config::Current()["DiskCacheSize"].toDouble();
if (path_.isEmpty()) {
return;
}
// Convert gigabytes to bytes
return qRound64(gigabytes * 1073741824);
}
if (clear_on_close_) {
// If we're not moving to new and we're set to clear on close, clear now or else it'll never
// get cleared later
ClearCache();
}
QString DiskManager::GetCacheIndexFilename()
{
QDir d(QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation));
d.mkpath(".");
return d.filePath("diskindex");
// Save current cache index
QFile cache_index_file(index_path_);
if (cache_index_file.open(QFile::WriteOnly)) {
QDataStream ds(&cache_index_file);
ds << limit_;
ds << clear_on_close_;
foreach (const HashTime& h, disk_data_) {
ds << h.file_name;
ds << h.hash;
ds << h.file_size;
}
cache_index_file.close();
} else {
qWarning() << "Failed to write cache index:" << index_path_;
}
}
OLIVE_NAMESPACE_EXIT
+93 -23
View File
@@ -21,6 +21,7 @@
#ifndef DISKMANAGER_H
#define DISKMANAGER_H
#include <QMap>
#include <QMutex>
#include <QObject>
@@ -28,6 +29,75 @@
OLIVE_NAMESPACE_ENTER
class DiskCacheFolder : public QObject
{
Q_OBJECT
public:
DiskCacheFolder(const QString& path, QObject* parent = nullptr);
virtual ~DiskCacheFolder() override;
bool ClearCache();
void Accessed(const QByteArray& hash);
void CreatedFile(const QString& file_name, const QByteArray& hash);
const QString& GetPath() const
{
return path_;
}
void SetPath(const QString& path);
qint64 GetLimit() const
{
return limit_;
}
bool GetClearOnClose() const
{
return clear_on_close_;
}
void SetLimit(qint64 l)
{
limit_ = l;
}
void SetClearOnClose(bool e)
{
clear_on_close_ = e;
}
signals:
void DeletedFrame(const QString& path, const QByteArray& hash);
private:
QByteArray DeleteLeastRecent();
void CloseCacheFolder();
QString path_;
QString index_path_;
struct HashTime {
QString file_name;
QByteArray hash;
qint64 file_size;
};
std::list<HashTime> disk_data_;
qint64 consumption_;
qint64 limit_;
bool clear_on_close_;
};
class DiskManager : public QObject
{
Q_OBJECT
@@ -38,16 +108,33 @@ public:
static DiskManager* instance();
void Accessed(const QByteArray& hash);
bool ClearDiskCache(const QString& cache_folder);
void Accessed(const QString& filename);
DiskCacheFolder* GetDefaultCacheFolder() const
{
// The first folder will always be the default
return open_folders_.first();
}
void CreatedFile(const QString& file_name, const QByteArray& hash);
const QString& GetDefaultCachePath() const
{
return GetDefaultCacheFolder()->GetPath();
}
bool ClearDiskCache(bool quick_delete);
DiskCacheFolder* GetOpenFolder(const QString& path);
const QVector<DiskCacheFolder*>& GetOpenFolders() const
{
return open_folders_;
}
public slots:
void Accessed(const QString& cache_folder, const QByteArray& hash);
void CreatedFile(const QString& cache_folder, const QString& file_name, const QByteArray& hash);
signals:
void DeletedFrame(const QByteArray& hash);
void DeletedFrame(const QString& path, const QByteArray& hash);
private:
DiskManager();
@@ -56,24 +143,7 @@ private:
static DiskManager* instance_;
QByteArray DeleteLeastRecent();
qint64 DiskLimit();
static QString GetCacheIndexFilename();
struct HashTime {
QString file_name;
QByteArray hash;
qint64 access_time;
qint64 file_size;
};
QList<HashTime> disk_data_;
qint64 consumption_;
QMutex lock_;
QVector<DiskCacheFolder*> open_folders_;
};
+24 -10
View File
@@ -227,13 +227,18 @@ QVector<rational> FrameHashCache::GetInvalidatedFrames(const TimeRange &intersec
bool FrameHashCache::SaveCacheFrame(const QByteArray& hash,
char* data,
const VideoParams& vparam,
int linesize_bytes)
int linesize_bytes) const
{
QString fn = CachePathName(hash);
if (SaveCacheFrame(fn, data, vparam, linesize_bytes)) {
// Register frame with the disk manager
DiskManager::instance()->CreatedFile(fn, hash);
QMetaObject::invokeMethod(DiskManager::instance(),
"CreatedFile",
Qt::QueuedConnection,
Q_ARG(QString, GetCacheDirectory()),
Q_ARG(QString, fn),
Q_ARG(QByteArray, hash));
return true;
} else {
@@ -241,7 +246,7 @@ bool FrameHashCache::SaveCacheFrame(const QByteArray& hash,
}
}
bool FrameHashCache::SaveCacheFrame(const QByteArray &hash, FramePtr frame)
bool FrameHashCache::SaveCacheFrame(const QByteArray &hash, FramePtr frame) const
{
if (frame) {
return SaveCacheFrame(hash, frame->data(), frame->video_params(), frame->linesize_bytes());
@@ -251,12 +256,12 @@ bool FrameHashCache::SaveCacheFrame(const QByteArray &hash, FramePtr frame)
}
}
FramePtr FrameHashCache::LoadCacheFrame(const QByteArray &hash)
FramePtr FrameHashCache::LoadCacheFrame(const QByteArray &hash) const
{
return LoadCacheFrame(CachePathName(hash));
}
FramePtr FrameHashCache::LoadCacheFrame(const QString &fn)
FramePtr FrameHashCache::LoadCacheFrame(const QString &fn) const
{
FramePtr frame = nullptr;
@@ -367,8 +372,13 @@ void FrameHashCache::ShiftEvent(const rational &from, const rational &to)
}
}
void FrameHashCache::HashDeleted(const QByteArray &hash)
void FrameHashCache::HashDeleted(const QString& s, const QByteArray &hash)
{
QString cache_dir = GetCacheDirectory();
if (cache_dir.isEmpty() || s != cache_dir) {
return;
}
QMutexLocker locker(lock());
TimeRangeList invalidated;
@@ -390,22 +400,26 @@ void FrameHashCache::HashDeleted(const QByteArray &hash)
}
}
QString FrameHashCache::CachePathName(const QByteArray& hash)
QString FrameHashCache::CachePathName(const QByteArray& hash) const
{
QString ext = GetFormatExtension();
QDir cache_dir(QDir(FileFunctions::GetMediaCacheLocation()).filePath(QString(hash.left(1).toHex())));
QDir cache_dir(QDir(GetCacheDirectory()).filePath(QString(hash.left(1).toHex())));
cache_dir.mkpath(".");
QString filename = QStringLiteral("%1%2").arg(QString(hash.mid(1).toHex()), ext);
// Register that in some way this hash has been accessed
DiskManager::instance()->Accessed(hash);
QMetaObject::invokeMethod(DiskManager::instance(),
"Accessed",
Qt::QueuedConnection,
Q_ARG(QString, GetCacheDirectory()),
Q_ARG(QByteArray, hash));
return cache_dir.filePath(filename);
}
bool FrameHashCache::SaveCacheFrame(const QString &filename, char *data, const VideoParams &vparam, int linesize_bytes)
bool FrameHashCache::SaveCacheFrame(const QString &filename, char *data, const VideoParams &vparam, int linesize_bytes) const
{
Q_ASSERT(PixelFormat::FormatIsFloat(vparam.format()));
+7 -7
View File
@@ -58,13 +58,13 @@ public:
/**
* @brief Return the path of the cached image at this time
*/
static QString CachePathName(const QByteArray &hash);
QString CachePathName(const QByteArray &hash) const;
static bool SaveCacheFrame(const QString& filename, char *data, const VideoParams &vparam, int linesize_bytes);
static bool SaveCacheFrame(const QByteArray& hash, char *data, const VideoParams &vparam, int linesize_bytes);
static bool SaveCacheFrame(const QByteArray& hash, FramePtr frame);
static FramePtr LoadCacheFrame(const QByteArray& hash);
static FramePtr LoadCacheFrame(const QString& fn);
bool SaveCacheFrame(const QString& filename, char *data, const VideoParams &vparam, int linesize_bytes) const;
bool SaveCacheFrame(const QByteArray& hash, char *data, const VideoParams &vparam, int linesize_bytes) const;
bool SaveCacheFrame(const QByteArray& hash, FramePtr frame) const;
FramePtr LoadCacheFrame(const QByteArray& hash) const;
FramePtr LoadCacheFrame(const QString& fn) const;
static QString GetFormatExtension();
@@ -87,7 +87,7 @@ private:
rational timebase_;
private slots:
void HashDeleted(const QByteArray& hash);
void HashDeleted(const QString &s, const QByteArray& hash);
};
+25
View File
@@ -22,6 +22,10 @@
#include <QDateTime>
#include "node/output/viewer/viewer.h"
#include "project/item/sequence/sequence.h"
#include "project/project.h"
OLIVE_NAMESPACE_ENTER
void PlaybackCache::Invalidate(const TimeRange &r)
@@ -190,4 +194,25 @@ void PlaybackCache::RemoveRangeFromJobs(const TimeRange &remove)
}
}
QString PlaybackCache::GetCacheDirectory() const
{
// NOTE: A lot of assumptions in this behavior
ViewerOutput* viewer = static_cast<ViewerOutput*>(parent());
if (!viewer) {
return QString();
}
Sequence* sequence = static_cast<Sequence*>(viewer->parent());
if (!sequence) {
return QString();
}
Project* project = sequence->project();
if (!project) {
return QString();
}
return project->cache_path();
}
OLIVE_NAMESPACE_EXIT
+2
View File
@@ -103,6 +103,8 @@ protected:
virtual void ShiftEvent(const rational& from, const rational& to);
QString GetCacheDirectory() const;
QMutex* lock()
{
return &lock_;