refactored import function for greater robustness

The import function was written early on in the rewrite as a multithreaded
background task that was considered somewhat flawed. While it worked for the
most part, there were possibilities of race conditions that could have
potentially been fatal, particularly since media could theoretically be
deleted while the import/probe tasks were running in the background.

With the save/load functions coming in, it became even more complicated as
projects may include metadata about the footage that can't be implemented
easily when the footage is imported/probed in the background. Making importing
a modal task fixes all of these issues, it's still done in a background thread
to not hang the GUI thread, but the GUI thread can be briefly "paused" in a
user friendly way so that all these functions can be safer.
This commit is contained in:
itsmattkc
2020-01-15 15:19:14 +11:00
parent 16904c976f
commit 75a9291cd2
24 changed files with 233 additions and 382 deletions
+3 -1
View File
@@ -37,7 +37,9 @@ QString OIIODecoder::id()
bool OIIODecoder::Probe(Footage *f)
{
auto in = OIIO::ImageInput::open(f->filename().toStdString());
std::string std_filename = f->filename().toStdString();
auto in = OIIO::ImageInput::open(std_filename);
if (!in) {
return false;
+37 -2
View File
@@ -41,13 +41,13 @@
#include "panel/panelmanager.h"
#include "panel/project/project.h"
#include "panel/viewer/viewer.h"
#include "project/projectimportmanager.h"
#include "project/projectloadmanager.h"
#include "project/projectsavemanager.h"
#include "project/item/footage/footage.h"
#include "project/item/sequence/sequence.h"
#include "render/colormanager.h"
#include "render/diskmanager.h"
#include "task/import/import.h"
#include "task/taskmanager.h"
#include "ui/style/style.h"
#include "undo/undostack.h"
@@ -161,7 +161,17 @@ void Core::ImportFiles(const QStringList &urls, ProjectViewModel* model, Folder*
return;
}
TaskManager::instance()->AddTask(std::make_shared<ImportTask>(model, parent, urls));
ProjectImportManager* pim = new ProjectImportManager(model, parent, urls);
if (!pim->GetFileCount()) {
// No files to import
delete pim;
return;
}
connect(pim, &ProjectImportManager::ImportComplete, this, &Core::ImportTaskComplete, Qt::BlockingQueuedConnection);
InitiateOpenSaveProcess(pim, tr("Importing %1 files").arg(pim->GetFileCount()), tr("Importing..."));
}
const Tool::Item &Core::tool()
@@ -358,6 +368,11 @@ void Core::AddOpenProject(ProjectPtr p)
emit ProjectOpened(p.get());
}
void Core::ImportTaskComplete(QUndoCommand *command)
{
undo_stack_.pushIfHasChildren(command);
}
void Core::DeclareTypesForQt()
{
qRegisterMetaType<Task::Status>("Task::Status");
@@ -581,6 +596,26 @@ void Core::OpenProjectInternal(const QString &filename)
InitiateOpenSaveProcess(plm, tr("Loading '%1'").arg(filename), tr("Load Project"));
}
int Core::CountFilesInFileList(const QFileInfoList &filenames)
{
int file_count = 0;
foreach (const QFileInfo& f, filenames) {
// For some reason QDir::NoDotAndDotDot doesn't work with entryInfoList, so we have to check manually
if (f.fileName() == "." || f.fileName() == "..") {
continue;
} else if (f.isDir()) {
QFileInfoList info_list = QDir(f.absoluteFilePath()).entryInfoList();
file_count += CountFilesInFileList(info_list);
} else {
file_count++;
}
}
return file_count;
}
void Core::InitiateOpenSaveProcess(ProjectFileManagerBase *manager, const QString& dialog_text, const QString& dialog_title)
{
// Create save dialog
+9
View File
@@ -21,10 +21,12 @@
#ifndef CORE_H
#define CORE_H
#include <QFileInfoList>
#include <QList>
#include <QTimer>
#include "common/rational.h"
#include "project/item/footage/footage.h"
#include "project/project.h"
#include "project/projectfilemanagerbase.h"
#include "project/projectviewmodel.h"
@@ -174,6 +176,11 @@ public:
*/
static QString ChannelLayoutToString(const uint64_t &layout);
/**
* @brief Recursively count files in a file/directory list
*/
static int CountFilesInFileList(const QFileInfoList &filenames);
public slots:
/**
* @brief Starts an open file dialog to load a project from file
@@ -358,6 +365,8 @@ private slots:
*/
void AddOpenProject(ProjectPtr p);
void ImportTaskComplete(QUndoCommand* command);
};
#endif // CORE_H
+1 -1
View File
@@ -284,7 +284,7 @@ Node *NodeInput::get_connected_node() const
{
NodeOutput* output = get_connected_output();
if (output != nullptr) {
if (output) {
return output->parentNode();
}
+2
View File
@@ -22,6 +22,8 @@ set(OLIVE_SOURCES
project/project.cpp
project/projectfilemanagerbase.h
project/projectfilemanagerbase.cpp
project/projectimportmanager.h
project/projectimportmanager.cpp
project/projectloadmanager.h
project/projectloadmanager.cpp
project/projectsavemanager.h
+1 -1
View File
@@ -66,8 +66,8 @@ void Folder::Load(QXmlStreamReader *reader)
continue;
}
child->Load(reader);
add_child(child);
child->Load(reader);
}
}
}
+2 -1
View File
@@ -24,6 +24,7 @@
#include "common/timecodefunctions.h"
#include "common/xmlreadloop.h"
#include "codec/decoder.h"
#include "ui/icons/icons.h"
Footage::Footage()
@@ -48,7 +49,7 @@ void Footage::Load(QXmlStreamReader *reader)
}
}
// FIXME: Probe here?
Decoder::ProbeMedia(this);
XMLReadLoop(reader, "footage") {
if (reader->isStartElement()) {
+4
View File
@@ -20,9 +20,13 @@
#include "project.h"
#include <QDir>
#include <QFileInfo>
#include "common/xmlreadloop.h"
#include "core.h"
#include "dialog/loadsave/loadsave.h"
#include "window/mainwindow/mainwindow.h"
Project::Project()
{
+12
View File
@@ -6,7 +6,19 @@ ProjectFileManagerBase::ProjectFileManagerBase() :
}
void ProjectFileManagerBase::Start()
{
Action();
emit Finished();
}
void ProjectFileManagerBase::Cancel()
{
cancelled_ = true;
}
const QAtomicInt &ProjectFileManagerBase::IsCancelled() const
{
return cancelled_;
}
+6 -1
View File
@@ -18,7 +18,7 @@ public slots:
* It's recommended to invoke this through Qt signals/slots/QueuedConnection after moving this object to a separate
* thread.
*/
virtual void Start() = 0;
void Start();
/**
* @brief Cancel the current save
@@ -28,6 +28,11 @@ public slots:
*/
void Cancel();
protected:
virtual void Action() = 0;
const QAtomicInt& IsCancelled() const;
signals:
void ProgressChanged(int);
+112
View File
@@ -0,0 +1,112 @@
#include "projectimportmanager.h"
#include <QDir>
#include <QFileInfo>
#include "core.h"
#include "codec/decoder.h"
#include "project/item/footage/footage.h"
ProjectImportManager::ProjectImportManager(ProjectViewModel *model, Folder *folder, const QStringList &filenames) :
model_(model),
folder_(folder)
{
foreach (const QString& f, filenames) {
filenames_.append(f);
}
file_count_ = Core::CountFilesInFileList(filenames_);
}
const int &ProjectImportManager::GetFileCount()
{
return file_count_;
}
void ProjectImportManager::Action()
{
QUndoCommand* command = new QUndoCommand();
int imported = 0;
Import(folder_, filenames_, imported, command);
if (IsCancelled()) {
delete command;
} else {
emit ImportComplete(command);
}
}
void ProjectImportManager::Import(Folder *folder, const QFileInfoList &import, int &counter, QUndoCommand* parent_command)
{
foreach (const QFileInfo& file_info, import) {
if (IsCancelled()) {
break;
}
// Check if this file is a diretory
if (file_info.isDir()) {
// QDir::entryList only returns filenames, we can use entryInfoList() to get full paths
QFileInfoList entry_list = QDir(file_info.absoluteFilePath()).entryInfoList();
// Strip out "." and ".." (for some reason QDir::NoDotAndDotDot doesn't work with entryInfoList, so we have to
// check manually)
for (int i=0;i<entry_list.size();i++) {
if (entry_list.at(i).fileName() == "." || entry_list.at(i).fileName() == "..") {
entry_list.removeAt(i);
i--;
}
}
// Only proceed if the empty actually has files in it
if (!entry_list.isEmpty()) {
// Create a folder corresponding to the directory
ItemPtr f = std::make_shared<Folder>();
f->set_name(file_info.fileName());
// Create undoable command that adds the items to the model
new ProjectViewModel::AddItemCommand(model_,
folder,
f,
parent_command);
// Recursively follow this path
Import(static_cast<Folder*>(f.get()), entry_list, counter, parent_command);
}
} else {
FootagePtr f = std::make_shared<Footage>();
f->set_filename(file_info.absoluteFilePath());
f->set_name(file_info.fileName());
f->set_timestamp(file_info.lastModified());
// Probe will fail if a project isn't set because ImageStream and its derivatives try to connect to the project's
// ColorManager instance
// FIXME: Perhaps re-think this approach at some point
f->set_project(model_->project());
Decoder::ProbeMedia(f.get());
f->set_project(nullptr);
if (f->status() != Footage::kInvalid) {
// Create undoable command that adds the items to the model
new ProjectViewModel::AddItemCommand(model_,
folder,
f,
parent_command);
}
counter++;
emit ProgressChanged((counter * 100) / file_count_);
}
}
}
+37
View File
@@ -0,0 +1,37 @@
#ifndef PROJECTIMPORTMANAGER_H
#define PROJECTIMPORTMANAGER_H
#include <QFileInfoList>
#include <QUndoCommand>
#include "projectfilemanagerbase.h"
#include "projectviewmodel.h"
class ProjectImportManager : public ProjectFileManagerBase
{
Q_OBJECT
public:
ProjectImportManager(ProjectViewModel* model, Folder* folder, const QStringList& filenames);
const int& GetFileCount();
protected:
virtual void Action() override;
signals:
void ImportComplete(QUndoCommand* command);
private:
void Import(Folder* folder, const QFileInfoList &import, int& counter, QUndoCommand *parent_command);
ProjectViewModel* model_;
Folder* folder_;
QFileInfoList filenames_;
int file_count_;
};
#endif // PROJECTIMPORTMANAGER_H
+1 -1
View File
@@ -9,7 +9,7 @@ ProjectLoadManager::ProjectLoadManager(const QString &filename) :
{
}
void ProjectLoadManager::Start()
void ProjectLoadManager::Action()
{
QFile project_file(filename_);
+2 -8
View File
@@ -9,14 +9,8 @@ class ProjectLoadManager : public ProjectFileManagerBase
public:
ProjectLoadManager(const QString& filename);
public slots:
/**
* @brief Start the load process
*
* It's recommended to invoke this through Qt signals/slots/QueuedConnection after moving this object to a separate
* thread.
*/
virtual void Start() override;
protected:
virtual void Action() override;
signals:
void ProjectLoaded(ProjectPtr project);
+1 -1
View File
@@ -9,7 +9,7 @@ ProjectSaveManager::ProjectSaveManager(Project *project) :
}
void ProjectSaveManager::Start()
void ProjectSaveManager::Action()
{
QFile project_file(project_->filename());
+2 -8
View File
@@ -9,14 +9,8 @@ class ProjectSaveManager : public ProjectFileManagerBase
public:
ProjectSaveManager(Project* project);
public slots:
/**
* @brief Start the save process
*
* It's recommended to invoke this through Qt signals/slots/QueuedConnection after moving this object to a separate
* thread.
*/
virtual void Start() override;
protected:
virtual void Action() override;
private:
Project* project_;
+1 -1
View File
@@ -354,7 +354,7 @@ bool ProjectViewModel::dropMimeData(const QMimeData *data, Qt::DropAction action
QUrl url = stream.readLine();
if (!url.isEmpty()) {
urls.append(url.path());
urls.append(url.toLocalFile());
}
}
-3
View File
@@ -14,9 +14,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
add_subdirectory(import)
add_subdirectory(probe)
set(OLIVE_SOURCES
${OLIVE_SOURCES}
task/task.h
-22
View File
@@ -1,22 +0,0 @@
# Olive - Non-Linear Video Editor
# Copyright (C) 2019 Olive Team
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set(OLIVE_SOURCES
${OLIVE_SOURCES}
task/import/import.h
task/import/import.cpp
PARENT_SCOPE
)
-160
View File
@@ -1,160 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "import.h"
#include <QApplication>
#include <QDebug>
#include <QDir>
#include <QFileInfo>
// FIXME: Only used for test code
#include "panel/panelmanager.h"
#include "panel/project/project.h"
// End test code
#include "core.h"
#include "project/item/footage/footage.h"
#include "task/probe/probe.h"
#include "task/taskmanager.h"
ImportTask::ImportTask(ProjectViewModel *model, Folder *parent, const QStringList &urls) :
model_(model),
urls_(urls),
parent_(parent),
command_(nullptr)
{
set_text(tr("Importing %1 files").arg(urls.size()));
}
bool ImportTask::Action()
{
parent_->LockDeletes();
command_ = new QUndoCommand();
Import(urls_, parent_, command_);
// If this task was cancelled, we won't bother pushing an undo command (we don't end up with anything undoable since
// the undo command executes the final import anyway)
if (cancelled()) {
delete command_;
command_ = nullptr;
parent_->UnlockDeletes();
}
return true;
}
bool ImportTask::Epilogue()
{
if (command_ != nullptr) {
Core::instance()->undo_stack()->push(command_);
}
parent_->UnlockDeletes();
return true;
}
void ImportTask::Import(const QStringList &files, Folder *folder, QUndoCommand *parent_command)
{
for (int i=0;i<files.size();i++) {
// Stop here if the Task has been cancelled
if (cancelled()) {
break;
}
const QString& url = files.at(i);
QFileInfo file_info(url);
// Check if this file is a diretory
if (file_info.isDir()) {
// Use QDir to get a list of all the files in the directory
QDir dir(url);
// QDir::entryList only returns filenames, we can use entryInfoList() to get full paths
QFileInfoList entry_list = dir.entryInfoList();
// Only proceed if the empty actually has files in it
if (!entry_list.isEmpty()) {
// Create a folder corresponding to the directory
ItemPtr f = std::make_shared<Folder>();
f->set_name(file_info.fileName());
// Create undoable command that adds the items to the model
new ProjectViewModel::AddItemCommand(model_,
folder,
f,
parent_command);
// Convert QFileInfoList into QStringList
QStringList full_urls;
foreach (QFileInfo info, entry_list) {
if (info.fileName() != ".." && info.fileName() != ".") {
full_urls.append(info.absoluteFilePath());
}
}
// Recursively follow this path
Import(full_urls, static_cast<Folder*>(f.get()), parent_command);
}
} else {
FootagePtr f = std::make_shared<Footage>();
// FIXME: Is it possible for a file to go missing between the Import dialog and here?
// And what is the behavior/result of that?
f->set_filename(url);
f->set_name(file_info.fileName());
f->set_timestamp(file_info.lastModified());
// Create undoable command that adds the items to the model
new ProjectViewModel::AddItemCommand(model_,
folder,
f,
parent_command);
// Create ProbeTask to analyze this media
TaskPtr pt = std::make_shared<ProbeTask>(f);
// The task won't work unless it's in the main thread and we're definitely not
// FIXME: Should Tasks check what thread they're in and move themselves to the main thread?
pt->moveToThread(qApp->thread());
// Queue task in task manager
new TaskManager::AddTaskCommand(pt, parent_command);
//olive::task_manager.AddTask(pt);
}
emit ProgressChanged(i * 100 / files.size());
}
}
-56
View File
@@ -1,56 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef IMPORT_H
#define IMPORT_H
#include "project/projectviewmodel.h"
#include "project/item/folder/folder.h"
#include "task/task.h"
/**
* @brief The ImportTask class
*
* A background task to create Footage objects from a list of URLs, and then create ProbeTasks for each of them.
*
* Using this Task is the best way to import media into a project since it will run in the background/multithreaded
* without pausing the main thread.
*/
class ImportTask : public Task
{
Q_OBJECT
public:
ImportTask(ProjectViewModel* model, Folder *parent, const QStringList& urls);
virtual bool Action() override;
virtual bool Epilogue() override;
private:
void Import(const QStringList& files, Folder* folder, QUndoCommand* parent_command);
ProjectViewModel* model_;
QStringList urls_;
Folder* parent_;
QUndoCommand* command_;
};
#endif // IMPORT_H
-22
View File
@@ -1,22 +0,0 @@
# Olive - Non-Linear Video Editor
# Copyright (C) 2019 Olive Team
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set(OLIVE_SOURCES
${OLIVE_SOURCES}
task/probe/probe.h
task/probe/probe.cpp
PARENT_SCOPE
)
-44
View File
@@ -1,44 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "probe.h"
#include <QFileInfo>
#include "codec/decoder.h"
ProbeTask::ProbeTask(FootagePtr footage) :
footage_(footage)
{
QString base_filename = QFileInfo(footage_->filename()).fileName();
set_text(tr("Probing \"%1\"").arg(base_filename));
}
bool ProbeTask::Action()
{
footage_->LockDeletes();
Decoder::ProbeMedia(footage_.get());
footage_->UnlockDeletes();
return true;
}
-49
View File
@@ -1,49 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef PROBE_H
#define PROBE_H
#include "project/item/footage/footage.h"
#include "task/task.h"
/**
* @brief The ProbeTask class
*
* A background task for probing a certain Footage file for its metadata and determining if we have a viable decoder
* for it.
*
* Currently this function just calls olive::ProbeMedia() which will call Footage::Clear(), clearing the Footage of
* any previous metadata before passing it through the available decoders until it finds one that can parse it.
* The ProbeTask mostly functions as a background/multithreaded wrapper for this functionality.
*/
class ProbeTask : public Task
{
Q_OBJECT
public:
ProbeTask(FootagePtr footage);
virtual bool Action() override;
private:
FootagePtr footage_;
};
#endif // PROBE_H