Skip to content

Improve project loading/saving time by using binary format #1056

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cedricp wants to merge 1 commit into
base: RB-2.6
Choose a base branch
from
Open

Improve project loading/saving time by using binary format #1056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Engine/AppInstance.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,10 +218,18 @@ GCC_DIAG_SUGGEST_OVERRIDE_ON
{
}

virtual void loadProjectGui(bool /*isAutosave*/, boost::archive::binary_iarchive & /*archive*/) const
{
}

virtual void saveProjectGui(boost::archive::xml_oarchive & /*archive*/)
{
}

virtual void saveProjectGui(boost::archive::binary_oarchive & /*archive*/)
{
}

virtual void setupViewersForViews(const std::vector<std::string>& /*viewNames*/)
{
}
Expand Down
2 changes: 2 additions & 0 deletions Engine/EngineFwd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ namespace boost {
namespace archive {
class xml_iarchive;
class xml_oarchive;
class binary_iarchive;
class binary_oarchive;
}
namespace serialization {
class access;
Expand Down
155 changes: 113 additions & 42 deletions Engine/Project.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,11 +287,6 @@ Project::loadProjectInternal(const QString & path,
}

bool ret = false;
FStreamsSupport::ifstream ifile;
FStreamsSupport::open( &ifile, filePath.toStdString() );
if (!ifile) {
throw std::runtime_error( tr("Failed to open %1").arg(filePath).toStdString() );
}

if ( (NATRON_VERSION_MAJOR == 1) && (NATRON_VERSION_MINOR == 0) && (NATRON_VERSION_REVISION == 0) ) {
///Try to determine if the project was made during Natron v1.0.0 - RC2 or RC3 to detect a bug we introduced at that time
Expand All @@ -315,21 +310,28 @@ Project::loadProjectInternal(const QString & path,
}

LoadProjectSplashScreen_RAII __raii_splashscreen__(getApp(), name);
bool xml_loaded = false;

try {
bool bgProject;
boost::archive::xml_iarchive iArchive(ifile);
{
FlagSetter __raii_loadingProjectInternal__(true, &_imp->isLoadingProjectInternal, &_imp->isLoadingProjectMutex);
// XML loading tests first, then binary loading
FStreamsSupport::ifstream ifile;
FStreamsSupport::open( &ifile, filePath.toStdString());
if (ifile) {
bool bgProject;
boost::archive::xml_iarchive iArchive(ifile);
{
FlagSetter __raii_loadingProjectInternal__(true, &_imp->isLoadingProjectInternal, &_imp->isLoadingProjectMutex);

iArchive >> boost::serialization::make_nvp("Background_project", bgProject);
ProjectSerialization projectSerializationObj( getApp() );
iArchive >> boost::serialization::make_nvp("Project", projectSerializationObj);
ret = load(projectSerializationObj, name, path, mustSave);
} // __raii_loadingProjectInternal__

iArchive >> boost::serialization::make_nvp("Background_project", bgProject);
ProjectSerialization projectSerializationObj( getApp() );
iArchive >> boost::serialization::make_nvp("Project", projectSerializationObj);
ret = load(projectSerializationObj, name, path, mustSave);
} // __raii_loadingProjectInternal__

if (!bgProject) {
getApp()->loadProjectGui(isAutoSave, iArchive);
if (!bgProject) {
getApp()->loadProjectGui(isAutoSave, iArchive);
}
xml_loaded = true;
}
} catch (const std::exception &e) {
const ProjectBeingLoadedInfo& pInfo = getApp()->getProjectBeingLoadedInfo();
Expand All @@ -339,7 +341,6 @@ Project::loadProjectInternal(const QString & path,
QString message = tr("This project was saved with a more recent version (%1.%2.%3) of %4. Projects are not forward compatible and may only be opened in a version of %4 equal or more recent than the version that saved it.").arg(pInfo.vMajor).arg(pInfo.vMinor).arg(pInfo.vRev).arg(QString::fromUtf8(NATRON_APPLICATION_NAME));
throw std::runtime_error(message.toStdString());
}
throw std::runtime_error( tr("Unrecognized or damaged project file:").toStdString() + ' ' + e.what());
} catch (...) {
const ProjectBeingLoadedInfo& pInfo = getApp()->getProjectBeingLoadedInfo();
if (pInfo.vMajor > NATRON_VERSION_MAJOR ||
Expand All @@ -348,7 +349,48 @@ Project::loadProjectInternal(const QString & path,
QString message = tr("This project was saved with a more recent version (%1.%2.%3) of %4. Projects are not forward compatible and may only be opened in a version of %4 equal or more recent than the version that saved it.").arg(pInfo.vMajor).arg(pInfo.vMinor).arg(pInfo.vRev).arg(QString::fromUtf8(NATRON_APPLICATION_NAME));
throw std::runtime_error(message.toStdString());
}
throw std::runtime_error( tr("Unrecognized or damaged project file").toStdString() );
}

if (!xml_loaded){
try {
FStreamsSupport::ifstream ifile;
FStreamsSupport::open( &ifile, filePath.toStdString(), std::ios::in | std::ios::binary );
if (!ifile) {
throw std::runtime_error( tr("Failed to open %1").arg(filePath).toStdString() );
}
bool bgProject;
boost::archive::binary_iarchive iArchive(ifile);
{
FlagSetter __raii_loadingProjectInternal__(true, &_imp->isLoadingProjectInternal, &_imp->isLoadingProjectMutex);

iArchive >> boost::serialization::make_nvp("Background_project", bgProject);
ProjectSerialization projectSerializationObj( getApp() );
iArchive >> boost::serialization::make_nvp("Project", projectSerializationObj);
ret = load(projectSerializationObj, name, path, mustSave);
} // __raii_loadingProjectInternal__

if (!bgProject) {
getApp()->loadProjectGui(isAutoSave, iArchive);
}
} catch (const std::exception &e) {
const ProjectBeingLoadedInfo& pInfo = getApp()->getProjectBeingLoadedInfo();
if (pInfo.vMajor > NATRON_VERSION_MAJOR ||
(pInfo.vMajor == NATRON_VERSION_MAJOR && pInfo.vMinor > NATRON_VERSION_MINOR) ||
(pInfo.vMajor == NATRON_VERSION_MAJOR && pInfo.vMinor == NATRON_VERSION_MINOR && pInfo.vRev > NATRON_VERSION_REVISION)) {
QString message = tr("This project was saved with a more recent version (%1.%2.%3) of %4. Projects are not forward compatible and may only be opened in a version of %4 equal or more recent than the version that saved it.").arg(pInfo.vMajor).arg(pInfo.vMinor).arg(pInfo.vRev).arg(QString::fromUtf8(NATRON_APPLICATION_NAME));
throw std::runtime_error(message.toStdString());
}
throw std::runtime_error( tr("Unrecognized or damaged project file:").toStdString() + ' ' + e.what());
} catch (...) {
const ProjectBeingLoadedInfo& pInfo = getApp()->getProjectBeingLoadedInfo();
if (pInfo.vMajor > NATRON_VERSION_MAJOR ||
(pInfo.vMajor == NATRON_VERSION_MAJOR && pInfo.vMinor > NATRON_VERSION_MINOR) ||
(pInfo.vMajor == NATRON_VERSION_MAJOR && pInfo.vMinor == NATRON_VERSION_MINOR && pInfo.vRev > NATRON_VERSION_REVISION)) {
QString message = tr("This project was saved with a more recent version (%1.%2.%3) of %4. Projects are not forward compatible and may only be opened in a version of %4 equal or more recent than the version that saved it.").arg(pInfo.vMajor).arg(pInfo.vMinor).arg(pInfo.vRev).arg(QString::fromUtf8(NATRON_APPLICATION_NAME));
throw std::runtime_error(message.toStdString());
}
throw std::runtime_error( tr("Unrecognized or damaged project file").toStdString() );
}
}

Format f;
Expand Down Expand Up @@ -599,12 +641,6 @@ Project::saveProjectInternal(const QString & path,
tmpFilename.append( QString::number( time.toMSecsSinceEpoch() ) );

{
FStreamsSupport::ofstream ofile;
FStreamsSupport::open( &ofile, tmpFilename.toStdString() );
if (!ofile) {
throw std::runtime_error( tr("Failed to open file ").toStdString() + tmpFilename.toStdString() );
}

///Fix file paths before saving.
QString oldProjectPath = QString::fromUtf8( _imp->getProjectPath().c_str() );

Expand All @@ -615,27 +651,62 @@ Project::saveProjectInternal(const QString & path,
_imp->natronVersion->setValue( generateUserFriendlyNatronVersionName() );
}

try {
boost::archive::xml_oarchive oArchive(ofile);
bool bgProject = getApp()->isBackground();
oArchive << boost::serialization::make_nvp("Background_project", bgProject);
ProjectSerialization projectSerializationObj( getApp() );
save(&projectSerializationObj);
oArchive << boost::serialization::make_nvp("Project", projectSerializationObj);
if (!bgProject) {
AppInstancePtr app = getApp();
if (app) {
app->saveProjectGui(oArchive);
if (!appPTR->getCurrentSettings()->saveAsBinary()){
try {
FStreamsSupport::ofstream ofile;
FStreamsSupport::open( &ofile, tmpFilename.toStdString(), std::ios::out);
if (!ofile) {
throw std::runtime_error( tr("Failed to open file ").toStdString() + tmpFilename.toStdString() );
}
boost::archive::xml_oarchive oArchive(ofile);
bool bgProject = getApp()->isBackground();
oArchive << boost::serialization::make_nvp("Background_project", bgProject);
ProjectSerialization projectSerializationObj( getApp() );
save(&projectSerializationObj);
oArchive << boost::serialization::make_nvp("Project", projectSerializationObj);
if (!bgProject) {
AppInstancePtr app = getApp();
if (app) {
app->saveProjectGui(oArchive);
}
}
} catch (...) {
if (!autoSave && updateProjectProperties) {
///Reset the old project path in case of failure.
_imp->autoSetProjectDirectory(oldProjectPath);
}
throw;
}
} catch (...) {
if (!autoSave && updateProjectProperties) {
///Reset the old project path in case of failure.
_imp->autoSetProjectDirectory(oldProjectPath);
}
else
{
try {
FStreamsSupport::ofstream ofile;
FStreamsSupport::open( &ofile, tmpFilename.toStdString(), std::ios::out | std::ios::binary);
if (!ofile) {
throw std::runtime_error( tr("Failed to open file ").toStdString() + tmpFilename.toStdString() );
}
boost::archive::binary_oarchive oArchive(ofile);
bool bgProject = getApp()->isBackground();
oArchive << boost::serialization::make_nvp("Background_project", bgProject);
ProjectSerialization projectSerializationObj( getApp() );
save(&projectSerializationObj);
oArchive << boost::serialization::make_nvp("Project", projectSerializationObj);
if (!bgProject) {
AppInstancePtr app = getApp();
if (app) {
app->saveProjectGui(oArchive);
}
}
} catch (...) {
if (!autoSave && updateProjectProperties) {
///Reset the old project path in case of failure.
_imp->autoSetProjectDirectory(oldProjectPath);
}
throw;
}
throw;
}
} // ofile
}

if (!autoSave) {
// rotate backups
Expand Down
13 changes: 13 additions & 0 deletions Engine/Settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,12 @@ Settings::initializeKnobsGeneral()
"last saved), *.~2~ (third last saved).") );
_generalTab->addKnob(_saveVersions);

_saveAsBinary = AppManager::createKnob<KnobBool>( this, tr("Save projects as binary") );
_saveAsBinary->setName("saveAsBinary");
_saveAsBinary->setHintToolTip( tr("When checked, %1 will save projects in a binary format which is faster to load and save. "
"When unchecked, projects will be saved in a human-readable text format. "
"Note that the text format is not guaranteed to be stable across versions of %1.").arg( QString::fromUtf8(NATRON_APPLICATION_NAME) ) );

_hostName = AppManager::createKnob<KnobChoice>( this, tr("Appear to plug-ins as") );
_hostName->setName("pluginHostName");
_hostName->setHintToolTip( tr("%1 will appear with the name of the selected application to the OpenFX plug-ins. "
Expand Down Expand Up @@ -1469,6 +1475,7 @@ Settings::setDefaultValues()
_autoSaveUnSavedProjects->setDefaultValue(true);
_autoSaveDelay->setDefaultValue(5, 0);
_saveVersions->setDefaultValue(1);
_saveAsBinary->setDefaultValue(false);
_hostName->setDefaultValue(0);
_customHostName->setDefaultValue(NATRON_ORGANIZATION_DOMAIN_TOPLEVEL "." NATRON_ORGANIZATION_DOMAIN_SUB "." NATRON_APPLICATION_NAME);

Expand Down Expand Up @@ -2912,6 +2919,12 @@ Settings::saveVersions() const
return _saveVersions->getValue();
}

bool
Settings::saveAsBinary() const
{
return _saveAsBinary->getValue();
}


bool
Settings::isSnapToNodeEnabled() const
Expand Down
3 changes: 3 additions & 0 deletions Engine/Settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,8 @@ GCC_DIAG_SUGGEST_OVERRIDE_ON

int saveVersions() const;

bool saveAsBinary() const;

bool isSnapToNodeEnabled() const;

bool isCheckForUpdatesEnabled() const;
Expand Down Expand Up @@ -434,6 +436,7 @@ GCC_DIAG_SUGGEST_OVERRIDE_ON
KnobBoolPtr _autoSaveUnSavedProjects;
KnobIntPtr _autoSaveDelay;
KnobIntPtr _saveVersions;
KnobBoolPtr _saveAsBinary;
KnobChoicePtr _hostName;
KnobStringPtr _customHostName;

Expand Down
2 changes: 2 additions & 0 deletions Gui/Gui.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,8 +305,10 @@ GCC_DIAG_SUGGEST_OVERRIDE_ON
static QPixmap screenShot(QWidget* w);

void loadProjectGui(bool isAutosave, boost::archive::xml_iarchive & obj) const;
void loadProjectGui(bool isAutosave, boost::archive::binary_iarchive & obj) const;

void saveProjectGui(boost::archive::xml_oarchive & archive);
void saveProjectGui(boost::archive::binary_oarchive & archive);

void setColorPickersColor(double r, double g, double b, double a);

Expand Down
14 changes: 14 additions & 0 deletions Gui/Gui20.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1465,13 +1465,27 @@ Gui::loadProjectGui(bool isAutosave, boost::archive::xml_iarchive & obj) const
_imp->_projectGui->load(isAutosave, obj);
}

void
Gui::loadProjectGui(bool isAutosave, boost::archive::binary_iarchive & obj) const
{
assert(_imp->_projectGui);
_imp->_projectGui->load(isAutosave, obj);
}

void
Gui::saveProjectGui(boost::archive::xml_oarchive & archive)
{
assert(_imp->_projectGui);
_imp->_projectGui->save(archive);
}

void
Gui::saveProjectGui(boost::archive::binary_oarchive & archive)
{
assert(_imp->_projectGui);
_imp->_projectGui->save(archive);
}

bool
Gui::isAboutToClose() const
{
Expand Down
14 changes: 14 additions & 0 deletions Gui/GuiAppInstance.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,12 @@ GuiAppInstance::loadProjectGui(bool isAutosave, boost::archive::xml_iarchive & a
_imp->_gui->loadProjectGui(isAutosave, archive);
}

void
GuiAppInstance::loadProjectGui(bool isAutosave, boost::archive::binary_iarchive & archive) const
{
_imp->_gui->loadProjectGui(isAutosave, archive);
}

void
GuiAppInstance::saveProjectGui(boost::archive::xml_oarchive & archive)
{
Expand All @@ -796,6 +802,14 @@ GuiAppInstance::saveProjectGui(boost::archive::xml_oarchive & archive)
}
}

void
GuiAppInstance::saveProjectGui(boost::archive::binary_oarchive & archive)
{
if (_imp->_gui) {
_imp->_gui->saveProjectGui(archive);
}
}

void
GuiAppInstance::setupViewersForViews(const std::vector<std::string>& viewNames)
{
Expand Down
2 changes: 2 additions & 0 deletions Gui/GuiAppInstance.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,9 @@ GCC_DIAG_SUGGEST_OVERRIDE_ON
Natron::StandardButtonEnum defaultButton,
bool* stopAsking) OVERRIDE FINAL WARN_UNUSED_RETURN;
virtual void loadProjectGui(bool isAutosave, boost::archive::xml_iarchive & archive) const OVERRIDE FINAL;
virtual void loadProjectGui(bool isAutosave, boost::archive::binary_iarchive & archive) const OVERRIDE FINAL;
virtual void saveProjectGui(boost::archive::xml_oarchive & archive) OVERRIDE FINAL;
virtual void saveProjectGui(boost::archive::binary_oarchive & archive) OVERRIDE FINAL;
virtual void notifyRenderStarted(const QString & sequenceName,
int firstFrame, int lastFrame,
int frameStep, bool canPause,
Expand Down
22 changes: 18 additions & 4 deletions Gui/ProjectGui.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,10 +261,18 @@ AddFormatDialog::getFormat() const
return Format(0, 0, w, h, name.toStdString(), pa);
}

template
void
ProjectGui::save<boost::archive::xml_oarchive>(boost::archive::xml_oarchive & archive) const;

template
void
ProjectGui::save<boost::archive::binary_oarchive>(boost::archive::binary_oarchive & archive) const;

// Version is handled in ProjectGuiSerialization
template<>
template<class Archive>
void
ProjectGui::save<boost::archive::xml_oarchive>(boost::archive::xml_oarchive & archive) const
ProjectGui::save(Archive & archive) const
{
ProjectGuiSerialization projectGuiSerializationObj;

Expand Down Expand Up @@ -430,9 +438,15 @@ loadNodeGuiSerialization(Gui* gui,
}
} // loadNodeGuiSerialization

template<>
template void
ProjectGui::load<boost::archive::xml_iarchive>(bool isAutosave, boost::archive::xml_iarchive & archive);

template void
ProjectGui::load<boost::archive::binary_iarchive>(bool isAutosave, boost::archive::binary_iarchive & archive);

template<class Archive>
void
ProjectGui::load<boost::archive::xml_iarchive>(bool isAutosave, boost::archive::xml_iarchive & archive)
ProjectGui::load(bool isAutosave, Archive & archive)
{
ProjectGuiSerialization obj;

Expand Down
Loading