Implementation of durable incremental backups (#525)

That is a partial fix of issue YTDB-333, the implementation of a new
algorithm for incremental backups.
The detailed design document can be read there
https://youtrack.jetbrains.com/articles/YTDB-A-10/Incremental-backup .
Key points of this change:
1. Incremental backup is made durable by keeping a single backup file's
hash code inside the metadata.
2. The last backup file is checked during the next iteration of the
incremental backup. If the backup is broken, the file is removed, and a
backup is performed since the changes of the previous backup.
3. It is possible to provide a virtual presentation of the file system
by providing lambda functions that implement the required file system
functionality.
4. New methods for public API were introduced:
`YTDBGraphGraphTraversalSource#backup`,
`YTDBGraphGraphTraversalSource#fullBackup` and `YouTrackDB#restore`.

Author: andrii0lomakin

.github/pull_request_template.md

@@ -4,9 +4,9 @@ A clear and concise explanation of why this change is necessary and what problem
 This section should provide context and justify the need for the change.
 This section is **MANDATORY**.
 
-
 #### Changes:
 
 A detailed description of the changes made in this pull request.
-This section should include any relevant code snippets, screenshots, or other supporting materials such as design documents and APRs.
+This section should include any relevant code snippets, screenshots, or other supporting materials
+such as design documents and APRs.
 This section is **MANDATORY**.

.github/workflows/maven-pipeline.yml

@@ -141,7 +141,7 @@ jobs:
           MAVEN_MIROR_PASSWORD: ${{ secrets.MAVEN_MIROR_PASSWORD }}
   notify-failure:
     name: Notify build failure on Zulip
-    needs: [ test, deploy, deploy-docker]
+    needs: [ test, deploy, deploy-docker ]
     if: failure()
     runs-on: ubuntu-latest
     steps:

.idea/inspectionProfiles/youtrackdb.xml

@@ -216,6 +216,10 @@
       <artifactId>youtrackdb-gremlin-annotations</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
     <dependency>
       <groupId>it.unimi.dsi</groupId>
       <artifactId>fastutil</artifactId>

core/data/tinkerpop-modern.json

@@ -303,8 +303,17 @@ YTDBGraphTraversalSource openTraversal(@Nonnull String databaseName, @Nonnull St
   @Nonnull
   YTDBGraphTraversalSource openTraversal(@Nonnull String databaseName);
 
-  /// Creates a database by restoring it from incremental backup. The backup should be created with
-  /// [#incrementalBackup(Path)].
+  /// Creates a database by restoring it from backup. The backup should be created with
+  /// [YTDBGraphTraversalSource#backup(Path)].
+  ///
+  /// At the moment only disk-based databases are supported, you cannot restore memory databases.
+  ///
+  /// @param databaseName Name of a database to be created.
+  /// @param path         Path to the backup directory.
+  void restore(@Nonnull String databaseName, @Nonnull String path);
+
+  /// Creates a database by restoring it from backup. The backup should be created with
+  /// [YTDBGraphTraversalSource#backup(Path)].
   ///
   /// At the moment only disk-based databases are supported, you cannot restore memory databases.
   ///
@@ -372,4 +381,17 @@ default void createSystemUser(@Nonnull String username, @Nonnull String password
 
     return userCredentialsArray;
   }
+
+  /// Creates a database by restoring it from backup. The backup should be created with
+  /// [YTDBGraphTraversalSource#backup(Path)].
+  ///
+  /// At the moment only disk-based databases are supported, you can not restore memory databases.
+  ///
+  /// @param databaseName Name of a database to be created.
+  /// @param path         Path to the backup directory.
+  /// @param expectedUUID UUID of the database to be restored. If null, the database will be
+  ///                     restored only if the directory contains backup only from one database.
+  /// @param config       database configuration
+  void restore(@Nonnull String databaseName, @Nonnull String path, @Nullable String expectedUUID,
+      @Nonnull Configuration config);
 }

core/data/tinkerpop-modern.kryo

@@ -54,11 +54,11 @@ public static YouTrackDB instance(@Nonnull String directoryPath,
   /// Create a new YouTrackDB manager instance for an embedded deployment with custom
   /// configuration.
   ///
-  /// A created instance is cached and reused if the method is called for the second time unless
-  /// it is closed by the user.
+  /// A created instance is cached and reused if the method is called for the second time unless it
+  /// is closed by the user.
   ///
-  /// @param directoryPath the directory where the databases are stored. For in memory database
-  ///                      use "."
+  /// @param directoryPath the directory where the databases are stored. For in memory database use
+  ///                      "."
   /// @param configuration custom configuration for current environment
   public static YouTrackDB instance(@Nonnull Path directoryPath,
       @Nonnull Configuration configuration) {
@@ -71,7 +71,6 @@ public static YouTrackDB instance(@Nonnull Path directoryPath,
   ///
   /// @param serverAddress server address
   /// @param serverPort    server port
-  ///
   /// @see YouTrackDB#createSystemUser(String, String, String...)
   /// @see YouTrackDB#createSystemUser(String, String, PredefinedSystemRole...)
   public static YouTrackDB instance(@Nonnull String serverAddress, int serverPort) {
@@ -89,7 +88,7 @@ public static YouTrackDB instance(@Nonnull String serverAddress, int serverPort)
     } catch (IllegalAccessException e) {
       throw new RuntimeException(e);
     }
-   }
+  }
 
 
   /// Creates a new YouTrackDB manager instance for a case when YTDB database is managed by Gremlin
@@ -104,7 +103,6 @@ public static YouTrackDB instance(@Nonnull String serverAddress, int serverPort)
   /// @param serverPort    server port
   /// @param username      user name
   /// @param password      user password
-  ///
   /// @see YouTrackDB#createSystemUser(String, String, String...)
   /// @see YouTrackDB#createSystemUser(String, String, PredefinedSystemRole...)
   public static YouTrackDB instance(@Nonnull String serverAddress, int serverPort,
@@ -127,21 +125,20 @@ public static YouTrackDB instance(@Nonnull String serverAddress, int serverPort,
   }
 
   /// Creates a new YouTrackDB manager instance for a case when YTDB database is managed by Gremlin
-  /// Server. The name of passed in user should belong to the user either registered in the
-  /// server configuration or system database.
+  /// Server. The name of passed in user should belong to the user either registered in the server
+  /// configuration or system database.
   ///
   /// Server connection will use port `8182` by default.
   ///
   /// Integration with SSO and user directories is going to be added soon.
   ///
-  /// As username and password are already passed during establishing the connection,
-  /// the method [YouTrackDB#openTraversal(String)]
-  /// is recommended to be used to send Gremlin queries to the server.
+  /// As username and password are already passed during establishing the connection, the method
+  /// [YouTrackDB#openTraversal(String)] is recommended to be used to send Gremlin queries to the
+  /// server.
   ///
   /// @param serverAddress server address
   /// @param username      user name
   /// @param password      user password
-  ///
   /// @see YouTrackDB#createSystemUser(String, String, String...)
   /// @see YouTrackDB#createSystemUser(String, String, PredefinedSystemRole...)
   public static YouTrackDB instance(@Nonnull String serverAddress, @Nonnull String username,

core/data/tinkerpop-modern.xml

@@ -377,6 +377,13 @@ public enum GlobalConfiguration {
       Integer.class,
       0),
 
+  STORAGE_BACKUP_MINIMUM_TIMEOUT_INTERVAL("youtrackdb.storage.minimumTimeOutInterval",
+      "Minimum interval between backups (in seconds). If users performs backup too often, "
+          + "storage will wait for this interval,"
+          + " as file system metadata update can be performed with delay in some filesystems. "
+          + "Default is 5 seconds.",
+      Integer.class, 5),
+
   // DATABASE
   DB_POOL_MIN("youtrackdb.db.pool.min", "Default database pool minimum size", Integer.class, 1),