Merge pull request ClickHouse#71722 from ClickHouse/clickhouse-local-persistency

Persistent databases in clickhouse-local

Author: alexey-milovidov

programs/local/LocalServer.cpp

@@ -52,6 +52,7 @@
 #include <boost/program_options/options_description.hpp>
 #include <base/argsToConfig.h>
 #include <filesystem>
+#include <Common/filesystemHelpers.h>
 
 #include "config.h"
 
@@ -280,7 +281,19 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str
 static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context)
 {
     auto overlay = std::make_shared<DatabasesOverlay>(name_, context);
-    overlay->registerNextDatabase(std::make_shared<DatabaseAtomic>(name_, fs::weakly_canonical(context->getPath()), UUIDHelpers::generateV4(), context));
+
+    UUID default_database_uuid;
+
+    fs::path existing_path_symlink = fs::weakly_canonical(context->getPath()) / "metadata" / "default";
+    if (FS::isSymlinkNoThrow(existing_path_symlink))
+        default_database_uuid = parse<UUID>(FS::readSymlink(existing_path_symlink).filename());
+    else
+        default_database_uuid = UUIDHelpers::generateV4();
+
+    fs::path default_database_metadata_path = fs::weakly_canonical(context->getPath()) / "store"
+        / DatabaseCatalog::getPathForUUID(default_database_uuid);
+
+    overlay->registerNextDatabase(std::make_shared<DatabaseAtomic>(name_, default_database_metadata_path, default_database_uuid, context));
     overlay->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context));
     return overlay;
 }
@@ -292,7 +305,7 @@ void LocalServer::tryInitPath()
 
     if (getClientConfiguration().has("path"))
     {
-        // User-supplied path.
+        /// User-supplied path.
         path = getClientConfiguration().getString("path");
         Poco::trimInPlace(path);
 
@@ -306,15 +319,15 @@ void LocalServer::tryInitPath()
     }
     else
     {
-        // The path is not provided explicitly - use a unique path in the system temporary directory
-        // (or in the current dir if a temporary doesn't exist)
+        /// The user requested to use a temporary path - use a unique path in the system temporary directory
+        /// (or in the current dir if a temporary doesn't exist)
         LoggerRawPtr log = &logger();
         std::filesystem::path parent_folder;
         std::filesystem::path default_path;
 
         try
         {
-            // try to guess a tmp folder name, and check if it's a directory (throw exception otherwise)
+            /// Try to guess a tmp folder name, and check if it's a directory (throw an exception otherwise).
             parent_folder = std::filesystem::temp_directory_path();
 
         }
@@ -342,7 +355,7 @@ void LocalServer::tryInitPath()
         temporary_directory_to_delete = default_path;
 
         path = default_path.string();
-        LOG_DEBUG(log, "Working directory created: {}", path);
+        LOG_DEBUG(log, "Working directory will be created as needed: {}", path);
     }
 
     global_context->setPath(fs::path(path) / "");
@@ -883,30 +896,38 @@ void LocalServer::processConfig()
 
     if (getClientConfiguration().has("path"))
     {
+        attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false);
+        attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
+        attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
+
         String path = global_context->getPath();
-        fs::create_directories(fs::path(path));
 
         /// Lock path directory before read
+        fs::create_directories(fs::path(path));
         status.emplace(fs::path(path) / "status", StatusFile::write_full_info);
 
-        LOG_DEBUG(log, "Loading metadata from {}", path);
-        auto load_system_metadata_tasks = loadMetadataSystem(global_context);
-        attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false);
-        attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
-        attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
-        waitLoad(TablesLoaderForegroundPoolId, load_system_metadata_tasks);
-
-        if (!getClientConfiguration().has("only-system-tables"))
+        if (fs::exists(fs::path(path) / "metadata"))
         {
-            DatabaseCatalog::instance().createBackgroundTasks();
-            waitLoad(loadMetadata(global_context));
-            DatabaseCatalog::instance().startupBackgroundTasks();
-        }
+            LOG_DEBUG(log, "Loading metadata from {}", path);
 
-        /// For ClickHouse local if path is not set the loader will be disabled.
-        global_context->getUserDefinedSQLObjectsStorage().loadObjects();
+            if (fs::exists(std::filesystem::path(path) / "metadata" / "system.sql"))
+            {
+                LoadTaskPtrs load_system_metadata_tasks = loadMetadataSystem(global_context);
+                waitLoad(TablesLoaderForegroundPoolId, load_system_metadata_tasks);
+            }
 
-        LOG_DEBUG(log, "Loaded metadata.");
+            if (!getClientConfiguration().has("only-system-tables"))
+            {
+                DatabaseCatalog::instance().createBackgroundTasks();
+                waitLoad(loadMetadata(global_context));
+                DatabaseCatalog::instance().startupBackgroundTasks();
+            }
+
+            /// For ClickHouse local if path is not set the loader will be disabled.
+            global_context->getUserDefinedSQLObjectsStorage().loadObjects();
+
+            LOG_DEBUG(log, "Loaded metadata.");
+        }
     }
     else if (!getClientConfiguration().has("no-system-tables"))
     {
@@ -981,7 +1002,7 @@ void LocalServer::addExtraOptions(OptionsDescription & options_description)
         ("logger.level", po::value<std::string>(), "Log level")
 
         ("no-system-tables", "do not attach system tables (better startup time)")
-        ("path", po::value<std::string>(), "Storage path")
+        ("path", po::value<std::string>(), "Storage path. If it was not specified, we will use a temporary directory, that is cleaned up on exit.")
         ("only-system-tables", "attach only system tables from specified path")
         ("top_level_domains_path", po::value<std::string>(), "Path to lists with custom TLDs")
         ;
@@ -1023,8 +1044,6 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp
         getClientConfiguration().setBool("no-system-tables", true);
     if (options.count("only-system-tables"))
         getClientConfiguration().setBool("only-system-tables", true);
-    if (options.count("database"))
-        getClientConfiguration().setString("default_database", options["database"].as<std::string>());
 
     if (options.count("input-format"))
         getClientConfiguration().setString("table-data-format", options["input-format"].as<std::string>());

programs/server/Server.cpp

@@ -2400,7 +2400,7 @@ try
 
     /// Set current database name before loading tables and databases because
     /// system logs may copy global context.
-    std::string default_database = server_settings[ServerSetting::default_database].toString();
+    std::string default_database = server_settings[ServerSetting::default_database];
     if (default_database.empty())
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "default_database cannot be empty");
     global_context->setCurrentDatabaseNameInGlobalContext(default_database);

src/Access/IAccessStorage.cpp

@@ -308,7 +308,7 @@ bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, bo
 {
     if (isReadOnly())
         throwReadonlyCannotInsert(entity->getType(), entity->getName());
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "insertImpl() is not implemented in {}", getStorageType());
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "insertImpl is not implemented in {}", getStorageType());
 }
 
 
@@ -404,7 +404,7 @@ bool IAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
             return false;
         throwReadonlyCannotRemove(entity->getType(), entity->getName());
     }
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "removeImpl() is not implemented in {}", getStorageType());
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "removeImpl is not implemented in {}", getStorageType());
 }
 
 
@@ -500,7 +500,7 @@ bool IAccessStorage::updateImpl(const UUID & id, const UpdateFunc &, bool throw_
             return false;
         throwReadonlyCannotUpdate(entity->getType(), entity->getName());
     }
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "updateImpl() is not implemented in {}", getStorageType());
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "updateImpl is not implemented in {}", getStorageType());
 }
 
 

src/Access/RowPolicy.cpp

@@ -45,7 +45,7 @@ void RowPolicy::setFullName(const RowPolicyName & full_name_)
 
 void RowPolicy::setName(const String &)
 {
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RowPolicy::setName() is not implemented");
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RowPolicy::setName is not implemented");
 }
 
 

src/Client/LocalConnection.cpp

@@ -616,7 +616,7 @@ bool LocalConnection::pollImpl()
 
 UInt64 LocalConnection::receivePacketType()
 {
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "receivePacketType() is not implemented for LocalConnection");
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "receivePacketType is not implemented for LocalConnection");
 }
 
 Packet LocalConnection::receivePacket()

src/DataTypes/DataTypeObjectDeprecated.h

@@ -30,7 +30,7 @@ class DataTypeObjectDeprecated : public IDataType
 
     Field getDefault() const override
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDefault() is not implemented for data type {}", getName());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDefault is not implemented for data type {}", getName());
     }
 
     bool haveSubtypes() const override { return false; }

src/DataTypes/IDataType.h

@@ -362,7 +362,7 @@ class IDataType : private boost::noncopyable, public std::enable_shared_from_thi
         bool throw_if_null) const
     {
         if (throw_if_null)
-            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDynamicSubcolumnData() is not implemented for type {}", getName());
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDynamicSubcolumnData is not implemented for type {}", getName());
         return nullptr;
     }
 };

src/DataTypes/IDataTypeDummy.h

@@ -28,17 +28,17 @@ class IDataTypeDummy : public IDataType
 public:
     MutableColumnPtr createColumn() const override
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method createColumn() is not implemented for data type {}", getName());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method createColumn is not implemented for data type {}", getName());
     }
 
     Field getDefault() const override
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDefault() is not implemented for data type {}", getName());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDefault is not implemented for data type {}", getName());
     }
 
     void insertDefaultInto(IColumn &) const override
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertDefaultInto() is not implemented for data type {}", getName());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertDefaultInto is not implemented for data type {}", getName());
     }
 
     bool haveSubtypes() const override { return false; }

src/Databases/DatabaseAtomic.cpp

@@ -4,17 +4,13 @@
 #include <Databases/DatabaseFactory.h>
 #include <Databases/DatabaseOnDisk.h>
 #include <Databases/DatabaseReplicated.h>
-#include <IO/ReadBufferFromFile.h>
-#include <IO/ReadHelpers.h>
-#include <Interpreters/Context.h>
 #include <Interpreters/DDLTask.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Storages/StorageMaterializedView.h>
 #include <Common/logger_useful.h>
 #include <Common/PoolId.h>
 #include <Common/atomicRename.h>
-#include <Common/filesystemHelpers.h>
 #include <Core/Settings.h>
 
 
@@ -54,8 +50,9 @@ class AtomicDatabaseTablesSnapshotIterator final : public DatabaseTablesSnapshot
 
 DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, const String & logger_name, ContextPtr context_)
     : DatabaseOrdinary(name_, metadata_path_, "store/", logger_name, context_)
-    , path_to_table_symlinks(fs::path("data") / escapeForFileName(name_) / "")
-    , path_to_metadata_symlink(fs::path("metadata") / escapeForFileName(name_))
+    , root_path(fs::weakly_canonical(context_->getPath()))
+    , path_to_table_symlinks(root_path / "data" / escapeForFileName(name_) / "")
+    , path_to_metadata_symlink(root_path / "metadata" / escapeForFileName(name_))
     , db_uuid(uuid)
 {
     assert(db_uuid != UUIDHelpers::Nil);
@@ -436,6 +433,8 @@ void DatabaseAtomic::setDetachedTableNotInUseForce(const UUID & uuid)
 DatabaseAtomic::DetachedTables DatabaseAtomic::cleanupDetachedTables()
 {
     DetachedTables not_in_use;
+    if (detached_tables.empty())
+        return not_in_use;
     auto it = detached_tables.begin();
     LOG_DEBUG(log, "There are {} detached tables. Start searching non used tables.", detached_tables.size());
     while (it != detached_tables.end())
@@ -573,15 +572,16 @@ void DatabaseAtomic::tryCreateSymlink(const StoragePtr & table, bool if_data_pat
 {
     if (!db_disk->isSymlinkSupported())
         return;
+
     try
     {
         String table_name = table->getStorageID().getTableName();
 
         if (!table->storesDataOnDisk())
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Table {} doesn't have data path to create symlink", table_name);
 
-        String link = path_to_table_symlinks + escapeForFileName(table_name);
-        fs::path data = fs::weakly_canonical(table->getDataPaths()[0]);
+        String link = path_to_table_symlinks / escapeForFileName(table_name);
+        fs::path data = fs::proximate(table->getDataPaths()[0], path_to_table_symlinks);
 
         /// If it already points where needed.
         if (db_disk->equivalentNoThrow(data, link))
@@ -590,7 +590,7 @@ void DatabaseAtomic::tryCreateSymlink(const StoragePtr & table, bool if_data_pat
         if (if_data_path_exist && !db_disk->existsFileOrDirectory(data))
             return;
 
-        db_disk->createDirectoriesSymlink(data, link);
+        db_disk->createDirectorySymlink(data, link);
     }
     catch (...)
     {
@@ -605,7 +605,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name)
 
     try
     {
-        String path = path_to_table_symlinks + escapeForFileName(table_name);
+        String path = path_to_table_symlinks / escapeForFileName(table_name);
         db_disk->removeFileIfExists(path);
     }
     catch (...)
@@ -621,21 +621,24 @@ void DatabaseAtomic::tryCreateMetadataSymlink()
 
     /// Symlinks in data/db_name/ directory and metadata/db_name/ are not used by ClickHouse,
     /// it's needed only for convenient introspection.
-    assert(path_to_metadata_symlink != metadata_path);
-    fs::path metadata_symlink(path_to_metadata_symlink);
-    if (db_disk->existsFileOrDirectory(metadata_symlink))
+    chassert(path_to_metadata_symlink != metadata_path);
+    if (db_disk->existsFileOrDirectory(path_to_metadata_symlink))
     {
-        if (!db_disk->isSymlink(metadata_symlink))
-            throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Directory {} exists", path_to_metadata_symlink);
+        if (!db_disk->isSymlink(path_to_metadata_symlink))
+            throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Directory {} already exists", path_to_metadata_symlink);
     }
     else
     {
         try
         {
             /// fs::exists could return false for broken symlink
-            if (db_disk->isSymlinkNoThrow(metadata_symlink))
-                db_disk->removeFileIfExists(metadata_symlink);
-            db_disk->createDirectoriesSymlink(metadata_path, path_to_metadata_symlink);
+            if (db_disk->isSymlinkNoThrow(path_to_metadata_symlink))
+                db_disk->removeFileIfExists(path_to_metadata_symlink);
+
+            String symlink = fs::proximate(root_path / metadata_path, path_to_metadata_symlink.parent_path());
+
+            LOG_TEST(log, "Creating directory symlink, path_to_metadata_symlink: {}, metadata_path: {}, symlink content: {}", path_to_metadata_symlink, metadata_path, symlink);
+            db_disk->createDirectorySymlink(symlink, path_to_metadata_symlink);
         }
         catch (...)
         {
@@ -670,8 +673,8 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new
     }
 
     auto new_name_escaped = escapeForFileName(new_name);
-    auto old_database_metadata_path = fs::path("metadata") / (escapeForFileName(getDatabaseName()) + ".sql");
-    auto new_database_metadata_path = fs::path("metadata") / (new_name_escaped + ".sql");
+    auto old_database_metadata_path = root_path / "metadata" / (escapeForFileName(getDatabaseName()) + ".sql");
+    auto new_database_metadata_path = root_path / "metadata" / (new_name_escaped + ".sql");
     db_disk->moveFile(old_database_metadata_path, new_database_metadata_path);
 
     String old_path_to_table_symlinks;
@@ -699,9 +702,9 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new
             snapshot.database = database_name;
         }
 
-        path_to_metadata_symlink = fs::path("metadata") / new_name_escaped;
+        path_to_metadata_symlink = root_path / "metadata" / new_name_escaped;
         old_path_to_table_symlinks = path_to_table_symlinks;
-        path_to_table_symlinks = fs::path("data") / new_name_escaped / "";
+        path_to_table_symlinks = root_path / "data" / new_name_escaped / "";
     }
 
     if (db_disk->isSymlinkSupported())

src/Databases/DatabaseAtomic.h

@@ -88,8 +88,9 @@ class DatabaseAtomic : public DatabaseOrdinary
     NameToPathMap table_name_to_path TSA_GUARDED_BY(mutex);
 
     DetachedTables detached_tables TSA_GUARDED_BY(mutex);
-    String path_to_table_symlinks;
-    String path_to_metadata_symlink;
+    std::filesystem::path root_path;
+    std::filesystem::path path_to_table_symlinks;
+    std::filesystem::path path_to_metadata_symlink;
     const UUID db_uuid;
 
     LoadTaskPtr startup_atomic_database_task TSA_GUARDED_BY(mutex);