From 1fbf1976335b9e7a7a893108fd7f28d77dcad4d9 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Wed, 16 Apr 2025 19:46:31 +0800 Subject: [PATCH 1/4] refactor: reorganize default catalog module structure Moving default catalog into separate crate --- Cargo.lock | 24 ++++++++++++++ Cargo.toml | 1 + src/query/default_catalog/Cargo.toml | 33 +++++++++++++++++++ .../src/databases/database.rs | 0 .../src/databases/database_context.rs | 3 +- .../src/databases/database_factory.rs | 0 .../src/databases/default/default_database.rs | 0 .../src/databases/default/mod.rs | 0 .../information_schema_database.rs | 4 +-- .../src/databases/information_schema/mod.rs | 0 .../default_catalog/src/databases/mod.rs | 26 +++++++++++++++ .../src/databases/system/mod.rs | 0 .../src/databases/system/system_database.rs | 4 +-- .../src}/default/catalog_context.rs | 2 +- .../src}/default/database_catalog.rs | 21 +++++++----- .../src}/default/immutable_catalog.rs | 4 +-- .../src}/default/mod.rs | 5 ++- .../src}/default/mutable_catalog.rs | 8 ++--- .../src}/default/session_catalog.rs | 5 +-- .../src/default/table_function_factory.rs | 10 ++++++ .../src}/default/table_memory_meta.rs | 3 +- src/query/default_catalog/src/lib.rs | 19 +++++++++++ src/query/service/Cargo.toml | 3 +- src/query/service/src/catalogs/mod.rs | 4 +-- src/query/service/src/databases/mod.rs | 13 +------- src/query/service/src/global_services.rs | 7 +++- src/query/service/src/table_functions/mod.rs | 2 +- .../table_functions/table_function_factory.rs | 15 +++++---- .../tests/it/catalogs/immutable_catalogs.rs | 2 +- src/query/service/tests/it/tests/catalog.rs | 6 +++- 30 files changed, 171 insertions(+), 53 deletions(-) create mode 100644 src/query/default_catalog/Cargo.toml rename src/query/{service => default_catalog}/src/databases/database.rs (100%) rename src/query/{service => default_catalog}/src/databases/database_context.rs (94%) rename src/query/{service => default_catalog}/src/databases/database_factory.rs (100%) rename src/query/{service => default_catalog}/src/databases/default/default_database.rs (100%) rename src/query/{service => default_catalog}/src/databases/default/mod.rs (100%) rename src/query/{service => default_catalog}/src/databases/information_schema/information_schema_database.rs (97%) rename src/query/{service => default_catalog}/src/databases/information_schema/mod.rs (100%) create mode 100644 src/query/default_catalog/src/databases/mod.rs rename src/query/{service => default_catalog}/src/databases/system/mod.rs (100%) rename src/query/{service => default_catalog}/src/databases/system/system_database.rs (99%) rename src/query/{service/src/catalogs => default_catalog/src}/default/catalog_context.rs (94%) rename src/query/{service/src/catalogs => default_catalog/src}/default/database_catalog.rs (98%) rename src/query/{service/src/catalogs => default_catalog/src}/default/immutable_catalog.rs (99%) rename src/query/{service/src/catalogs => default_catalog/src}/default/mod.rs (86%) rename src/query/{service/src/catalogs => default_catalog/src}/default/mutable_catalog.rs (99%) rename src/query/{service/src/catalogs => default_catalog/src}/default/session_catalog.rs (99%) create mode 100644 src/query/default_catalog/src/default/table_function_factory.rs rename src/query/{service/src/catalogs => default_catalog/src}/default/table_memory_meta.rs (98%) create mode 100644 src/query/default_catalog/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ce4d89490d54c..5e8fc31b8777d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5153,6 +5153,7 @@ dependencies = [ "databend-enterprise-stream-handler", "databend-enterprise-vacuum-handler", "databend-enterprise-virtual-column", + "databend-query-default-catalog", "databend-storages-common-blocks", "databend-storages-common-cache", "databend-storages-common-index", @@ -5233,6 +5234,29 @@ dependencies = [ "xorf", ] +[[package]] +name = "databend-query-default-catalog" +version = "0.1.0" +dependencies = [ + "async-backtrace", + "async-trait", + "dashmap 6.1.0", + "databend-common-catalog", + "databend-common-config", + "databend-common-exception", + "databend-common-meta-api", + "databend-common-meta-app", + "databend-common-meta-store", + "databend-common-meta-types", + "databend-common-storages-factory", + "databend-common-storages-information-schema", + "databend-common-storages-system", + "databend-storages-common-session", + "databend-storages-common-table-meta", + "fastrace", + "log", +] + [[package]] name = "databend-sqllogictests" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 431dda1d0272a..d821bddd524e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -208,6 +208,7 @@ databend-functions-scalar-math = { path = "src/query/functions/src/scalars/mathe databend-functions-scalar-numeric-basic-arithmetic = { path = "src/query/functions/src/scalars/numeric_basic_arithmetic" } databend-meta = { path = "src/meta/service" } databend-query = { path = "src/query/service" } +databend-query-default-catalog = { path = "src/query/default_catalog" } databend-sqllogictests = { path = "tests/sqllogictests" } databend-storages-common-blocks = { path = "src/query/storages/common/blocks" } databend-storages-common-cache = { path = "src/query/storages/common/cache" } diff --git a/src/query/default_catalog/Cargo.toml b/src/query/default_catalog/Cargo.toml new file mode 100644 index 0000000000000..bbaab14cc0d8c --- /dev/null +++ b/src/query/default_catalog/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "databend-query-default-catalog" +version = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +edition = { workspace = true } + +[dependencies] + +async-backtrace = { workspace = true } +async-trait = { workspace = true } +dashmap = { workspace = true } +fastrace = { workspace = true } +log = { workspace = true } + +databend-common-catalog = { workspace = true } +databend-common-config = { workspace = true } +databend-common-exception = { workspace = true } +databend-common-meta-api = { workspace = true } +databend-common-meta-app = { workspace = true } +databend-common-meta-store = { workspace = true } +databend-common-meta-types = { workspace = true } +databend-common-storages-factory = { workspace = true } +databend-common-storages-information-schema = { workspace = true } +databend-common-storages-system = { workspace = true } +databend-storages-common-session = { workspace = true } +databend-storages-common-table-meta = { workspace = true } + +[features] +jemalloc = ["databend-common-storages-system/jemalloc"] + + diff --git a/src/query/service/src/databases/database.rs b/src/query/default_catalog/src/databases/database.rs similarity index 100% rename from src/query/service/src/databases/database.rs rename to src/query/default_catalog/src/databases/database.rs diff --git a/src/query/service/src/databases/database_context.rs b/src/query/default_catalog/src/databases/database_context.rs similarity index 94% rename from src/query/service/src/databases/database_context.rs rename to src/query/default_catalog/src/databases/database_context.rs index a807beda998a1..c713c75ee3768 100644 --- a/src/query/service/src/databases/database_context.rs +++ b/src/query/default_catalog/src/databases/database_context.rs @@ -16,8 +16,7 @@ use std::sync::Arc; use databend_common_meta_app::tenant::Tenant; use databend_common_meta_store::MetaStore; - -use crate::storages::StorageFactory; +use databend_common_storages_factory::StorageFactory; /// Database Context. #[derive(Clone)] diff --git a/src/query/service/src/databases/database_factory.rs b/src/query/default_catalog/src/databases/database_factory.rs similarity index 100% rename from src/query/service/src/databases/database_factory.rs rename to src/query/default_catalog/src/databases/database_factory.rs diff --git a/src/query/service/src/databases/default/default_database.rs b/src/query/default_catalog/src/databases/default/default_database.rs similarity index 100% rename from src/query/service/src/databases/default/default_database.rs rename to src/query/default_catalog/src/databases/default/default_database.rs diff --git a/src/query/service/src/databases/default/mod.rs b/src/query/default_catalog/src/databases/default/mod.rs similarity index 100% rename from src/query/service/src/databases/default/mod.rs rename to src/query/default_catalog/src/databases/default/mod.rs diff --git a/src/query/service/src/databases/information_schema/information_schema_database.rs b/src/query/default_catalog/src/databases/information_schema/information_schema_database.rs similarity index 97% rename from src/query/service/src/databases/information_schema/information_schema_database.rs rename to src/query/default_catalog/src/databases/information_schema/information_schema_database.rs index fef701e3a3c2a..3f861d4a81643 100644 --- a/src/query/service/src/databases/information_schema/information_schema_database.rs +++ b/src/query/default_catalog/src/databases/information_schema/information_schema_database.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use databend_common_catalog::table::Table; use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::DatabaseId; use databend_common_meta_app::schema::DatabaseInfo; @@ -28,9 +29,8 @@ use databend_common_storages_information_schema::StatisticsTable; use databend_common_storages_information_schema::TablesTable; use databend_common_storages_information_schema::ViewsTable; -use crate::catalogs::InMemoryMetas; use crate::databases::Database; -use crate::storages::Table; +use crate::default::InMemoryMetas; #[derive(Clone)] pub struct InformationSchemaDatabase { diff --git a/src/query/service/src/databases/information_schema/mod.rs b/src/query/default_catalog/src/databases/information_schema/mod.rs similarity index 100% rename from src/query/service/src/databases/information_schema/mod.rs rename to src/query/default_catalog/src/databases/information_schema/mod.rs diff --git a/src/query/default_catalog/src/databases/mod.rs b/src/query/default_catalog/src/databases/mod.rs new file mode 100644 index 0000000000000..7cd74c91deccf --- /dev/null +++ b/src/query/default_catalog/src/databases/mod.rs @@ -0,0 +1,26 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod database; +mod database_context; +mod database_factory; +mod default; +mod information_schema; +mod system; + +pub use database::Database; +pub use database_context::DatabaseContext; +pub use database_factory::DatabaseFactory; +pub use information_schema::InformationSchemaDatabase; +pub use system::SystemDatabase; diff --git a/src/query/service/src/databases/system/mod.rs b/src/query/default_catalog/src/databases/system/mod.rs similarity index 100% rename from src/query/service/src/databases/system/mod.rs rename to src/query/default_catalog/src/databases/system/mod.rs diff --git a/src/query/service/src/databases/system/system_database.rs b/src/query/default_catalog/src/databases/system/system_database.rs similarity index 99% rename from src/query/service/src/databases/system/system_database.rs rename to src/query/default_catalog/src/databases/system/system_database.rs index 1fd89ed755469..ee83aca6feb72 100644 --- a/src/query/service/src/databases/system/system_database.rs +++ b/src/query/default_catalog/src/databases/system/system_database.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; +use databend_common_catalog::table::Table; use databend_common_config::InnerConfig; use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent; use databend_common_meta_app::schema::DatabaseId; @@ -71,9 +72,8 @@ use databend_common_storages_system::ViewsTableWithHistory; use databend_common_storages_system::ViewsTableWithoutHistory; use databend_common_storages_system::VirtualColumnsTable; -use crate::catalogs::InMemoryMetas; use crate::databases::Database; -use crate::storages::Table; +use crate::default::InMemoryMetas; #[derive(Clone)] pub struct SystemDatabase { diff --git a/src/query/service/src/catalogs/default/catalog_context.rs b/src/query/default_catalog/src/default/catalog_context.rs similarity index 94% rename from src/query/service/src/catalogs/default/catalog_context.rs rename to src/query/default_catalog/src/default/catalog_context.rs index 6253d84efa4d4..917567ecb3fa6 100644 --- a/src/query/service/src/catalogs/default/catalog_context.rs +++ b/src/query/default_catalog/src/default/catalog_context.rs @@ -15,9 +15,9 @@ use std::sync::Arc; use databend_common_meta_store::MetaStore; +use databend_common_storages_factory::StorageFactory; use crate::databases::DatabaseFactory; -use crate::storages::StorageFactory; #[derive(Clone)] pub struct CatalogContext { diff --git a/src/query/service/src/catalogs/default/database_catalog.rs b/src/query/default_catalog/src/default/database_catalog.rs similarity index 98% rename from src/query/service/src/catalogs/default/database_catalog.rs rename to src/query/default_catalog/src/default/database_catalog.rs index fcb68ba79c212..2b1d1a7d0642c 100644 --- a/src/query/service/src/catalogs/default/database_catalog.rs +++ b/src/query/default_catalog/src/default/database_catalog.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use databend_common_catalog::catalog::Catalog; use databend_common_catalog::catalog::StorageDescription; use databend_common_catalog::database::Database; +use databend_common_catalog::table::Table; use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_function::TableFunction; use databend_common_config::InnerConfig; @@ -113,11 +114,11 @@ use databend_common_meta_types::SeqV; use databend_storages_common_session::SessionState; use log::info; -use crate::catalogs::default::ImmutableCatalog; -use crate::catalogs::default::MutableCatalog; -use crate::catalogs::default::SessionCatalog; -use crate::storages::Table; -use crate::table_functions::TableFunctionFactory; +use crate::default::table_function_factory::TableFunctionFactory; +use crate::default::ImmutableCatalog; +use crate::default::MutableCatalog; +use crate::default::SessionCatalog; + /// Combine two catalogs together /// - read/search like operations are always performed at /// upper layer first, and bottom layer later(if necessary) @@ -129,7 +130,7 @@ pub struct DatabaseCatalog { /// bottom layer, writing goes here mutable_catalog: Arc, /// table function engine factories - table_function_factory: Arc, + table_function_factory: Arc, } impl Debug for DatabaseCatalog { @@ -140,15 +141,17 @@ impl Debug for DatabaseCatalog { impl DatabaseCatalog { #[async_backtrace::framed] - pub async fn try_create_with_config(conf: InnerConfig) -> Result { + pub async fn try_create_with_config( + conf: InnerConfig, + table_function_factory: Arc, + ) -> Result { let immutable_catalog = ImmutableCatalog::try_create_with_config(&conf).await?; let mutable_catalog = MutableCatalog::try_create_with_config(conf).await?; let session_catalog = SessionCatalog::create(mutable_catalog, SessionState::default()); - let table_function_factory = TableFunctionFactory::create(); let res = DatabaseCatalog { immutable_catalog: Arc::new(immutable_catalog), mutable_catalog: Arc::new(session_catalog), - table_function_factory: Arc::new(table_function_factory), + table_function_factory, }; Ok(res) } diff --git a/src/query/service/src/catalogs/default/immutable_catalog.rs b/src/query/default_catalog/src/default/immutable_catalog.rs similarity index 99% rename from src/query/service/src/catalogs/default/immutable_catalog.rs rename to src/query/default_catalog/src/default/immutable_catalog.rs index 3f7ef2bc68610..e0f734334ca3d 100644 --- a/src/query/service/src/catalogs/default/immutable_catalog.rs +++ b/src/query/default_catalog/src/default/immutable_catalog.rs @@ -18,6 +18,7 @@ use std::fmt::Formatter; use std::sync::Arc; use databend_common_catalog::catalog::Catalog; +use databend_common_catalog::table::Table; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -99,11 +100,10 @@ use databend_common_meta_types::SeqV; use databend_storages_common_table_meta::table_id_ranges::SYS_DB_ID_BEGIN; use databend_storages_common_table_meta::table_id_ranges::SYS_TBL_ID_BEGIN; -use crate::catalogs::InMemoryMetas; use crate::databases::Database; use crate::databases::InformationSchemaDatabase; use crate::databases::SystemDatabase; -use crate::storages::Table; +use crate::default::table_memory_meta::InMemoryMetas; /// System Catalog contains ... all the system databases (no surprise :) #[derive(Clone)] diff --git a/src/query/service/src/catalogs/default/mod.rs b/src/query/default_catalog/src/default/mod.rs similarity index 86% rename from src/query/service/src/catalogs/default/mod.rs rename to src/query/default_catalog/src/default/mod.rs index 70f330a9853ab..71c473292eb20 100644 --- a/src/query/service/src/catalogs/default/mod.rs +++ b/src/query/default_catalog/src/default/mod.rs @@ -17,10 +17,13 @@ mod database_catalog; mod immutable_catalog; mod mutable_catalog; mod session_catalog; -pub mod table_memory_meta; +mod table_function_factory; +mod table_memory_meta; pub use catalog_context::CatalogContext; pub use database_catalog::DatabaseCatalog; pub use immutable_catalog::ImmutableCatalog; pub use mutable_catalog::MutableCatalog; pub use session_catalog::SessionCatalog; +pub use table_function_factory::TableFunctionFactory; +pub use table_memory_meta::InMemoryMetas; diff --git a/src/query/service/src/catalogs/default/mutable_catalog.rs b/src/query/default_catalog/src/default/mutable_catalog.rs similarity index 99% rename from src/query/service/src/catalogs/default/mutable_catalog.rs rename to src/query/default_catalog/src/default/mutable_catalog.rs index bb6104f936898..9424bc7f0ce7d 100644 --- a/src/query/service/src/catalogs/default/mutable_catalog.rs +++ b/src/query/default_catalog/src/default/mutable_catalog.rs @@ -19,6 +19,8 @@ use std::sync::Arc; use std::time::Instant; use databend_common_catalog::catalog::Catalog; +use databend_common_catalog::catalog::StorageDescription; +use databend_common_catalog::table::Table; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -123,17 +125,15 @@ use databend_common_meta_app::KeyWithTenant; use databend_common_meta_store::MetaStoreProvider; use databend_common_meta_types::seq_value::SeqV; use databend_common_meta_types::MetaId; +use databend_common_storages_factory::StorageFactory; use fastrace::func_name; use log::info; use log::warn; -use crate::catalogs::default::catalog_context::CatalogContext; use crate::databases::Database; use crate::databases::DatabaseContext; use crate::databases::DatabaseFactory; -use crate::storages::StorageDescription; -use crate::storages::StorageFactory; -use crate::storages::Table; +use crate::default::CatalogContext; /// Catalog based on MetaStore /// - System Database NOT included diff --git a/src/query/service/src/catalogs/default/session_catalog.rs b/src/query/default_catalog/src/default/session_catalog.rs similarity index 99% rename from src/query/service/src/catalogs/default/session_catalog.rs rename to src/query/default_catalog/src/default/session_catalog.rs index 1c9aef0af73d0..3900aa63a8715 100644 --- a/src/query/service/src/catalogs/default/session_catalog.rs +++ b/src/query/default_catalog/src/default/session_catalog.rs @@ -16,6 +16,7 @@ use std::any::Any; use std::fmt::Debug; use std::sync::Arc; +use databend_common_catalog::catalog::Catalog; use databend_common_catalog::catalog::StorageDescription; use databend_common_catalog::database::Database; use databend_common_catalog::table::Table; @@ -114,8 +115,8 @@ use databend_storages_common_session::TxnState; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use databend_storages_common_table_meta::table_id_ranges::is_temp_table_id; -use crate::catalogs::default::MutableCatalog; -use crate::catalogs::Catalog; +use crate::default::MutableCatalog; + #[derive(Clone, Debug)] pub struct SessionCatalog { inner: MutableCatalog, diff --git a/src/query/default_catalog/src/default/table_function_factory.rs b/src/query/default_catalog/src/default/table_function_factory.rs new file mode 100644 index 0000000000000..b1fa147325a7c --- /dev/null +++ b/src/query/default_catalog/src/default/table_function_factory.rs @@ -0,0 +1,10 @@ +use std::sync::Arc; + +use databend_common_catalog::table_args::TableArgs; +use databend_common_catalog::table_function::TableFunction; +use databend_common_exception::Result; +pub trait TableFunctionFactory { + fn get(&self, func_name: &str, tbl_args: TableArgs) -> Result>; + fn exists(&self, func_name: &str) -> bool; + fn list(&self) -> Vec; +} diff --git a/src/query/service/src/catalogs/default/table_memory_meta.rs b/src/query/default_catalog/src/default/table_memory_meta.rs similarity index 98% rename from src/query/service/src/catalogs/default/table_memory_meta.rs rename to src/query/default_catalog/src/default/table_memory_meta.rs index 0bb61a8fd985c..cab79e66bf75e 100644 --- a/src/query/service/src/catalogs/default/table_memory_meta.rs +++ b/src/query/default_catalog/src/default/table_memory_meta.rs @@ -17,12 +17,11 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use dashmap::DashMap; +use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_meta_types::MetaId; -use crate::storages::Table; - pub struct DbTables { name_to_table: DashMap>, id_to_table: DashMap>, diff --git a/src/query/default_catalog/src/lib.rs b/src/query/default_catalog/src/lib.rs new file mode 100644 index 0000000000000..fedc2944236df --- /dev/null +++ b/src/query/default_catalog/src/lib.rs @@ -0,0 +1,19 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(clippy::result_large_err)] +pub mod databases; +mod default; + +pub use default::*; diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index 9ce2612fe9bef..9999615b23d5e 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -12,7 +12,7 @@ default = ["simd"] simd = ["databend-common-column/simd"] python-udf = ["arrow-udf-runtime/python"] disable_initial_exec_tls = ["databend-common-base/disable_initial_exec_tls"] -jemalloc = ["databend-common-storages-system/jemalloc"] +jemalloc = ["databend-common-storages-system/jemalloc", "databend-query-default-catalog/jemalloc"] memory-profiling = ["databend-common-base/memory-profiling", "databend-common-http/memory-profiling"] storage-hdfs = ["opendal/services-hdfs", "databend-common-storage/storage-hdfs"] io-uring = [ @@ -106,6 +106,7 @@ databend-enterprise-resources-management = { workspace = true } databend-enterprise-stream-handler = { workspace = true } databend-enterprise-vacuum-handler = { workspace = true } databend-enterprise-virtual-column = { workspace = true } +databend-query-default-catalog = { workspace = true } databend-storages-common-blocks = { workspace = true } databend-storages-common-cache = { workspace = true } databend-storages-common-index = { workspace = true } diff --git a/src/query/service/src/catalogs/mod.rs b/src/query/service/src/catalogs/mod.rs index 7064bfb752e6e..e127d9bdc2a4b 100644 --- a/src/query/service/src/catalogs/mod.rs +++ b/src/query/service/src/catalogs/mod.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod default; pub use databend_common_catalog::catalog::Catalog; pub use databend_common_storages_hive as hive; -pub use default::table_memory_meta::InMemoryMetas; -pub use default::DatabaseCatalog; +pub use databend_query_default_catalog::*; diff --git a/src/query/service/src/databases/mod.rs b/src/query/service/src/databases/mod.rs index 7cd74c91deccf..0da461f282095 100644 --- a/src/query/service/src/databases/mod.rs +++ b/src/query/service/src/databases/mod.rs @@ -12,15 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod database; -mod database_context; -mod database_factory; -mod default; -mod information_schema; -mod system; - -pub use database::Database; -pub use database_context::DatabaseContext; -pub use database_factory::DatabaseFactory; -pub use information_schema::InformationSchemaDatabase; -pub use system::SystemDatabase; +pub use databend_query_default_catalog::databases::*; diff --git a/src/query/service/src/global_services.rs b/src/query/service/src/global_services.rs index cabfd9e7dca23..a94651b20bfcd 100644 --- a/src/query/service/src/global_services.rs +++ b/src/query/service/src/global_services.rs @@ -53,6 +53,7 @@ use crate::servers::http::v1::ClientSessionManager; use crate::servers::http::v1::HttpQueryManager; use crate::sessions::QueriesQueueManager; use crate::sessions::SessionManager; +use crate::table_functions::TableFunctionFactoryRegistry; pub struct GlobalServices; @@ -101,7 +102,11 @@ impl GlobalServices { { // Init default catalog. - let default_catalog = DatabaseCatalog::try_create_with_config(config.clone()).await?; + let default_catalog = DatabaseCatalog::try_create_with_config( + config.clone(), + Arc::new(TableFunctionFactoryRegistry::create()), + ) + .await?; let catalog_creator: Vec<(CatalogType, Arc)> = vec![ (CatalogType::Iceberg, Arc::new(IcebergCreator)), diff --git a/src/query/service/src/table_functions/mod.rs b/src/query/service/src/table_functions/mod.rs index dc801b03ed018..c4d5262a74570 100644 --- a/src/query/service/src/table_functions/mod.rs +++ b/src/query/service/src/table_functions/mod.rs @@ -36,4 +36,4 @@ pub use openai::GPT2SQLTable; pub use others::LicenseInfoTable; pub use others::TenantQuotaTable; pub use table_function::TableFunction; -pub use table_function_factory::TableFunctionFactory; +pub use table_function_factory::TableFunctionFactoryRegistry; diff --git a/src/query/service/src/table_functions/table_function_factory.rs b/src/query/service/src/table_functions/table_function_factory.rs index 9b16261b83bd7..d7ad3bc41d4f4 100644 --- a/src/query/service/src/table_functions/table_function_factory.rs +++ b/src/query/service/src/table_functions/table_function_factory.rs @@ -34,6 +34,7 @@ use databend_common_storages_fuse::table_functions::SetCacheCapacity; use databend_common_storages_fuse::table_functions::TableFunctionTemplate; use databend_common_storages_iceberg::IcebergInspectTable; use databend_common_storages_stream::stream_status_table_func::StreamStatusTable; +use databend_query_default_catalog::TableFunctionFactory; use databend_storages_common_table_meta::table_id_ranges::SYS_TBL_FUC_ID_END; use databend_storages_common_table_meta::table_id_ranges::SYS_TBL_FUNC_ID_BEGIN; use itertools::Itertools; @@ -90,11 +91,11 @@ where } #[derive(Default)] -pub struct TableFunctionFactory { +pub struct TableFunctionFactoryRegistry { creators: TableFunctionCreators, } -impl TableFunctionFactory { +impl TableFunctionFactoryRegistry { pub fn create() -> Self { let mut id = SYS_TBL_FUNC_ID_BEGIN; let mut next_id = || -> MetaId { @@ -353,12 +354,14 @@ impl TableFunctionFactory { (next_id(), Arc::new(IcebergInspectTable::create)), ); - TableFunctionFactory { + TableFunctionFactoryRegistry { creators: RwLock::new(creators), } } +} - pub fn get(&self, func_name: &str, tbl_args: TableArgs) -> Result> { +impl TableFunctionFactory for TableFunctionFactoryRegistry { + fn get(&self, func_name: &str, tbl_args: TableArgs) -> Result> { let lock = self.creators.read(); let func_name = func_name.to_lowercase(); let (id, factory) = lock.get(&func_name).ok_or_else(|| { @@ -368,13 +371,13 @@ impl TableFunctionFactory { Ok(func) } - pub fn exists(&self, func_name: &str) -> bool { + fn exists(&self, func_name: &str) -> bool { let lock = self.creators.read(); let func_name = func_name.to_lowercase(); lock.contains_key(&func_name) } - pub fn list(&self) -> Vec { + fn list(&self) -> Vec { self.creators .read() .iter() diff --git a/src/query/service/tests/it/catalogs/immutable_catalogs.rs b/src/query/service/tests/it/catalogs/immutable_catalogs.rs index 365a3b3a83bb9..c534605f31344 100644 --- a/src/query/service/tests/it/catalogs/immutable_catalogs.rs +++ b/src/query/service/tests/it/catalogs/immutable_catalogs.rs @@ -20,8 +20,8 @@ use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::DropDatabaseReq; use databend_common_meta_app::schema::RenameDatabaseReq; use databend_common_meta_app::tenant::Tenant; -use databend_query::catalogs::default::ImmutableCatalog; use databend_query::catalogs::Catalog; +use databend_query::catalogs::ImmutableCatalog; use crate::tests::create_catalog; diff --git a/src/query/service/tests/it/tests/catalog.rs b/src/query/service/tests/it/tests/catalog.rs index 4cf83524348b3..be40f9b603824 100644 --- a/src/query/service/tests/it/tests/catalog.rs +++ b/src/query/service/tests/it/tests/catalog.rs @@ -12,10 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use databend_common_exception::Result; use databend_query::catalogs::DatabaseCatalog; +use databend_query::table_functions::TableFunctionFactoryRegistry; pub async fn create_catalog() -> Result { let conf = databend_query::test_kits::ConfigBuilder::create().config(); - DatabaseCatalog::try_create_with_config(conf).await + DatabaseCatalog::try_create_with_config(conf, Arc::new(TableFunctionFactoryRegistry::create())) + .await } From 71cad1a3882060e8b72bdef0b3d2664daaa62c42 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Wed, 16 Apr 2025 20:43:23 +0800 Subject: [PATCH 2/4] Add license header --- .../src/default/table_function_factory.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/query/default_catalog/src/default/table_function_factory.rs b/src/query/default_catalog/src/default/table_function_factory.rs index b1fa147325a7c..0d84f4e1f8dbc 100644 --- a/src/query/default_catalog/src/default/table_function_factory.rs +++ b/src/query/default_catalog/src/default/table_function_factory.rs @@ -1,3 +1,17 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::sync::Arc; use databend_common_catalog::table_args::TableArgs; From 8ebb208c12d754c509a7a953fd5f2b83c3fbcf9a Mon Sep 17 00:00:00 2001 From: dantengsky Date: Wed, 16 Apr 2025 21:02:36 +0800 Subject: [PATCH 3/4] remove unused dependencies --- Cargo.lock | 1 - src/query/service/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e8fc31b8777d..e43ce288db7dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5132,7 +5132,6 @@ dependencies = [ "databend-common-storages-fuse", "databend-common-storages-hive", "databend-common-storages-iceberg", - "databend-common-storages-information-schema", "databend-common-storages-memory", "databend-common-storages-null", "databend-common-storages-orc", diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index 9999615b23d5e..3b25cbaa793a6 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -85,7 +85,6 @@ databend-common-storages-factory = { workspace = true } databend-common-storages-fuse = { workspace = true } databend-common-storages-hive = { workspace = true } databend-common-storages-iceberg = { workspace = true } -databend-common-storages-information-schema = { workspace = true } databend-common-storages-memory = { workspace = true } databend-common-storages-null = { workspace = true } databend-common-storages-orc = { workspace = true } From 4ca54bc1cdd3bff49b95136e05f26208cba0bf00 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Wed, 16 Apr 2025 21:16:12 +0800 Subject: [PATCH 4/4] taplo fmt --- src/query/default_catalog/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/query/default_catalog/Cargo.toml b/src/query/default_catalog/Cargo.toml index bbaab14cc0d8c..43f5105d8332f 100644 --- a/src/query/default_catalog/Cargo.toml +++ b/src/query/default_catalog/Cargo.toml @@ -29,5 +29,3 @@ databend-storages-common-table-meta = { workspace = true } [features] jemalloc = ["databend-common-storages-system/jemalloc"] - -