From 69ae8b59223feb6fca2005c8801a3e4da43218c7 Mon Sep 17 00:00:00 2001 From: alvin-phoenix-ai Date: Sun, 12 Jul 2026 21:16:50 +0800 Subject: [PATCH 1/2] [Refactor] Extract schema scanner factory from Exec Signed-off-by: alvin-phoenix-ai --- be/AGENTS.md | 9 +- be/module_boundary_manifest.json | 13 +- be/src/exec/CMakeLists.txt | 17 +- .../exec/builtin_schema_scanner_factory.cpp | 207 ++++++++++++++++++ be/src/exec/builtin_schema_scanner_factory.h | 25 +++ be/src/exec/exec_env.cpp | 17 +- be/src/exec/exec_env.h | 7 + .../pipeline/scan/schema_chunk_source.cpp | 10 +- be/src/exec/schema_scan_node.cpp | 12 +- be/src/exec/schema_scanner.h | 3 - be/src/exec/schema_scanner_factory.cpp | 184 +--------------- be/src/exec/schema_scanner_factory.h | 34 +++ be/src/service/CMakeLists.txt | 3 +- be/src/service/service_be/starrocks_be.cpp | 3 +- be/test/CMakeLists.txt | 4 +- be/test/data_workflows/CMakeLists.txt | 2 +- be/test/exec/CMakeLists.txt | 1 + .../schema_scanner_core_test.cpp | 44 ++++ .../schema_scanner/schema_scanner_test.cpp | 10 +- be/test/runtime/exec_env_test.cpp | 26 +++ be/test/test_main.cpp | 3 +- be/test/testutil/init_test_env.h | 5 +- 22 files changed, 434 insertions(+), 205 deletions(-) create mode 100644 be/src/exec/builtin_schema_scanner_factory.cpp create mode 100644 be/src/exec/builtin_schema_scanner_factory.h create mode 100644 be/src/exec/schema_scanner_factory.h diff --git a/be/AGENTS.md b/be/AGENTS.md index 9d9e19ff58c67e..20edfb32f487a7 100644 --- a/be/AGENTS.md +++ b/be/AGENTS.md @@ -295,11 +295,18 @@ Default BE module bootstrap composition for built-in module registration, includ ### ExecSchemaScannerCore (`execschemascannercore`) Schema scanner base contract and shared mechanics without concrete scanner, pipeline, storage, service, or ExecEnv coupling. - Targets: `ExecSchemaScannerCore` -- Allowed internal include prefixes: `exec/schema_scanner.h`, `exprs/`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` +- Allowed internal include prefixes: `exec/schema_scanner.h`, `exec/schema_scanner_factory.h`, `exprs/`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` - Allowed target deps: `Expr`, `Runtime`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen` - Core tests: `schema_scanner_core_test` - Remediation: Keep SchemaScannerCore limited to the base SchemaScanner contract; move concrete scanner creation and service-specific logic into higher schema scanner modules. +### SchemaScannerBuiltin (`schemascannerbuiltin`) +Temporary top-level composition target for the builtin schema scanner factory while concrete scanners remain in Exec compatibility targets. +- Targets: `SchemaScannerBuiltin` +- Allowed internal include prefixes: `exec/builtin_schema_scanner_factory.h`, `exec/schema_scanner.h`, `exec/schema_scanner_factory.h`, `exec/schema_scanner/`, `gen_cpp/` +- Allowed target deps: `Exec`, `ExecSchemaScannerCore`, `ExecSchemaScanners` +- Remediation: Keep builtin scanner selection above Exec; PR2 must replace the temporary Exec dependency with explicit layered schema scanner targets. + ### ExecSchemaScanners (`execschemascanners`) Clean concrete schema scanners that do not depend on SchemaHelper, FE RPC/client helpers, storage, service, cache, pipeline, or ExecEnv. - Targets: `ExecSchemaScanners` diff --git a/be/module_boundary_manifest.json b/be/module_boundary_manifest.json index 99a715f1ce0009..cc3a1af3c5a840 100644 --- a/be/module_boundary_manifest.json +++ b/be/module_boundary_manifest.json @@ -591,6 +591,7 @@ "allowed_test_targets": ["data_workflows_test"], "allowed_test_link_deps": [ "DataWorkflows", + "SchemaScannerBuiltin", "Storage", "Exec", "ConnectorFile", @@ -1157,7 +1158,8 @@ "doc_label": "ExecSchemaScannerCore", "summary": "Schema scanner base contract and shared mechanics without concrete scanner, pipeline, storage, service, or ExecEnv coupling.", "owned_targets": ["ExecSchemaScannerCore"], - "allowed_include_prefixes": ["exec/schema_scanner.h", "exprs/", "runtime/", "column/", "types/", "common/", "base/", "gutil/", "gen_cpp/"], + "additional_owned_globs": ["be/src/exec/schema_scanner_factory.h"], + "allowed_include_prefixes": ["exec/schema_scanner.h", "exec/schema_scanner_factory.h", "exprs/", "runtime/", "column/", "types/", "common/", "base/", "gutil/", "gen_cpp/"], "forbidden_include_prefixes": ["exec/schema_scanner/", "exec/pipeline/", "storage/", "http/", "service/", "cache/"], "forbidden_includes": [ "exec/exec_env.h", @@ -1169,6 +1171,15 @@ "allowed_test_link_deps": ["ExecSchemaScannerCore", "Expr", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], "remediation": "Keep SchemaScannerCore limited to the base SchemaScanner contract; move concrete scanner creation and service-specific logic into higher schema scanner modules." }, + { + "id": "schemascannerbuiltin", + "doc_label": "SchemaScannerBuiltin", + "summary": "Temporary top-level composition target for the builtin schema scanner factory while concrete scanners remain in Exec compatibility targets.", + "owned_targets": ["SchemaScannerBuiltin"], + "allowed_include_prefixes": ["exec/builtin_schema_scanner_factory.h", "exec/schema_scanner.h", "exec/schema_scanner_factory.h", "exec/schema_scanner/", "gen_cpp/"], + "allowed_target_deps": ["Exec", "ExecSchemaScannerCore", "ExecSchemaScanners"], + "remediation": "Keep builtin scanner selection above Exec; PR2 must replace the temporary Exec dependency with explicit layered schema scanner targets." + }, { "id": "execschemascanners", "doc_label": "ExecSchemaScanners", diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 8cf8024eae1a1b..72db248d79c35c 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -22,6 +22,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/exec") set(EXEC_SCHEMA_SCANNER_CORE_FILES schema_scanner.cpp + schema_scanner_factory.cpp ) ADD_BE_LIB(ExecSchemaScannerCore @@ -313,7 +314,6 @@ set(EXEC_NON_PIPELINE_FILES dict_decode_node.cpp repeat_node.cpp table_function_node.cpp - schema_scanner_factory.cpp schema_scan_node.cpp dictionary_cache_writer.cpp arrow_flight_batch_reader.cpp @@ -490,3 +490,18 @@ ADD_BE_LIB(Exec target_link_libraries(Exec PUBLIC ExecRuntime ExecPrimitive ExecSchemaScannerCore ExecSchemaScanners ExecJoinCore Runtime ComputeEnv Cache) target_link_libraries(Exec PRIVATE Formats Platform Connector ConnectorLake ConnectorFile ConnectorHive ConnectorIceberg) + +ADD_BE_LIB(SchemaScannerBuiltin + builtin_schema_scanner_factory.cpp +) + +target_link_libraries(SchemaScannerBuiltin PUBLIC + ExecSchemaScannerCore +) + +# Temporary PR1 dependency: concrete schema scanners are still compiled into +# Exec and ExecSchemaScanners. PR2 will replace these with layered scanner targets. +target_link_libraries(SchemaScannerBuiltin PRIVATE + Exec + ExecSchemaScanners +) diff --git a/be/src/exec/builtin_schema_scanner_factory.cpp b/be/src/exec/builtin_schema_scanner_factory.cpp new file mode 100644 index 00000000000000..c1aa97f9a1c032 --- /dev/null +++ b/be/src/exec/builtin_schema_scanner_factory.cpp @@ -0,0 +1,207 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/builtin_schema_scanner_factory.h" + +#include "exec/schema_scanner/schema_analyze_status.h" +#include "exec/schema_scanner/schema_applicable_roles_scanner.h" +#include "exec/schema_scanner/schema_be_bvars_scanner.h" +#include "exec/schema_scanner/schema_be_cloud_native_compactions_scanner.h" +#include "exec/schema_scanner/schema_be_compactions_scanner.h" +#include "exec/schema_scanner/schema_be_configs_scanner.h" +#include "exec/schema_scanner/schema_be_datacache_metrics_scanner.h" +#include "exec/schema_scanner/schema_be_logs_scanner.h" +#include "exec/schema_scanner/schema_be_metrics_scanner.h" +#include "exec/schema_scanner/schema_be_tablet_write_log_scanner.h" +#include "exec/schema_scanner/schema_be_tablets_scanner.h" +#include "exec/schema_scanner/schema_be_threads_scanner.h" +#include "exec/schema_scanner/schema_be_txns_scanner.h" +#include "exec/schema_scanner/schema_charsets_scanner.h" +#include "exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h" +#include "exec/schema_scanner/schema_cluster_snapshots_scanner.h" +#include "exec/schema_scanner/schema_collation_character_set_applicability_scanner.h" +#include "exec/schema_scanner/schema_collations_scanner.h" +#include "exec/schema_scanner/schema_column_stats_usage_scanner.h" +#include "exec/schema_scanner/schema_columns_scanner.h" +#include "exec/schema_scanner/schema_dummy_scanner.h" +#include "exec/schema_scanner/schema_fe_metrics_scanner.h" +#include "exec/schema_scanner/schema_fe_tablet_schedules_scanner.h" +#include "exec/schema_scanner/schema_fe_threads_scanner.h" +#include "exec/schema_scanner/schema_keywords_scanner.h" +#include "exec/schema_scanner/schema_load_tracking_logs_scanner.h" +#include "exec/schema_scanner/schema_loads_scanner.h" +#include "exec/schema_scanner/schema_materialized_view_refresh_jobs_scanner.h" +#include "exec/schema_scanner/schema_materialized_views_scanner.h" +#include "exec/schema_scanner/schema_partitions_meta_scanner.h" +#include "exec/schema_scanner/schema_pipe_files.h" +#include "exec/schema_scanner/schema_pipes.h" +#include "exec/schema_scanner/schema_recyclebin_catalogs.h" +#include "exec/schema_scanner/schema_routine_load_jobs_scanner.h" +#include "exec/schema_scanner/schema_schema_privileges_scanner.h" +#include "exec/schema_scanner/schema_schemata_scanner.h" +#include "exec/schema_scanner/schema_stream_loads_scanner.h" +#include "exec/schema_scanner/schema_table_privileges_scanner.h" +#include "exec/schema_scanner/schema_tables_config_scanner.h" +#include "exec/schema_scanner/schema_tables_scanner.h" +#include "exec/schema_scanner/schema_tablet_reshard_jobs_scanner.h" +#include "exec/schema_scanner/schema_task_runs_scanner.h" +#include "exec/schema_scanner/schema_tasks_scanner.h" +#include "exec/schema_scanner/schema_temp_tables_scanner.h" +#include "exec/schema_scanner/schema_user_privileges_scanner.h" +#include "exec/schema_scanner/schema_variables_scanner.h" +#include "exec/schema_scanner/schema_views_scanner.h" +#include "exec/schema_scanner/schema_warehouse_metrics.h" +#include "exec/schema_scanner/schema_warehouse_queries.h" +#include "exec/schema_scanner/starrocks_grants_to_scanner.h" +#include "exec/schema_scanner/starrocks_role_edges_scanner.h" +#include "exec/schema_scanner/sys_fe_locks.h" +#include "exec/schema_scanner/sys_fe_memory_usage.h" +#include "exec/schema_scanner/sys_object_dependencies.h" + +namespace starrocks { + +namespace { + +class BuiltinSchemaScannerFactory final : public SchemaScannerFactory { +public: + std::unique_ptr create(TSchemaTableType::type type) const override; +}; + +std::unique_ptr BuiltinSchemaScannerFactory::create(TSchemaTableType::type type) const { + switch (type) { + case TSchemaTableType::SCH_TABLES: + return std::make_unique(); + case TSchemaTableType::SCH_SCHEMATA: + return std::make_unique(); + case TSchemaTableType::SCH_COLUMNS: + return std::make_unique(); + case TSchemaTableType::SCH_CHARSETS: + return std::make_unique(); + case TSchemaTableType::SCH_COLLATIONS: + return std::make_unique(); + case TSchemaTableType::SCH_COLLATION_CHARACTER_SET_APPLICABILITY: + return std::make_unique(); + case TSchemaTableType::SCH_GLOBAL_VARIABLES: + return std::make_unique(TVarType::GLOBAL); + case TSchemaTableType::SCH_SESSION_VARIABLES: + case TSchemaTableType::SCH_VARIABLES: + return std::make_unique(TVarType::SESSION); + case TSchemaTableType::SCH_USER_PRIVILEGES: + return std::make_unique(); + case TSchemaTableType::SCH_SCHEMA_PRIVILEGES: + return std::make_unique(); + case TSchemaTableType::SCH_TABLE_PRIVILEGES: + return std::make_unique(); + case TSchemaTableType::SCH_VIEWS: + return std::make_unique(); + case TSchemaTableType::SCH_TASKS: + return std::make_unique(); + case TSchemaTableType::SCH_TASK_RUNS: + return std::make_unique(); + case TSchemaTableType::SCH_MATERIALIZED_VIEWS: + return std::make_unique(); + case TSchemaTableType::SCH_MATERIALIZED_VIEW_REFRESH_JOBS: + return std::make_unique(); + case TSchemaTableType::SCH_LOADS: + return std::make_unique(); + case TSchemaTableType::SCH_LOAD_TRACKING_LOGS: + return std::make_unique(); + case TSchemaTableType::SCH_TABLES_CONFIG: + return std::make_unique(); + case TSchemaTableType::SCH_VERBOSE_SESSION_VARIABLES: + return std::make_unique(TVarType::VERBOSE); + case TSchemaTableType::SCH_BE_TABLETS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_METRICS: + return std::make_unique(); + case TSchemaTableType::SCH_FE_METRICS: + return std::make_unique(); + case TSchemaTableType::SCH_FE_THREADS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_TXNS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_CONFIGS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_THREADS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_LOGS: + return std::make_unique(); + case TSchemaTableType::SCH_FE_TABLET_SCHEDULES: + return std::make_unique(); + case TSchemaTableType::SCH_BE_COMPACTIONS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_BVARS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_CLOUD_NATIVE_COMPACTIONS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_TABLET_WRITE_LOG: + return std::make_unique(); + case TSchemaTableType::STARROCKS_ROLE_EDGES: + return std::make_unique(); + case TSchemaTableType::STARROCKS_GRANT_TO_ROLES: + return std::make_unique(TGrantsToType::ROLE); + case TSchemaTableType::STARROCKS_GRANT_TO_USERS: + return std::make_unique(TGrantsToType::USER); + case TSchemaTableType::STARROCKS_OBJECT_DEPENDENCIES: + return std::make_unique(); + case TSchemaTableType::SCH_ROUTINE_LOAD_JOBS: + return std::make_unique(); + case TSchemaTableType::SCH_STREAM_LOADS: + return std::make_unique(); + case TSchemaTableType::SCH_PIPE_FILES: + return std::make_unique(); + case TSchemaTableType::SCH_PIPES: + return std::make_unique(); + case TSchemaTableType::SYS_FE_LOCKS: + return std::make_unique(); + case TSchemaTableType::SCH_BE_DATACACHE_METRICS: + return std::make_unique(); + case TSchemaTableType::SCH_PARTITIONS_META: + return std::make_unique(); + case TSchemaTableType::SYS_FE_MEMORY_USAGE: + return std::make_unique(); + case TSchemaTableType::SCH_TEMP_TABLES: + return std::make_unique(); + case TSchemaTableType::SCH_RECYCLEBIN_CATALOGS: + return std::make_unique(); + case TSchemaTableType::SCH_COLUMN_STATS_USAGE: + return std::make_unique(); + case TSchemaTableType::SCH_ANALYZE_STATUS: + return std::make_unique(); + case TSchemaTableType::SCH_CLUSTER_SNAPSHOTS: + return std::make_unique(); + case TSchemaTableType::SCH_CLUSTER_SNAPSHOT_JOBS: + return std::make_unique(); + case TSchemaTableType::SCH_APPLICABLE_ROLES: + return std::make_unique(); + case TSchemaTableType::SCH_KEYWORDS: + return std::make_unique(); + case TSchemaTableType::SCH_WAREHOUSE_METRICS: + return std::make_unique(); + case TSchemaTableType::SCH_WAREHOUSE_QUERIES: + return std::make_unique(); + case TSchemaTableType::SCH_TABLET_RESHARD_JOBS: + return std::make_unique(); + default: + return std::make_unique(); + } +} + +} // namespace + +std::unique_ptr create_builtin_schema_scanner_factory() { + return std::make_unique(); +} + +} // namespace starrocks diff --git a/be/src/exec/builtin_schema_scanner_factory.h b/be/src/exec/builtin_schema_scanner_factory.h new file mode 100644 index 00000000000000..eadcff921eb01c --- /dev/null +++ b/be/src/exec/builtin_schema_scanner_factory.h @@ -0,0 +1,25 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "exec/schema_scanner_factory.h" + +namespace starrocks { + +std::unique_ptr create_builtin_schema_scanner_factory(); + +} // namespace starrocks diff --git a/be/src/exec/exec_env.cpp b/be/src/exec/exec_env.cpp index 4dcc4e0e24fa65..0c7a6cb60e08a4 100644 --- a/be/src/exec/exec_env.cpp +++ b/be/src/exec/exec_env.cpp @@ -54,6 +54,7 @@ #include "exec/lookup_stream_mgr.h" #include "exec/pipeline/query_context.h" #include "exec/runtime/query_context_manager.h" +#include "exec/schema_scanner_factory.h" #include "exec/stream_load/stream_load_executor.h" #include "exec/stream_load/transaction_mgr.h" #include "exec_primitive/pipeline/primitives/driver_executor.h" @@ -84,7 +85,10 @@ ExecEnv* ExecEnv::GetInstance() { return &s_exec_env; } -ExecEnv::ExecEnv() : _runtime_env(RuntimeEnv::GetInstance()) { +ExecEnv::ExecEnv() : ExecEnv(nullptr) {} + +ExecEnv::ExecEnv(std::unique_ptr schema_scanner_factory) + : _runtime_env(RuntimeEnv::GetInstance()), _schema_scanner_factory(std::move(schema_scanner_factory)) { _refresh_service_contexts(); } ExecEnv::~ExecEnv() = default; @@ -178,7 +182,8 @@ void ExecEnv::set_runtime_filter_services(RuntimeFilterSender* sender, RuntimeFi _refresh_service_contexts(); } -Status ExecEnv::init(ProcessMetricsRegistry* process_metrics_registry, RuntimeEnv* runtime_env) { +Status ExecEnv::init(ProcessMetricsRegistry* process_metrics_registry, RuntimeEnv* runtime_env, + std::unique_ptr schema_scanner_factory) { DCHECK(process_metrics_registry != nullptr); DCHECK(runtime_env != nullptr); _runtime_env = runtime_env; @@ -191,6 +196,9 @@ Status ExecEnv::init(ProcessMetricsRegistry* process_metrics_registry, RuntimeEn if (_compute_env == nullptr) { return Status::InternalError("ComputeEnv is not attached"); } + if (schema_scanner_factory != nullptr) { + _schema_scanner_factory = std::move(schema_scanner_factory); + } _process_metrics_registry = process_metrics_registry; auto* process_metrics = process_metrics_registry->root_registry(); _table_metrics_mgr = process_metrics_registry->table_metrics_mgr(); @@ -235,6 +243,10 @@ Status ExecEnv::init(ProcessMetricsRegistry* process_metrics_registry, RuntimeEn return Status::OK(); } +Status ExecEnv::init(ProcessMetricsRegistry* process_metrics_registry, RuntimeEnv* runtime_env) { + return init(process_metrics_registry, runtime_env, nullptr); +} + DataStreamMgr* ExecEnv::stream_mgr() { return _compute_env == nullptr ? nullptr : _compute_env->stream_mgr(); } @@ -323,6 +335,7 @@ void ExecEnv::destroy() { _table_metrics_mgr = nullptr; _process_metrics_registry = nullptr; _compute_env = nullptr; + _schema_scanner_factory.reset(); _refresh_service_contexts(); } diff --git a/be/src/exec/exec_env.h b/be/src/exec/exec_env.h index 7af9562019112a..60bedf8d389070 100644 --- a/be/src/exec/exec_env.h +++ b/be/src/exec/exec_env.h @@ -35,6 +35,7 @@ #pragma once #include +#include #include "common/status.h" #include "common/thread/threadpool.h" @@ -67,6 +68,7 @@ class ThreadPool; class PriorityThreadPool; class ResultBufferMgr; class ResultQueueMgr; +class SchemaScannerFactory; class WebPageHandler; class StreamLoadExecutor; class RuntimeFilterCache; @@ -94,6 +96,8 @@ class ExecEnv { public: // Initial exec environment. must call this to init all Status init(ProcessMetricsRegistry* process_metrics_registry, RuntimeEnv* runtime_env); + Status init(ProcessMetricsRegistry* process_metrics_registry, RuntimeEnv* runtime_env, + std::unique_ptr schema_scanner_factory); void stop(); void clear_query_contexts(); void destroy(); @@ -104,6 +108,7 @@ class ExecEnv { // only used for test ExecEnv(); + explicit ExecEnv(std::unique_ptr schema_scanner_factory); // Empty destructor because the compiler-generated one requires full // declarations for classes in scoped_ptrs. @@ -149,6 +154,7 @@ class ExecEnv { pipeline::QueryContextManager* query_context_mgr() { return _query_context_mgr; } ComputeEnv* compute_env() const { return _compute_env; } + const SchemaScannerFactory* schema_scanner_factory() const { return _schema_scanner_factory.get(); } int64_t max_executor_threads() const { return _runtime_env->max_executor_threads(); } @@ -170,6 +176,7 @@ class ExecEnv { TableMetricsManager* _table_metrics_mgr = nullptr; pipeline::QueryContextManager* _query_context_mgr = nullptr; ComputeEnv* _compute_env = nullptr; + std::unique_ptr _schema_scanner_factory; TransactionMgr* _transaction_mgr = nullptr; BatchWriteMgr* _batch_write_mgr = nullptr; diff --git a/be/src/exec/pipeline/scan/schema_chunk_source.cpp b/be/src/exec/pipeline/scan/schema_chunk_source.cpp index 1ceeb764c7078f..96fb71837fc086 100644 --- a/be/src/exec/pipeline/scan/schema_chunk_source.cpp +++ b/be/src/exec/pipeline/scan/schema_chunk_source.cpp @@ -18,7 +18,9 @@ #include #include "compute_env/workgroup/work_group.h" +#include "exec/exec_env.h" #include "exec/schema_scanner.h" +#include "exec/schema_scanner_factory.h" #include "exprs/chunk_predicate_evaluator.h" #include "runtime/descriptors_ext.h" #include "runtime/runtime_state.h" @@ -48,10 +50,10 @@ Status SchemaChunkSource::prepare(RuntimeState* state) { _filter_timer = ADD_TIMER(_runtime_profile, "FilterTime"); - _schema_scanner = SchemaScanner::create(schema_table->schema_table_type()); - if (_schema_scanner == nullptr) { - return Status::InternalError("schema scanner get null pointer"); - } + ASSIGN_OR_RETURN( + _schema_scanner, + create_schema_scanner(state->exec_env() == nullptr ? nullptr : state->exec_env()->schema_scanner_factory(), + schema_table->schema_table_type())); RETURN_IF_ERROR(_schema_scanner->init(param, _ctx->object_pool())); diff --git a/be/src/exec/schema_scan_node.cpp b/be/src/exec/schema_scan_node.cpp index 470c619d75992e..99b9c79dc8c2cd 100644 --- a/be/src/exec/schema_scan_node.cpp +++ b/be/src/exec/schema_scan_node.cpp @@ -18,9 +18,11 @@ #include "column/column_helper.h" #include "common/runtime_profile.h" +#include "exec/exec_env.h" #include "exec/pipeline/scan/schema_scan_context.h" #include "exec/pipeline/scan/schema_scan_operator.h" #include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner_factory.h" #include "exprs/chunk_predicate_evaluator.h" #include "runtime/descriptors_ext.h" #include "runtime/runtime_state.h" @@ -116,12 +118,10 @@ Status SchemaScanNode::prepare(RuntimeState* state) { _scanner_param._fill_chunk_timer = ADD_TIMER(_runtime_profile, "FillChunk"); _filter_timer = ADD_TIMER(_runtime_profile, "FilterTime"); - // new one scanner - _schema_scanner = SchemaScanner::create(schema_table->schema_table_type()); - - if (nullptr == _schema_scanner) { - return Status::InternalError("schema scanner get nullptr pointer."); - } + ASSIGN_OR_RETURN( + _schema_scanner, + create_schema_scanner(state->exec_env() == nullptr ? nullptr : state->exec_env()->schema_scanner_factory(), + schema_table->schema_table_type())); RETURN_IF_ERROR(_schema_scanner->init(&_scanner_param, _pool)); diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index 1031c35bc80629..9bca28287aa609 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -106,9 +106,6 @@ class SchemaScanner { virtual Status start(RuntimeState* state); // Must only return one row at most each time virtual Status get_next(ChunkPtr* chunk, bool* eos); - // factory function - static std::unique_ptr create(TSchemaTableType::type type); - TAuthInfo build_auth_info(); static void set_starrocks_server(StarRocksServer* starrocks_server) { _s_starrocks_server = starrocks_server; } diff --git a/be/src/exec/schema_scanner_factory.cpp b/be/src/exec/schema_scanner_factory.cpp index 4b7ba822876337..0b2a9597d95578 100644 --- a/be/src/exec/schema_scanner_factory.cpp +++ b/be/src/exec/schema_scanner_factory.cpp @@ -12,184 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - -#include "exec/schema_scanner.h" -#include "exec/schema_scanner/schema_analyze_status.h" -#include "exec/schema_scanner/schema_applicable_roles_scanner.h" -#include "exec/schema_scanner/schema_be_bvars_scanner.h" -#include "exec/schema_scanner/schema_be_cloud_native_compactions_scanner.h" -#include "exec/schema_scanner/schema_be_compactions_scanner.h" -#include "exec/schema_scanner/schema_be_configs_scanner.h" -#include "exec/schema_scanner/schema_be_datacache_metrics_scanner.h" -#include "exec/schema_scanner/schema_be_logs_scanner.h" -#include "exec/schema_scanner/schema_be_metrics_scanner.h" -#include "exec/schema_scanner/schema_be_tablet_write_log_scanner.h" -#include "exec/schema_scanner/schema_be_tablets_scanner.h" -#include "exec/schema_scanner/schema_be_threads_scanner.h" -#include "exec/schema_scanner/schema_be_txns_scanner.h" -#include "exec/schema_scanner/schema_charsets_scanner.h" -#include "exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h" -#include "exec/schema_scanner/schema_cluster_snapshots_scanner.h" -#include "exec/schema_scanner/schema_collation_character_set_applicability_scanner.h" -#include "exec/schema_scanner/schema_collations_scanner.h" -#include "exec/schema_scanner/schema_column_stats_usage_scanner.h" -#include "exec/schema_scanner/schema_columns_scanner.h" -#include "exec/schema_scanner/schema_dummy_scanner.h" -#include "exec/schema_scanner/schema_fe_metrics_scanner.h" -#include "exec/schema_scanner/schema_fe_tablet_schedules_scanner.h" -#include "exec/schema_scanner/schema_fe_threads_scanner.h" -#include "exec/schema_scanner/schema_keywords_scanner.h" -#include "exec/schema_scanner/schema_load_tracking_logs_scanner.h" -#include "exec/schema_scanner/schema_loads_scanner.h" -#include "exec/schema_scanner/schema_materialized_view_refresh_jobs_scanner.h" -#include "exec/schema_scanner/schema_materialized_views_scanner.h" -#include "exec/schema_scanner/schema_partitions_meta_scanner.h" -#include "exec/schema_scanner/schema_pipe_files.h" -#include "exec/schema_scanner/schema_pipes.h" -#include "exec/schema_scanner/schema_recyclebin_catalogs.h" -#include "exec/schema_scanner/schema_routine_load_jobs_scanner.h" -#include "exec/schema_scanner/schema_schema_privileges_scanner.h" -#include "exec/schema_scanner/schema_schemata_scanner.h" -#include "exec/schema_scanner/schema_stream_loads_scanner.h" -#include "exec/schema_scanner/schema_table_privileges_scanner.h" -#include "exec/schema_scanner/schema_tables_config_scanner.h" -#include "exec/schema_scanner/schema_tables_scanner.h" -#include "exec/schema_scanner/schema_tablet_reshard_jobs_scanner.h" -#include "exec/schema_scanner/schema_task_runs_scanner.h" -#include "exec/schema_scanner/schema_tasks_scanner.h" -#include "exec/schema_scanner/schema_temp_tables_scanner.h" -#include "exec/schema_scanner/schema_user_privileges_scanner.h" -#include "exec/schema_scanner/schema_variables_scanner.h" -#include "exec/schema_scanner/schema_views_scanner.h" -#include "exec/schema_scanner/schema_warehouse_metrics.h" -#include "exec/schema_scanner/schema_warehouse_queries.h" -#include "exec/schema_scanner/starrocks_grants_to_scanner.h" -#include "exec/schema_scanner/starrocks_role_edges_scanner.h" -#include "exec/schema_scanner/sys_fe_locks.h" -#include "exec/schema_scanner/sys_fe_memory_usage.h" -#include "exec/schema_scanner/sys_object_dependencies.h" +#include "exec/schema_scanner_factory.h" namespace starrocks { -std::unique_ptr SchemaScanner::create(TSchemaTableType::type type) { - switch (type) { - case TSchemaTableType::SCH_TABLES: - return std::make_unique(); - case TSchemaTableType::SCH_SCHEMATA: - return std::make_unique(); - case TSchemaTableType::SCH_COLUMNS: - return std::make_unique(); - case TSchemaTableType::SCH_CHARSETS: - return std::make_unique(); - case TSchemaTableType::SCH_COLLATIONS: - return std::make_unique(); - case TSchemaTableType::SCH_COLLATION_CHARACTER_SET_APPLICABILITY: - return std::make_unique(); - case TSchemaTableType::SCH_GLOBAL_VARIABLES: - return std::make_unique(TVarType::GLOBAL); - case TSchemaTableType::SCH_SESSION_VARIABLES: - case TSchemaTableType::SCH_VARIABLES: - return std::make_unique(TVarType::SESSION); - case TSchemaTableType::SCH_USER_PRIVILEGES: - return std::make_unique(); - case TSchemaTableType::SCH_SCHEMA_PRIVILEGES: - return std::make_unique(); - case TSchemaTableType::SCH_TABLE_PRIVILEGES: - return std::make_unique(); - case TSchemaTableType::SCH_VIEWS: - return std::make_unique(); - case TSchemaTableType::SCH_TASKS: - return std::make_unique(); - case TSchemaTableType::SCH_TASK_RUNS: - return std::make_unique(); - case TSchemaTableType::SCH_MATERIALIZED_VIEWS: - return std::make_unique(); - case TSchemaTableType::SCH_MATERIALIZED_VIEW_REFRESH_JOBS: - return std::make_unique(); - case TSchemaTableType::SCH_LOADS: - return std::make_unique(); - case TSchemaTableType::SCH_LOAD_TRACKING_LOGS: - return std::make_unique(); - case TSchemaTableType::SCH_TABLES_CONFIG: - return std::make_unique(); - case TSchemaTableType::SCH_VERBOSE_SESSION_VARIABLES: - return std::make_unique(TVarType::VERBOSE); - case TSchemaTableType::SCH_BE_TABLETS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_METRICS: - return std::make_unique(); - case TSchemaTableType::SCH_FE_METRICS: - return std::make_unique(); - case TSchemaTableType::SCH_FE_THREADS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_TXNS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_CONFIGS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_THREADS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_LOGS: - return std::make_unique(); - case TSchemaTableType::SCH_FE_TABLET_SCHEDULES: - return std::make_unique(); - case TSchemaTableType::SCH_BE_COMPACTIONS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_BVARS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_CLOUD_NATIVE_COMPACTIONS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_TABLET_WRITE_LOG: - return std::make_unique(); - case TSchemaTableType::STARROCKS_ROLE_EDGES: - return std::make_unique(); - case TSchemaTableType::STARROCKS_GRANT_TO_ROLES: - return std::make_unique(TGrantsToType::ROLE); - case TSchemaTableType::STARROCKS_GRANT_TO_USERS: - return std::make_unique(TGrantsToType::USER); - case TSchemaTableType::STARROCKS_OBJECT_DEPENDENCIES: - return std::make_unique(); - case TSchemaTableType::SCH_ROUTINE_LOAD_JOBS: - return std::make_unique(); - case TSchemaTableType::SCH_STREAM_LOADS: - return std::make_unique(); - case TSchemaTableType::SCH_PIPE_FILES: - return std::make_unique(); - case TSchemaTableType::SCH_PIPES: - return std::make_unique(); - case TSchemaTableType::SYS_FE_LOCKS: - return std::make_unique(); - case TSchemaTableType::SCH_BE_DATACACHE_METRICS: - return std::make_unique(); - case TSchemaTableType::SCH_PARTITIONS_META: - return std::make_unique(); - case TSchemaTableType::SYS_FE_MEMORY_USAGE: - return std::make_unique(); - case TSchemaTableType::SCH_TEMP_TABLES: - return std::make_unique(); - case TSchemaTableType::SCH_RECYCLEBIN_CATALOGS: - return std::make_unique(); - case TSchemaTableType::SCH_COLUMN_STATS_USAGE: - return std::make_unique(); - case TSchemaTableType::SCH_ANALYZE_STATUS: - return std::make_unique(); - case TSchemaTableType::SCH_CLUSTER_SNAPSHOTS: - return std::make_unique(); - case TSchemaTableType::SCH_CLUSTER_SNAPSHOT_JOBS: - return std::make_unique(); - case TSchemaTableType::SCH_APPLICABLE_ROLES: - return std::make_unique(); - case TSchemaTableType::SCH_KEYWORDS: - return std::make_unique(); - case TSchemaTableType::SCH_WAREHOUSE_METRICS: - return std::make_unique(); - case TSchemaTableType::SCH_WAREHOUSE_QUERIES: - return std::make_unique(); - case TSchemaTableType::SCH_TABLET_RESHARD_JOBS: - return std::make_unique(); - default: - return std::make_unique(); +StatusOr> create_schema_scanner(const SchemaScannerFactory* factory, + TSchemaTableType::type type) { + if (factory == nullptr) { + return Status::InternalError("schema scanner factory is not installed"); + } + auto scanner = factory->create(type); + if (scanner == nullptr) { + return Status::InternalError("schema scanner factory returned nullptr"); } + return scanner; } } // namespace starrocks diff --git a/be/src/exec/schema_scanner_factory.h b/be/src/exec/schema_scanner_factory.h new file mode 100644 index 00000000000000..390d6af5fd42eb --- /dev/null +++ b/be/src/exec/schema_scanner_factory.h @@ -0,0 +1,34 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "common/statusor.h" +#include "exec/schema_scanner.h" + +namespace starrocks { + +class SchemaScannerFactory { +public: + virtual ~SchemaScannerFactory() = default; + + virtual std::unique_ptr create(TSchemaTableType::type type) const = 0; +}; + +StatusOr> create_schema_scanner(const SchemaScannerFactory* factory, + TSchemaTableType::type type); + +} // namespace starrocks diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index 9aa70754baa48b..ffde9946f33a76 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -37,7 +37,8 @@ ADD_BE_LIB(Service target_link_libraries(Service PRIVATE Script Runtime Orchestration Cache AgentServer ComputeEnv ExecRuntime Platform ConnectorFile ColumnCore) target_link_libraries(ServiceBE PUBLIC Service) -target_link_libraries(ServiceBE PRIVATE AgentServer ModuleBootstrap DataWorkflows Orchestration Storage) +target_link_libraries(ServiceBE PRIVATE AgentServer ModuleBootstrap DataWorkflows Orchestration SchemaScannerBuiltin + Storage) # only build starrocks_be when TEST is off if ((NOT ${MAKE_TEST} STREQUAL "ON") AND (NOT BUILD_FORMAT_LIB)) diff --git a/be/src/service/service_be/starrocks_be.cpp b/be/src/service/service_be/starrocks_be.cpp index 1ceb1b7f4fec8c..a0fbc9d426277c 100644 --- a/be/src/service/service_be/starrocks_be.cpp +++ b/be/src/service/service_be/starrocks_be.cpp @@ -41,6 +41,7 @@ #include "compute_env/compute_env.h" #include "compute_env/staros/staros_worker_runtime.h" #include "data_workflows/data_workflows_env.h" +#include "exec/builtin_schema_scanner_factory.h" #include "exec/exec_env.h" #include "exec/pipeline/driver_executor_factory.h" #include "exec/pipeline/driver_queue_factory.h" @@ -227,7 +228,7 @@ void start_be(const std::vector& paths, bool as_cn) { exec_env->set_compute_env(compute_env.get()); EXIT_IF_ERROR(runtime_env->init_lake_thread_pools(process_metrics)); - EXIT_IF_ERROR(exec_env->init(process_metrics_registry, runtime_env)); + EXIT_IF_ERROR(exec_env->init(process_metrics_registry, runtime_env, create_builtin_schema_scanner_factory())); LOG(INFO) << process_name << " start step " << start_step++ << ": exec env init successfully"; EXIT_IF_ERROR(init_storage_env(runtime_env, platform_env, compute_env.get())); diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index df49376946ea99..afa363d266f8c8 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -239,7 +239,7 @@ add_executable(starrocks_test test_main.cpp $ ${TEST_EXTRA_SOURCES}) -TARGET_LINK_LIBRARIES(starrocks_dw_test ${TEST_LINK_LIBS}) +TARGET_LINK_LIBRARIES(starrocks_dw_test SchemaScannerBuiltin ${TEST_LINK_LIBS}) STARROCKS_FORCE_LOAD_LIBS(starrocks_dw_test ${EXPR_FORCE_LOAD_LIBS}) diff --git a/be/test/data_workflows/CMakeLists.txt b/be/test/data_workflows/CMakeLists.txt index 1973aec49c4613..7904538a66ebd7 100644 --- a/be/test/data_workflows/CMakeLists.txt +++ b/be/test/data_workflows/CMakeLists.txt @@ -34,5 +34,5 @@ add_library(starrocks_data_workflows_test_objs OBJECT ${DATA_WORKFLOWS_TEST_FILE target_link_libraries(starrocks_data_workflows_test_objs PRIVATE StarRocksPCH) set_target_properties(starrocks_data_workflows_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") add_executable(data_workflows_test ${CMAKE_SOURCE_DIR}/test/test_main.cpp $) -target_link_libraries(data_workflows_test ${TEST_LINK_LIBS}) +target_link_libraries(data_workflows_test SchemaScannerBuiltin ${TEST_LINK_LIBS}) STARROCKS_FORCE_LOAD_LIBS(data_workflows_test ${EXPR_FORCE_LOAD_LIBS}) diff --git a/be/test/exec/CMakeLists.txt b/be/test/exec/CMakeLists.txt index 60720c7b07be27..8e82c6e196233b 100644 --- a/be/test/exec/CMakeLists.txt +++ b/be/test/exec/CMakeLists.txt @@ -85,6 +85,7 @@ set(SCHEMA_SCANNER_CORE_TEST_LINK_LIBS gtest ${WL_END_GROUP} ) +list(REMOVE_ITEM SCHEMA_SCANNER_CORE_TEST_LINK_LIBS arrow_testing) add_library(starrocks_schema_scanner_core_test_objs OBJECT ${SCHEMA_SCANNER_CORE_TEST_FILES}) target_link_libraries(starrocks_schema_scanner_core_test_objs PRIVATE StarRocksPCH) diff --git a/be/test/exec/schema_scanner/schema_scanner_core_test.cpp b/be/test/exec/schema_scanner/schema_scanner_core_test.cpp index 7c342eb6ebd2b3..0b68b5ff2da8db 100644 --- a/be/test/exec/schema_scanner/schema_scanner_core_test.cpp +++ b/be/test/exec/schema_scanner/schema_scanner_core_test.cpp @@ -19,6 +19,7 @@ #include "base/string/slice.h" #include "column/chunk.h" #include "exec/schema_scanner.h" +#include "exec/schema_scanner_factory.h" #include "gen_cpp/Types_types.h" #include "runtime/runtime_state.h" #include "types/logical_type.h" @@ -32,6 +33,21 @@ class TestSchemaScanner : public SchemaScanner { const SchemaScannerState& scanner_state() const { return _ss_state; } }; +class RecordingSchemaScannerFactory final : public SchemaScannerFactory { +public: + std::unique_ptr create(TSchemaTableType::type type) const override { + last_type = type; + return std::make_unique(nullptr, 0); + } + + mutable TSchemaTableType::type last_type = static_cast(-1); +}; + +class NullSchemaScannerFactory final : public SchemaScannerFactory { +public: + std::unique_ptr create(TSchemaTableType::type type) const override { return nullptr; } +}; + TEST(SchemaScannerCoreTest, InitRejectsInvalidParameters) { SchemaScanner scanner(nullptr, 0); SchemaScannerParam param; @@ -120,4 +136,32 @@ TEST(SchemaScannerCoreTest, InitSchemaScannerStateRejectsMissingEndpoint) { ASSERT_FALSE(scanner.init_schema_scanner_state(&state).ok()); } +TEST(SchemaScannerCoreTest, FactoryHelperDelegatesRequestedType) { + RecordingSchemaScannerFactory factory; + + auto scanner = create_schema_scanner(&factory, TSchemaTableType::SCH_TABLES); + + ASSERT_TRUE(scanner.ok()) << scanner.status().to_string(); + EXPECT_NE(nullptr, scanner.value().get()); + EXPECT_EQ(TSchemaTableType::SCH_TABLES, factory.last_type); +} + +TEST(SchemaScannerCoreTest, FactoryHelperRejectsMissingFactory) { + auto scanner = create_schema_scanner(nullptr, TSchemaTableType::SCH_TABLES); + + ASSERT_FALSE(scanner.ok()); + EXPECT_TRUE(scanner.status().is_internal_error()); + EXPECT_EQ("schema scanner factory is not installed", scanner.status().message()); +} + +TEST(SchemaScannerCoreTest, FactoryHelperRejectsNullScanner) { + NullSchemaScannerFactory factory; + + auto scanner = create_schema_scanner(&factory, TSchemaTableType::SCH_TABLES); + + ASSERT_FALSE(scanner.ok()); + EXPECT_TRUE(scanner.status().is_internal_error()); + EXPECT_EQ("schema scanner factory returned nullptr", scanner.status().message()); +} + } // namespace starrocks diff --git a/be/test/exec/schema_scanner/schema_scanner_test.cpp b/be/test/exec/schema_scanner/schema_scanner_test.cpp index bdc4773790f016..875f3aabfe4084 100644 --- a/be/test/exec/schema_scanner/schema_scanner_test.cpp +++ b/be/test/exec/schema_scanner/schema_scanner_test.cpp @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "exec/schema_scanner.h" - #include +#include "exec/builtin_schema_scanner_factory.h" #include "exec/schema_scanner/schema_tablet_reshard_jobs_scanner.h" #include "gen_cpp/Descriptors_types.h" @@ -24,20 +23,21 @@ namespace starrocks { class SchemaScannerTest : public ::testing::Test {}; TEST_F(SchemaScannerTest, test_create) { + auto factory = create_builtin_schema_scanner_factory(); { - auto scanner = SchemaScanner::create(TSchemaTableType::SCH_TABLET_RESHARD_JOBS); + auto scanner = factory->create(TSchemaTableType::SCH_TABLET_RESHARD_JOBS); ASSERT_NE(scanner, nullptr); auto* reshard_jobs_scanner = dynamic_cast(scanner.get()); ASSERT_NE(reshard_jobs_scanner, nullptr); } { // Test an existing one to ensure it still works - auto scanner = SchemaScanner::create(TSchemaTableType::SCH_TABLES); + auto scanner = factory->create(TSchemaTableType::SCH_TABLES); ASSERT_NE(scanner, nullptr); } { // Test default case - auto scanner = SchemaScanner::create(static_cast(-1)); + auto scanner = factory->create(static_cast(-1)); ASSERT_NE(scanner, nullptr); } } diff --git a/be/test/runtime/exec_env_test.cpp b/be/test/runtime/exec_env_test.cpp index f068e227e9423b..7ad4c96b19d104 100644 --- a/be/test/runtime/exec_env_test.cpp +++ b/be/test/runtime/exec_env_test.cpp @@ -26,6 +26,7 @@ #include "compute_env/profile_report_worker.h" #include "exec/pipeline/driver_executor_factory.h" #include "exec/pipeline/driver_queue_factory.h" +#include "exec/schema_scanner_factory.h" #include "platform/platform_env.h" #include "runtime/runtime_env.h" #include "runtime/runtime_filter_query_lifecycle.h" @@ -45,8 +46,33 @@ class FakeRuntimeFilterServices : public RuntimeFilterSender, public RuntimeFilt int, int64_t) override {} }; +class FakeSchemaScannerFactory final : public SchemaScannerFactory { +public: + explicit FakeSchemaScannerFactory(bool* destroyed) : _destroyed(destroyed) {} + ~FakeSchemaScannerFactory() override { *_destroyed = true; } + + std::unique_ptr create(TSchemaTableType::type type) const override { return nullptr; } + +private: + bool* _destroyed; +}; + } // namespace +TEST(ExecEnvTest, owns_schema_scanner_factory) { + bool destroyed = false; + auto factory = std::make_unique(&destroyed); + auto* expected = factory.get(); + ExecEnv env(std::move(factory)); + + EXPECT_EQ(expected, env.schema_scanner_factory()); + EXPECT_FALSE(destroyed); + + env.destroy(); + EXPECT_TRUE(destroyed); + EXPECT_EQ(nullptr, env.schema_scanner_factory()); +} + TEST(ExecEnvTest, refresh_service_contexts_keeps_context_views_in_sync) { ExecEnv env; auto* runtime_env = RuntimeEnv::GetInstance(); diff --git a/be/test/test_main.cpp b/be/test/test_main.cpp index 111564f09719a1..bfdc05f4dfbf2e 100644 --- a/be/test/test_main.cpp +++ b/be/test/test_main.cpp @@ -14,6 +14,7 @@ #include +#include "exec/builtin_schema_scanner_factory.h" #include "formats/orc/lzo_decompressor_registration.h" #include "testutil/init_test_env.h" @@ -24,5 +25,5 @@ int main(int argc, char** argv) { return 1; } - return starrocks::init_test_env(argc, argv); + return starrocks::init_test_env(argc, argv, starrocks::create_builtin_schema_scanner_factory()); } diff --git a/be/test/testutil/init_test_env.h b/be/test/testutil/init_test_env.h index bb4d829ae7a570..0994df861665aa 100644 --- a/be/test/testutil/init_test_env.h +++ b/be/test/testutil/init_test_env.h @@ -41,6 +41,7 @@ #include "exec/pipeline/driver_executor_factory.h" #include "exec/pipeline/driver_queue_factory.h" #include "exec/pipeline/query_context.h" +#include "exec/schema_scanner_factory.h" #include "exec_primitive/pipeline/primitives/pipeline_metrics.h" #include "gtest/gtest.h" #include "module/connector_bootstrap.h" @@ -64,7 +65,7 @@ namespace starrocks { extern void shutdown_tracer(); -int init_test_env(int argc, char** argv) { +int init_test_env(int argc, char** argv, std::unique_ptr schema_scanner_factory = nullptr) { ::testing::InitGoogleTest(&argc, argv); if (getenv("STARROCKS_HOME") == nullptr) { fprintf(stderr, "you need set STARROCKS_HOME environment variable.\n"); @@ -182,7 +183,7 @@ int init_test_env(int argc, char** argv) { exec_env->set_compute_env(compute_env.get()); st = runtime_env->init_lake_thread_pools(process_metrics); CHECK(st.ok()) << st; - st = exec_env->init(process_metrics_registry, runtime_env); + st = exec_env->init(process_metrics_registry, runtime_env, std::move(schema_scanner_factory)); CHECK(st.ok()) << st; StorageEnvOptions storage_env_options; From bf6d451801c0bb0d4479b5b7db0f45a7ff85ad87 Mon Sep 17 00:00:00 2001 From: alvin-phoenix-ai Date: Mon, 13 Jul 2026 07:39:52 +0800 Subject: [PATCH 2/2] [Refactor] Split schema scanners into layered modules Signed-off-by: alvin-phoenix-ai --- be/AGENTS.md | 59 ++++-- be/module_boundary_manifest.json | 177 +++++++++++++--- be/src/base/string/parse_util.cpp | 58 +++++ be/src/base/string/parse_util.h | 2 + be/src/exec/AGENTS.md | 4 +- be/src/exec/CMakeLists.txt | 199 ++++++++++++------ ...ma_be_cloud_native_compactions_scanner.cpp | 3 +- .../schema_be_compactions_scanner.cpp | 2 +- .../schema_be_datacache_metrics_scanner.cpp | 1 - .../schema_scanner/schema_be_logs_scanner.cpp | 2 +- .../schema_be_tablet_write_log_scanner.cpp | 2 +- .../schema_be_tablets_scanner.cpp | 1 - .../schema_scanner/schema_be_txns_scanner.cpp | 2 +- .../schema_scanner/schema_column_filler.h | 5 + .../schema_fe_metrics_scanner.cpp | 2 +- .../schema_fe_tablet_schedules_scanner.cpp | 1 - be/src/exec/schema_scanner/schema_helper.cpp | 6 - be/src/exec/schema_scanner/schema_helper.h | 2 - .../schema_partitions_meta_scanner.cpp | 4 +- be/src/storage/utils.cpp | 53 ----- be/src/storage/utils.h | 2 - be/test/CMakeLists.txt | 10 - be/test/base/string/parse_util_test.cpp | 84 ++++++++ be/test/exec/CMakeLists.txt | 179 ++++++++++++++-- .../schema_scanner_cache_test.cpp | 41 ++++ .../schema_scanner_frontend_test_main.cpp | 23 ++ .../schema_scanner_storage_test.cpp | 49 +++++ .../schema_scanner/schema_scanner_test.cpp | 168 +++++++++++++-- be/test/storage/utils_test.cpp | 81 ------- build-support/check_be_module_boundaries.py | 52 ++++- .../test_check_be_module_boundaries.py | 65 ++++++ 31 files changed, 1028 insertions(+), 311 deletions(-) create mode 100644 be/test/exec/schema_scanner/schema_scanner_cache_test.cpp create mode 100644 be/test/exec/schema_scanner/schema_scanner_frontend_test_main.cpp create mode 100644 be/test/exec/schema_scanner/schema_scanner_storage_test.cpp diff --git a/be/AGENTS.md b/be/AGENTS.md index 20edfb32f487a7..f0687f68d5ce6f 100644 --- a/be/AGENTS.md +++ b/be/AGENTS.md @@ -292,28 +292,53 @@ Default BE module bootstrap composition for built-in module registration, includ - Core tests: `module_bootstrap_test` - Remediation: Keep ModuleBootstrap as top-level default module composition; module implementations should expose registration hooks here instead of depending on service startup directly. -### ExecSchemaScannerCore (`execschemascannercore`) -Schema scanner base contract and shared mechanics without concrete scanner, pipeline, storage, service, or ExecEnv coupling. -- Targets: `ExecSchemaScannerCore` -- Allowed internal include prefixes: `exec/schema_scanner.h`, `exec/schema_scanner_factory.h`, `exprs/`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` +### SchemaScannerCore (`schemascannercore`) +Schema scanner contract, factory interface, shared mechanics, and column filling without concrete scanner, storage, cache, service, pipeline, or ExecEnv coupling. +- Targets: `SchemaScannerCore` +- Allowed internal include prefixes: `exec/schema_scanner.h`, `exec/schema_scanner_factory.h`, `exec/schema_scanner/schema_column_filler.h`, `exprs/`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` - Allowed target deps: `Expr`, `Runtime`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen` - Core tests: `schema_scanner_core_test` -- Remediation: Keep SchemaScannerCore limited to the base SchemaScanner contract; move concrete scanner creation and service-specific logic into higher schema scanner modules. +- Remediation: Keep SchemaScannerCore limited to the scanner contract and dependency-neutral shared mechanics; move concrete scanners and composition into higher schema scanner layers. + +### SchemaScannerLocal (`schemascannerlocal`) +Static and BE-local process schema scanners without SchemaHelper, platform clients, storage, cache, service, pipeline, or ExecEnv coupling. +- Targets: `SchemaScannerLocal` +- Allowed internal include prefixes: `exec/schema_scanner.h`, `exec/schema_scanner/schema_column_filler.h`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` +- Allowed target deps: `SchemaScannerCore`, `Runtime`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen` +- Core tests: `schema_scanner_local_test` +- Remediation: Keep SchemaScannerLocal restricted to static and BE-local process data; move FE client, storage, cache, and composition behavior into their dedicated scanner layers. + +### SchemaScannerFrontend (`schemascannerfrontend`) +Frontend-backed schema scanners and SchemaHelper above Platform and the local/core scanner layers, without storage, cache, service, pipeline, or ExecEnv coupling. +- Targets: `SchemaScannerFrontend` +- Allowed internal include prefixes: `exec/schema_scanner.h`, `exec/schema_scanner/schema_column_filler.h`, `platform/`, `exprs/`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` +- Allowed target deps: `SchemaScannerCore`, `SchemaScannerLocal`, `Platform`, `Expr`, `Runtime`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen` +- Core tests: `schema_scanner_frontend_test` +- Remediation: Keep SchemaScannerFrontend limited to FE RPC/HTTP-backed schema data over Platform and the local/core scanner layers; move storage and cache access into their dedicated layers. + +### SchemaScannerStorage (`schemascannerstorage`) +Storage-backed schema scanners above concrete Storage and the frontend/local/core scanner layers, without cache, service, pipeline, or ExecEnv coupling. +- Targets: `SchemaScannerStorage` +- Allowed internal include prefixes: `exec/schema_scanner.h`, `exec/schema_scanner/schema_column_filler.h`, `storage/`, `storage_primitive/`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` +- Allowed target deps: `SchemaScannerCore`, `SchemaScannerFrontend`, `SchemaScannerLocal`, `Storage`, `StoragePrimitive`, `Runtime`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen` +- Core tests: `schema_scanner_storage_test` +- Remediation: Keep SchemaScannerStorage limited to tablet, transaction, compaction, and write-log schema data; route FE access through SchemaScannerFrontend and keep cache/composition behavior elsewhere. + +### SchemaScannerCache (`schemascannercache`) +Data-cache schema scanner above Cache and SchemaScannerCore without storage, platform, service, pipeline, or ExecEnv coupling. +- Targets: `SchemaScannerCache` +- Allowed internal include prefixes: `exec/schema_scanner.h`, `cache/`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` +- Allowed target deps: `SchemaScannerCore`, `Cache`, `Runtime`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen` +- Core tests: `schema_scanner_cache_test` +- Remediation: Keep SchemaScannerCache limited to data-cache schema data over Cache and SchemaScannerCore; keep storage, FE client, and composition behavior in their dedicated layers. ### SchemaScannerBuiltin (`schemascannerbuiltin`) -Temporary top-level composition target for the builtin schema scanner factory while concrete scanners remain in Exec compatibility targets. +Top-level builtin schema scanner factory composition over the core, local, frontend, storage, and cache scanner layers without depending on Exec. - Targets: `SchemaScannerBuiltin` - Allowed internal include prefixes: `exec/builtin_schema_scanner_factory.h`, `exec/schema_scanner.h`, `exec/schema_scanner_factory.h`, `exec/schema_scanner/`, `gen_cpp/` -- Allowed target deps: `Exec`, `ExecSchemaScannerCore`, `ExecSchemaScanners` -- Remediation: Keep builtin scanner selection above Exec; PR2 must replace the temporary Exec dependency with explicit layered schema scanner targets. - -### ExecSchemaScanners (`execschemascanners`) -Clean concrete schema scanners that do not depend on SchemaHelper, FE RPC/client helpers, storage, service, cache, pipeline, or ExecEnv. -- Targets: `ExecSchemaScanners` -- Allowed internal include prefixes: `exec/schema_scanner.h`, `exec/schema_scanner/schema_column_filler.h`, `runtime/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/` -- Allowed target deps: `ExecSchemaScannerCore`, `Runtime`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen` -- Core tests: `exec_schema_scanners_test` -- Remediation: Keep this first schema scanner target limited to clean local/static scanners; leave SchemaHelper, storage, HTTP, cache, service, and ExecEnv users in higher compatibility modules until they get explicit boundaries. +- Allowed target deps: `SchemaScannerCore`, `SchemaScannerLocal`, `SchemaScannerFrontend`, `SchemaScannerStorage`, `SchemaScannerCache` +- Core tests: `schema_scanner_builtin_test` +- Remediation: Keep builtin scanner selection as composition over the explicit scanner strata; do not add an Exec dependency or bypass the owning scanner layer. ### ExecJoinCore (`execjoincore`) Reusable exec join hash table algorithms without join nodes, pipeline, storage, service, or util coupling. @@ -335,7 +360,7 @@ Orchestration layer below Service for query, fragment, and ingestion lifecycle e Diagnostic script execution and command dispatch layer below Service, HttpService, Tools, and AgentServer, and above the remaining reusable BE modules. - Targets: `Script` - Allowed internal include prefixes: `script/`, `base/`, `cache/`, `column/`, `common/`, `connector/`, `compute_env/`, `data_workflows/`, `exec/`, `exec_primitive/`, `exprs/`, `formats/`, `fs/`, `gen_cpp/`, `geo/`, `gutil/`, `io/`, `orchestration/`, `platform/`, `runtime/`, `storage/`, `storage_primitive/`, `types/` -- Allowed target deps: `Base`, `Gutil`, `Common`, `Cache`, `IO`, `FileSystem`, `Platform`, `Types`, `ColumnCore`, `ChunkCore`, `ColumnSortCore`, `Runtime`, `Runtime`, `Runtime`, `Formats`, `StoragePrimitive`, `StorageBase`, `Storage`, `ComputeEnv`, `DataWorkflows`, `Expr`, `ExprDict`, `ExprTableFunction`, `ExprUtility`, `ExecPrimitive`, `ExecRuntime`, `ExecSchemaScannerCore`, `ExecSchemaScanners`, `ExecJoinCore`, `Exec`, `ConnectorPrimitive`, `Connector`, `ConnectorBenchmark`, `ConnectorCacheStats`, `ConnectorElasticsearch`, `ConnectorMySQL`, `Orchestration`, `Geo`, `StarRocksGen` +- Allowed target deps: `Base`, `Gutil`, `Common`, `Cache`, `IO`, `FileSystem`, `Platform`, `Types`, `ColumnCore`, `ChunkCore`, `ColumnSortCore`, `Runtime`, `Runtime`, `Runtime`, `Formats`, `StoragePrimitive`, `StorageBase`, `Storage`, `ComputeEnv`, `DataWorkflows`, `Expr`, `ExprDict`, `ExprTableFunction`, `ExprUtility`, `ExecPrimitive`, `ExecRuntime`, `ExecJoinCore`, `Exec`, `ConnectorPrimitive`, `Connector`, `ConnectorBenchmark`, `ConnectorCacheStats`, `ConnectorElasticsearch`, `ConnectorMySQL`, `Orchestration`, `Geo`, `StarRocksGen` - Core tests: `script_test` - Remediation: Keep Script below Service, HttpService, Tools, and AgentServer; lower reusable behavior can live in lower BE modules that Script is allowed to depend on. diff --git a/be/module_boundary_manifest.json b/be/module_boundary_manifest.json index cc3a1af3c5a840..561063d821a6c3 100644 --- a/be/module_boundary_manifest.json +++ b/be/module_boundary_manifest.json @@ -1,4 +1,13 @@ { + "target_dependency_guards": [ + { + "id": "execschemascannerdependencyguard", + "target": "Exec", + "forbidden_dependency_prefixes": ["SchemaScanner"], + "allowed_dependencies": ["SchemaScannerCore"], + "remediation": "Keep Exec dependent only on the SchemaScannerCore contract; compose concrete schema scanner layers and the builtin factory above Exec." + } + ], "modules": [ { "id": "base", @@ -1154,13 +1163,25 @@ "remediation": "Keep ModuleBootstrap as top-level default module composition; module implementations should expose registration hooks here instead of depending on service startup directly." }, { - "id": "execschemascannercore", - "doc_label": "ExecSchemaScannerCore", - "summary": "Schema scanner base contract and shared mechanics without concrete scanner, pipeline, storage, service, or ExecEnv coupling.", - "owned_targets": ["ExecSchemaScannerCore"], - "additional_owned_globs": ["be/src/exec/schema_scanner_factory.h"], - "allowed_include_prefixes": ["exec/schema_scanner.h", "exec/schema_scanner_factory.h", "exprs/", "runtime/", "column/", "types/", "common/", "base/", "gutil/", "gen_cpp/"], - "forbidden_include_prefixes": ["exec/schema_scanner/", "exec/pipeline/", "storage/", "http/", "service/", "cache/"], + "id": "schemascannercore", + "doc_label": "SchemaScannerCore", + "summary": "Schema scanner contract, factory interface, shared mechanics, and column filling without concrete scanner, storage, cache, service, pipeline, or ExecEnv coupling.", + "owned_targets": ["SchemaScannerCore"], + "additional_owned_globs": ["be/src/exec/schema_scanner/schema_column_filler.h"], + "allowed_include_prefixes": [ + "exec/schema_scanner.h", + "exec/schema_scanner_factory.h", + "exec/schema_scanner/schema_column_filler.h", + "exprs/", + "runtime/", + "column/", + "types/", + "common/", + "base/", + "gutil/", + "gen_cpp/" + ], + "forbidden_include_prefixes": ["exec/schema_scanner/", "exec/pipeline/", "storage/", "http/", "service/", "cache/", "platform/"], "forbidden_includes": [ "exec/exec_env.h", "exec_primitive/exec_node.h", @@ -1168,38 +1189,134 @@ ], "allowed_target_deps": ["Expr", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], "allowed_test_targets": ["schema_scanner_core_test"], - "allowed_test_link_deps": ["ExecSchemaScannerCore", "Expr", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], - "remediation": "Keep SchemaScannerCore limited to the base SchemaScanner contract; move concrete scanner creation and service-specific logic into higher schema scanner modules." + "allowed_test_link_deps": ["SchemaScannerCore", "Expr", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "remediation": "Keep SchemaScannerCore limited to the scanner contract and dependency-neutral shared mechanics; move concrete scanners and composition into higher schema scanner layers." }, { - "id": "schemascannerbuiltin", - "doc_label": "SchemaScannerBuiltin", - "summary": "Temporary top-level composition target for the builtin schema scanner factory while concrete scanners remain in Exec compatibility targets.", - "owned_targets": ["SchemaScannerBuiltin"], - "allowed_include_prefixes": ["exec/builtin_schema_scanner_factory.h", "exec/schema_scanner.h", "exec/schema_scanner_factory.h", "exec/schema_scanner/", "gen_cpp/"], - "allowed_target_deps": ["Exec", "ExecSchemaScannerCore", "ExecSchemaScanners"], - "remediation": "Keep builtin scanner selection above Exec; PR2 must replace the temporary Exec dependency with explicit layered schema scanner targets." + "id": "schemascannerlocal", + "doc_label": "SchemaScannerLocal", + "summary": "Static and BE-local process schema scanners without SchemaHelper, platform clients, storage, cache, service, pipeline, or ExecEnv coupling.", + "owned_targets": ["SchemaScannerLocal"], + "allowed_include_prefixes": [ + "exec/schema_scanner.h", + "exec/schema_scanner/schema_column_filler.h", + "runtime/", + "column/", + "types/", + "common/", + "base/", + "gutil/", + "gen_cpp/" + ], + "forbidden_include_prefixes": ["platform/", "storage/", "cache/", "http/", "service/", "exec/pipeline/"], + "forbidden_includes": [ + "exec/schema_scanner/schema_helper.h", + "exec/exec_env.h", + "exec_primitive/exec_node.h", + "exec_primitive/data_sink.h" + ], + "allowed_target_deps": ["SchemaScannerCore", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "allowed_test_targets": ["schema_scanner_local_test"], + "allowed_test_link_deps": ["SchemaScannerLocal", "SchemaScannerCore", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "remediation": "Keep SchemaScannerLocal restricted to static and BE-local process data; move FE client, storage, cache, and composition behavior into their dedicated scanner layers." }, { - "id": "execschemascanners", - "doc_label": "ExecSchemaScanners", - "summary": "Clean concrete schema scanners that do not depend on SchemaHelper, FE RPC/client helpers, storage, service, cache, pipeline, or ExecEnv.", - "owned_targets": ["ExecSchemaScanners"], - "additional_owned_globs": [ - "be/src/exec/schema_scanner/schema_column_filler.h" + "id": "schemascannerfrontend", + "doc_label": "SchemaScannerFrontend", + "summary": "Frontend-backed schema scanners and SchemaHelper above Platform and the local/core scanner layers, without storage, cache, service, pipeline, or ExecEnv coupling.", + "owned_targets": ["SchemaScannerFrontend"], + "allowed_include_prefixes": [ + "exec/schema_scanner.h", + "exec/schema_scanner/schema_column_filler.h", + "platform/", + "exprs/", + "runtime/", + "column/", + "types/", + "common/", + "base/", + "gutil/", + "gen_cpp/" ], - "allowed_include_prefixes": ["exec/schema_scanner.h", "exec/schema_scanner/schema_column_filler.h", "runtime/", "column/", "types/", "common/", "base/", "gutil/", "gen_cpp/"], - "forbidden_include_prefixes": ["exec/pipeline/", "storage/", "http/", "service/", "cache/"], + "forbidden_include_prefixes": ["storage/", "cache/", "http/", "service/", "exec/pipeline/"], "forbidden_includes": [ - "exec/schema_scanner/schema_helper.h", "exec/exec_env.h", "exec_primitive/exec_node.h", "exec_primitive/data_sink.h" ], - "allowed_target_deps": ["ExecSchemaScannerCore", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], - "allowed_test_targets": ["exec_schema_scanners_test"], - "allowed_test_link_deps": ["ExecSchemaScanners", "ExecSchemaScannerCore", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], - "remediation": "Keep this first schema scanner target limited to clean local/static scanners; leave SchemaHelper, storage, HTTP, cache, service, and ExecEnv users in higher compatibility modules until they get explicit boundaries." + "allowed_target_deps": ["SchemaScannerCore", "SchemaScannerLocal", "Platform", "Expr", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "allowed_test_targets": ["schema_scanner_frontend_test"], + "allowed_test_link_deps": ["SchemaScannerFrontend", "SchemaScannerLocal", "SchemaScannerCore", "Platform", "Expr", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "remediation": "Keep SchemaScannerFrontend limited to FE RPC/HTTP-backed schema data over Platform and the local/core scanner layers; move storage and cache access into their dedicated layers." + }, + { + "id": "schemascannerstorage", + "doc_label": "SchemaScannerStorage", + "summary": "Storage-backed schema scanners above concrete Storage and the frontend/local/core scanner layers, without cache, service, pipeline, or ExecEnv coupling.", + "owned_targets": ["SchemaScannerStorage"], + "allowed_include_prefixes": [ + "exec/schema_scanner.h", + "exec/schema_scanner/schema_column_filler.h", + "storage/", + "storage_primitive/", + "runtime/", + "column/", + "types/", + "common/", + "base/", + "gutil/", + "gen_cpp/" + ], + "forbidden_include_prefixes": ["cache/", "platform/", "http/", "service/", "exec/pipeline/"], + "forbidden_includes": [ + "exec/exec_env.h", + "exec_primitive/exec_node.h", + "exec_primitive/data_sink.h" + ], + "allowed_target_deps": ["SchemaScannerCore", "SchemaScannerFrontend", "SchemaScannerLocal", "Storage", "StoragePrimitive", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "allowed_test_targets": ["schema_scanner_storage_test"], + "allowed_test_link_deps": ["SchemaScannerStorage", "SchemaScannerFrontend", "SchemaScannerLocal", "SchemaScannerCore", "Storage", "StoragePrimitive", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "remediation": "Keep SchemaScannerStorage limited to tablet, transaction, compaction, and write-log schema data; route FE access through SchemaScannerFrontend and keep cache/composition behavior elsewhere." + }, + { + "id": "schemascannercache", + "doc_label": "SchemaScannerCache", + "summary": "Data-cache schema scanner above Cache and SchemaScannerCore without storage, platform, service, pipeline, or ExecEnv coupling.", + "owned_targets": ["SchemaScannerCache"], + "allowed_include_prefixes": [ + "exec/schema_scanner.h", + "cache/", + "runtime/", + "column/", + "types/", + "common/", + "base/", + "gutil/", + "gen_cpp/" + ], + "forbidden_include_prefixes": ["storage/", "platform/", "http/", "service/", "exec/pipeline/"], + "forbidden_includes": [ + "exec/exec_env.h", + "exec_primitive/exec_node.h", + "exec_primitive/data_sink.h" + ], + "allowed_target_deps": ["SchemaScannerCore", "Cache", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "allowed_test_targets": ["schema_scanner_cache_test"], + "allowed_test_link_deps": ["SchemaScannerCache", "SchemaScannerCore", "Cache", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "remediation": "Keep SchemaScannerCache limited to data-cache schema data over Cache and SchemaScannerCore; keep storage, FE client, and composition behavior in their dedicated layers." + }, + { + "id": "schemascannerbuiltin", + "doc_label": "SchemaScannerBuiltin", + "summary": "Top-level builtin schema scanner factory composition over the core, local, frontend, storage, and cache scanner layers without depending on Exec.", + "owned_targets": ["SchemaScannerBuiltin"], + "allowed_include_prefixes": ["exec/builtin_schema_scanner_factory.h", "exec/schema_scanner.h", "exec/schema_scanner_factory.h", "exec/schema_scanner/", "gen_cpp/"], + "forbidden_include_prefixes": ["exec/pipeline/", "storage/", "cache/", "platform/", "http/", "service/"], + "forbidden_includes": ["exec/exec_env.h"], + "allowed_target_deps": ["SchemaScannerCore", "SchemaScannerLocal", "SchemaScannerFrontend", "SchemaScannerStorage", "SchemaScannerCache"], + "allowed_test_targets": ["schema_scanner_builtin_test"], + "allowed_test_link_deps": ["SchemaScannerBuiltin", "SchemaScannerStorage", "SchemaScannerCache", "SchemaScannerFrontend", "SchemaScannerLocal", "SchemaScannerCore", "Storage", "StoragePrimitive", "Cache", "Platform", "Expr", "Runtime", "ChunkCore", "ColumnCore", "Types", "Common", "Base", "Gutil", "StarRocksGen"], + "remediation": "Keep builtin scanner selection as composition over the explicit scanner strata; do not add an Exec dependency or bypass the owning scanner layer." }, { "id": "execjoincore", @@ -1315,8 +1432,6 @@ "ExprDict", "ExprTableFunction", "ExprUtility", "ExecPrimitive", "ExecRuntime", - "ExecSchemaScannerCore", - "ExecSchemaScanners", "ExecJoinCore", "Exec", "ConnectorPrimitive", diff --git a/be/src/base/string/parse_util.cpp b/be/src/base/string/parse_util.cpp index 4d035a28977e5e..4c0c28e960d6bd 100644 --- a/be/src/base/string/parse_util.cpp +++ b/be/src/base/string/parse_util.cpp @@ -36,6 +36,11 @@ #include +#include +#include +#include +#include + #include "base/string/string_parser.hpp" namespace starrocks { @@ -112,4 +117,57 @@ StatusOr ParseUtil::parse_mem_spec(const std::string& mem_spec_str, con return bytes; } +int64_t ParseUtil::parse_data_size(const std::string& value_str) { + if (value_str.empty()) return 0; + + // Trim leading/trailing spaces + size_t start = value_str.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) return 0; + size_t end = value_str.find_last_not_of(" \t\n\r"); + std::string s = value_str.substr(start, end - start + 1); + + // Find where the number ends + size_t idx = 0; + bool dot_found = false; + while (idx < s.size() && (std::isdigit(static_cast(s[idx])) || (!dot_found && s[idx] == '.'))) { + if (s[idx] == '.') dot_found = true; + ++idx; + } + + double num = 0; + try { + num = std::stod(s.substr(0, idx)); + } catch (...) { + return 0; + } + + // Extract and normalize unit + std::string unit = s.substr(idx); + unit.erase(std::remove_if(unit.begin(), unit.end(), [](unsigned char c) { return std::isspace(c); }), unit.end()); + std::transform(unit.begin(), unit.end(), unit.begin(), + [](unsigned char c) { return static_cast(std::toupper(c)); }); + + static const std::unordered_map unit_map = {{"", 1LL}, + {"B", 1LL}, + {"K", 1024LL}, + {"KB", 1024LL}, + {"M", 1024LL * 1024}, + {"MB", 1024LL * 1024}, + {"G", 1024LL * 1024 * 1024}, + {"GB", 1024LL * 1024 * 1024}, + {"T", 1024LL * 1024 * 1024 * 1024}, + {"TB", 1024LL * 1024 * 1024 * 1024}, + {"P", 1024LL * 1024 * 1024 * 1024 * 1024}, + {"PB", 1024LL * 1024 * 1024 * 1024 * 1024}}; + + auto it = unit_map.find(unit); + if (it == unit_map.end()) return 0; + + double result = num * static_cast(it->second); + if (result >= 0x1p63 || result < static_cast(std::numeric_limits::min())) { + return 0; + } + return static_cast(result); +} + } // namespace starrocks diff --git a/be/src/base/string/parse_util.h b/be/src/base/string/parse_util.h index 379749c744a611..1392998b2e4362 100644 --- a/be/src/base/string/parse_util.h +++ b/be/src/base/string/parse_util.h @@ -55,6 +55,8 @@ class ParseUtil { // The caller needs to handle other legitimate negative values. // If parsing mem_spec_str fails, it will return an error. static StatusOr parse_mem_spec(const std::string& mem_spec_str, const int64_t memory_limit); + + static int64_t parse_data_size(const std::string& value_str); }; } // namespace starrocks diff --git a/be/src/exec/AGENTS.md b/be/src/exec/AGENTS.md index 4d43633d339a3d..1b96f39e5ac3a6 100644 --- a/be/src/exec/AGENTS.md +++ b/be/src/exec/AGENTS.md @@ -13,7 +13,9 @@ Local guide for work under `be/src/exec`. ## Current Direction - Keep `Exec` as the temporary compatibility umbrella while extracted modules move out of it. -- Treat existing targets such as `ExecPrimitive`, `ExecSchemaScannerCore`, `ExecSchemaScanners`, `ExecJoinCore`, and the sorting/spill slices now owned by `ComputeEnv` as the model for future extractions. +- Keep `Exec` linked only to the `SchemaScannerCore` contract. Compose `SchemaScannerLocal`, `SchemaScannerFrontend`, `SchemaScannerStorage`, `SchemaScannerCache`, and `SchemaScannerBuiltin` above Exec rather than introducing reverse dependencies. +- Schema scanner files remain under `be/src/exec/` temporarily; move them to their final package only after the layered targets and boundary checks are stable. +- Treat existing targets such as `ExecPrimitive`, `SchemaScannerCore`, `ExecJoinCore`, and the sorting/spill slices now owned by `ComputeEnv` as the model for future extractions. - Keep local or checkout-specific roadmap notes outside the committed guide unless they become shared repository policy. ## Validation diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 72db248d79c35c..7926f018bdd068 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -20,16 +20,16 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/exec") # where to put generated binaries set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/exec") -set(EXEC_SCHEMA_SCANNER_CORE_FILES +set(SCHEMA_SCANNER_CORE_FILES schema_scanner.cpp schema_scanner_factory.cpp ) -ADD_BE_LIB(ExecSchemaScannerCore - ${EXEC_SCHEMA_SCANNER_CORE_FILES} +ADD_BE_LIB(SchemaScannerCore + ${SCHEMA_SCANNER_CORE_FILES} ) -target_link_libraries(ExecSchemaScannerCore PUBLIC +target_link_libraries(SchemaScannerCore PUBLIC Expr Runtime ChunkCore @@ -41,9 +41,11 @@ target_link_libraries(ExecSchemaScannerCore PUBLIC StarRocksGen ) -set(EXEC_SCHEMA_SCANNER_FILES +set(SCHEMA_SCANNER_LOCAL_FILES schema_scanner/schema_be_bvars_scanner.cpp schema_scanner/schema_be_configs_scanner.cpp + schema_scanner/schema_be_logs_scanner.cpp + schema_scanner/schema_be_metrics_scanner.cpp schema_scanner/schema_be_threads_scanner.cpp schema_scanner/schema_charsets_scanner.cpp schema_scanner/schema_collation_character_set_applicability_scanner.cpp @@ -54,12 +56,131 @@ set(EXEC_SCHEMA_SCANNER_FILES schema_scanner/schema_triggers_scanner.cpp ) -ADD_BE_LIB(ExecSchemaScanners - ${EXEC_SCHEMA_SCANNER_FILES} +ADD_BE_LIB(SchemaScannerLocal + ${SCHEMA_SCANNER_LOCAL_FILES} +) + +target_link_libraries(SchemaScannerLocal PUBLIC + SchemaScannerCore +) + +target_link_libraries(SchemaScannerLocal PRIVATE + Runtime + ChunkCore + ColumnCore + Types + Common + Base + Gutil + StarRocksGen +) + +set(SCHEMA_SCANNER_FRONTEND_FILES + schema_scanner/schema_analyze_status.cpp + schema_scanner/schema_applicable_roles_scanner.cpp + schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp + schema_scanner/schema_cluster_snapshots_scanner.cpp + schema_scanner/schema_column_stats_usage_scanner.cpp + schema_scanner/schema_columns_scanner.cpp + schema_scanner/schema_fe_metrics_scanner.cpp + schema_scanner/schema_fe_tablet_schedules_scanner.cpp + schema_scanner/schema_fe_threads_scanner.cpp + schema_scanner/schema_helper.cpp + schema_scanner/schema_keywords_scanner.cpp + schema_scanner/schema_load_tracking_logs_scanner.cpp + schema_scanner/schema_loads_scanner.cpp + schema_scanner/schema_materialized_view_refresh_jobs_scanner.cpp + schema_scanner/schema_materialized_views_scanner.cpp + schema_scanner/schema_partitions_meta_scanner.cpp + schema_scanner/schema_pipe_files.cpp + schema_scanner/schema_pipes.cpp + schema_scanner/schema_recyclebin_catalogs.cpp + schema_scanner/schema_routine_load_jobs_scanner.cpp + schema_scanner/schema_schema_privileges_scanner.cpp + schema_scanner/schema_schemata_scanner.cpp + schema_scanner/schema_stream_loads_scanner.cpp + schema_scanner/schema_table_privileges_scanner.cpp + schema_scanner/schema_tables_config_scanner.cpp + schema_scanner/schema_tables_scanner.cpp + schema_scanner/schema_tablet_reshard_jobs_scanner.cpp + schema_scanner/schema_task_runs_scanner.cpp + schema_scanner/schema_tasks_scanner.cpp + schema_scanner/schema_temp_tables_scanner.cpp + schema_scanner/schema_user_privileges_scanner.cpp + schema_scanner/schema_variables_scanner.cpp + schema_scanner/schema_views_scanner.cpp + schema_scanner/schema_warehouse_metrics.cpp + schema_scanner/schema_warehouse_queries.cpp + schema_scanner/starrocks_grants_to_scanner.cpp + schema_scanner/starrocks_role_edges_scanner.cpp + schema_scanner/sys_fe_locks.cpp + schema_scanner/sys_fe_memory_usage.cpp + schema_scanner/sys_object_dependencies.cpp +) + +ADD_BE_LIB(SchemaScannerFrontend + ${SCHEMA_SCANNER_FRONTEND_FILES} ) -target_link_libraries(ExecSchemaScanners PUBLIC - ExecSchemaScannerCore +target_link_libraries(SchemaScannerFrontend PUBLIC + SchemaScannerCore +) + +target_link_libraries(SchemaScannerFrontend PRIVATE + SchemaScannerLocal + Platform + Expr + Runtime + ChunkCore + ColumnCore + Types + Common + Base + Gutil + StarRocksGen +) + +set(SCHEMA_SCANNER_STORAGE_FILES + schema_scanner/schema_be_tablets_scanner.cpp + schema_scanner/schema_be_txns_scanner.cpp + schema_scanner/schema_be_compactions_scanner.cpp + schema_scanner/schema_be_cloud_native_compactions_scanner.cpp + schema_scanner/schema_be_tablet_write_log_scanner.cpp +) + +ADD_BE_LIB(SchemaScannerStorage + ${SCHEMA_SCANNER_STORAGE_FILES} +) + +target_link_libraries(SchemaScannerStorage PUBLIC + SchemaScannerCore +) + +target_link_libraries(SchemaScannerStorage PRIVATE + SchemaScannerFrontend + SchemaScannerLocal + Storage + StoragePrimitive + Runtime + ChunkCore + ColumnCore + Types + Common + Base + Gutil + StarRocksGen +) + +ADD_BE_LIB(SchemaScannerCache + schema_scanner/schema_be_datacache_metrics_scanner.cpp +) + +target_link_libraries(SchemaScannerCache PUBLIC + SchemaScannerCore +) + +target_link_libraries(SchemaScannerCache PRIVATE + Cache Runtime ChunkCore ColumnCore @@ -317,54 +438,6 @@ set(EXEC_NON_PIPELINE_FILES schema_scan_node.cpp dictionary_cache_writer.cpp arrow_flight_batch_reader.cpp - schema_scanner/schema_analyze_status.cpp - schema_scanner/schema_tables_scanner.cpp - schema_scanner/schema_schemata_scanner.cpp - schema_scanner/schema_variables_scanner.cpp - schema_scanner/schema_columns_scanner.cpp - schema_scanner/schema_column_stats_usage_scanner.cpp - schema_scanner/schema_views_scanner.cpp - schema_scanner/schema_materialized_view_refresh_jobs_scanner.cpp - schema_scanner/schema_materialized_views_scanner.cpp - schema_scanner/schema_tasks_scanner.cpp - schema_scanner/schema_task_runs_scanner.cpp - schema_scanner/schema_loads_scanner.cpp - schema_scanner/schema_load_tracking_logs_scanner.cpp - schema_scanner/schema_partitions_meta_scanner.cpp - schema_scanner/schema_user_privileges_scanner.cpp - schema_scanner/schema_schema_privileges_scanner.cpp - schema_scanner/schema_table_privileges_scanner.cpp - schema_scanner/schema_tables_config_scanner.cpp - schema_scanner/schema_be_tablets_scanner.cpp - schema_scanner/schema_be_txns_scanner.cpp - schema_scanner/schema_be_logs_scanner.cpp - schema_scanner/schema_be_metrics_scanner.cpp - schema_scanner/schema_fe_metrics_scanner.cpp - schema_scanner/schema_fe_threads_scanner.cpp - schema_scanner/schema_fe_tablet_schedules_scanner.cpp - schema_scanner/schema_be_compactions_scanner.cpp - schema_scanner/schema_be_cloud_native_compactions_scanner.cpp - schema_scanner/schema_be_tablet_write_log_scanner.cpp - schema_scanner/schema_pipe_files.cpp - schema_scanner/schema_pipes.cpp - schema_scanner/schema_recyclebin_catalogs.cpp - schema_scanner/starrocks_role_edges_scanner.cpp - schema_scanner/starrocks_grants_to_scanner.cpp - schema_scanner/schema_helper.cpp - schema_scanner/schema_routine_load_jobs_scanner.cpp - schema_scanner/schema_stream_loads_scanner.cpp - schema_scanner/schema_be_datacache_metrics_scanner.cpp - schema_scanner/sys_object_dependencies.cpp - schema_scanner/sys_fe_locks.cpp - schema_scanner/sys_fe_memory_usage.cpp - schema_scanner/schema_temp_tables_scanner.cpp - schema_scanner/schema_warehouse_metrics.cpp - schema_scanner/schema_warehouse_queries.cpp - schema_scanner/schema_cluster_snapshots_scanner.cpp - schema_scanner/schema_cluster_snapshot_jobs_scanner.cpp - schema_scanner/schema_applicable_roles_scanner.cpp - schema_scanner/schema_keywords_scanner.cpp - schema_scanner/schema_tablet_reshard_jobs_scanner.cpp connector_scan_node.cpp ) @@ -488,7 +561,7 @@ ADD_BE_LIB(Exec $ ) -target_link_libraries(Exec PUBLIC ExecRuntime ExecPrimitive ExecSchemaScannerCore ExecSchemaScanners ExecJoinCore Runtime ComputeEnv Cache) +target_link_libraries(Exec PUBLIC ExecRuntime ExecPrimitive SchemaScannerCore ExecJoinCore Runtime ComputeEnv Cache) target_link_libraries(Exec PRIVATE Formats Platform Connector ConnectorLake ConnectorFile ConnectorHive ConnectorIceberg) ADD_BE_LIB(SchemaScannerBuiltin @@ -496,12 +569,12 @@ ADD_BE_LIB(SchemaScannerBuiltin ) target_link_libraries(SchemaScannerBuiltin PUBLIC - ExecSchemaScannerCore + SchemaScannerCore ) -# Temporary PR1 dependency: concrete schema scanners are still compiled into -# Exec and ExecSchemaScanners. PR2 will replace these with layered scanner targets. target_link_libraries(SchemaScannerBuiltin PRIVATE - Exec - ExecSchemaScanners + SchemaScannerLocal + SchemaScannerFrontend + SchemaScannerStorage + SchemaScannerCache ) diff --git a/be/src/exec/schema_scanner/schema_be_cloud_native_compactions_scanner.cpp b/be/src/exec/schema_scanner/schema_be_cloud_native_compactions_scanner.cpp index f1c0df0c5637e6..3b2a45dd080dac 100644 --- a/be/src/exec/schema_scanner/schema_be_cloud_native_compactions_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_be_cloud_native_compactions_scanner.cpp @@ -15,8 +15,7 @@ #include "exec/schema_scanner/schema_be_cloud_native_compactions_scanner.h" #include "common/system/master_info.h" -#include "exec/exec_env.h" -#include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner/schema_column_filler.h" #include "gutil/strings/substitute.h" #include "runtime/runtime_state.h" #include "storage/compaction_manager.h" diff --git a/be/src/exec/schema_scanner/schema_be_compactions_scanner.cpp b/be/src/exec/schema_scanner/schema_be_compactions_scanner.cpp index 5ced031dcc1f77..cc40e227afd6a0 100644 --- a/be/src/exec/schema_scanner/schema_be_compactions_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_be_compactions_scanner.cpp @@ -16,7 +16,7 @@ #include "base/metrics.h" #include "common/system/master_info.h" -#include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner/schema_column_filler.h" #include "gutil/strings/substitute.h" #include "storage/compaction_manager.h" #include "storage/storage_engine.h" diff --git a/be/src/exec/schema_scanner/schema_be_datacache_metrics_scanner.cpp b/be/src/exec/schema_scanner/schema_be_datacache_metrics_scanner.cpp index 7d07988844b670..03c94a9b7e2e9a 100644 --- a/be/src/exec/schema_scanner/schema_be_datacache_metrics_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_be_datacache_metrics_scanner.cpp @@ -16,7 +16,6 @@ #include "cache/datacache.h" #include "common/system/master_info.h" -#include "exec/exec_env.h" #include "gutil/strings/substitute.h" #include "types/datum.h" diff --git a/be/src/exec/schema_scanner/schema_be_logs_scanner.cpp b/be/src/exec/schema_scanner/schema_be_logs_scanner.cpp index 9b36e6a042cd90..78a22a3f4dd1f2 100644 --- a/be/src/exec/schema_scanner/schema_be_logs_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_be_logs_scanner.cpp @@ -16,7 +16,7 @@ #include "base/time/time.h" #include "common/system/master_info.h" -#include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner/schema_column_filler.h" #include "gutil/strings/substitute.h" #include "types/logical_type.h" diff --git a/be/src/exec/schema_scanner/schema_be_tablet_write_log_scanner.cpp b/be/src/exec/schema_scanner/schema_be_tablet_write_log_scanner.cpp index 4a52102e2b60d3..8b124beceafd13 100644 --- a/be/src/exec/schema_scanner/schema_be_tablet_write_log_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_be_tablet_write_log_scanner.cpp @@ -15,7 +15,7 @@ #include "exec/schema_scanner/schema_be_tablet_write_log_scanner.h" #include "column/column_helper.h" -#include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner/schema_column_filler.h" #include "runtime/chunk_helper.h" #include "runtime/runtime_state.h" #include "types/datetime_value.h" diff --git a/be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp b/be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp index 9635a900dc7522..735c339c60df72 100644 --- a/be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp @@ -15,7 +15,6 @@ #include "exec/schema_scanner/schema_be_tablets_scanner.h" #include "common/system/master_info.h" -#include "exec/exec_env.h" #include "exec/schema_scanner/schema_helper.h" #include "gen_cpp/Types_types.h" // for TStorageMedium::type #include "gutil/strings/substitute.h" diff --git a/be/src/exec/schema_scanner/schema_be_txns_scanner.cpp b/be/src/exec/schema_scanner/schema_be_txns_scanner.cpp index 0d1430c9083c7f..ab9a971d3ed1db 100644 --- a/be/src/exec/schema_scanner/schema_be_txns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_be_txns_scanner.cpp @@ -16,7 +16,7 @@ #include "base/metrics.h" #include "common/system/master_info.h" -#include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner/schema_column_filler.h" #include "gutil/strings/substitute.h" #include "storage/storage_engine.h" #include "storage/txn_manager.h" diff --git a/be/src/exec/schema_scanner/schema_column_filler.h b/be/src/exec/schema_scanner/schema_column_filler.h index e18155a66d6f5c..cafca28f48222c 100644 --- a/be/src/exec/schema_scanner/schema_column_filler.h +++ b/be/src/exec/schema_scanner/schema_column_filler.h @@ -58,4 +58,9 @@ void fill_column_with_slot(Column* result, void* slot) { } } +inline void fill_data_column_with_null(Column* data_column) { + auto* nullable_column = down_cast(data_column); + nullable_column->append_nulls(1); +} + } // namespace starrocks diff --git a/be/src/exec/schema_scanner/schema_fe_metrics_scanner.cpp b/be/src/exec/schema_scanner/schema_fe_metrics_scanner.cpp index 7112e4c65b0fa9..c59ae2eb251e78 100644 --- a/be/src/exec/schema_scanner/schema_fe_metrics_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_fe_metrics_scanner.cpp @@ -17,7 +17,7 @@ #include #include "base/metrics.h" -#include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner/schema_column_filler.h" #include "gen_cpp/FrontendService.h" #include "gutil/strings/substitute.h" #include "platform/thrift_rpc_helper.h" diff --git a/be/src/exec/schema_scanner/schema_fe_tablet_schedules_scanner.cpp b/be/src/exec/schema_scanner/schema_fe_tablet_schedules_scanner.cpp index de48ba1fceb86b..ec4f726448b8de 100644 --- a/be/src/exec/schema_scanner/schema_fe_tablet_schedules_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_fe_tablet_schedules_scanner.cpp @@ -17,7 +17,6 @@ #include "exec/schema_scanner/schema_helper.h" #include "gutil/strings/substitute.h" #include "runtime/runtime_state.h" -#include "storage/tablet.h" #include "types/logical_type.h" namespace starrocks { diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index d54880a0382206..b0156d055c0572 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -19,7 +19,6 @@ #include "base/network/network_util.h" #include "common/runtime_profile.h" #include "common/util/thrift_client_cache.h" -#include "exec/exec_env.h" #include "platform/thrift_rpc_helper.h" #include "runtime/runtime_state.h" @@ -292,9 +291,4 @@ Status SchemaHelper::get_tablet_reshard_jobs_info(const SchemaScannerState& stat [&req, &res](FrontendServiceConnection& client) { client->getTabletReshardJobsInfo(*res, req); }); } -void fill_data_column_with_null(Column* data_column) { - auto* nullable_column = down_cast(data_column); - nullable_column->append_nulls(1); -} - } // namespace starrocks diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 468afad4d103ff..a7b482b4aad3a1 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -149,6 +149,4 @@ class SchemaHelper { const std::function&)>& callback); }; -void fill_data_column_with_null(Column* data_column); - } // namespace starrocks diff --git a/be/src/exec/schema_scanner/schema_partitions_meta_scanner.cpp b/be/src/exec/schema_scanner/schema_partitions_meta_scanner.cpp index 868056f44b2b17..48916cbe964aae 100644 --- a/be/src/exec/schema_scanner/schema_partitions_meta_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_partitions_meta_scanner.cpp @@ -16,10 +16,10 @@ #include +#include "base/string/parse_util.h" #include "common/logging.h" #include "exec/schema_scanner/schema_helper.h" #include "runtime/runtime_state.h" -#include "storage/utils.h" #include "types/logical_type.h" namespace starrocks { @@ -266,7 +266,7 @@ Status SchemaPartitionsMetaScanner::fill_chunk(ChunkPtr* chunk) { } case 22: { // DATA_SIZE - int64_t data_size_bytes = parse_data_size(info.data_size); + int64_t data_size_bytes = ParseUtil::parse_data_size(info.data_size); fill_column_with_slot(column, (void*)&data_size_bytes); break; } diff --git a/be/src/storage/utils.cpp b/be/src/storage/utils.cpp index a3f464a0e4d473..813898dd0c58d5 100644 --- a/be/src/storage/utils.cpp +++ b/be/src/storage/utils.cpp @@ -356,57 +356,4 @@ int caculate_delta_writer_thread_num(int thread_num_from_config) { return std::max(CpuInfo::num_cores() / 2, 16); } -int64_t parse_data_size(const std::string& value_str) { - if (value_str.empty()) return 0; - - // Trim leading/trailing spaces - size_t start = value_str.find_first_not_of(" \t\n\r"); - if (start == std::string::npos) return 0; - size_t end = value_str.find_last_not_of(" \t\n\r"); - std::string s = value_str.substr(start, end - start + 1); - - // Find where the number ends - size_t idx = 0; - bool dot_found = false; - while (idx < s.size() && (std::isdigit(s[idx]) || (!dot_found && s[idx] == '.'))) { - if (s[idx] == '.') dot_found = true; - ++idx; - } - - double num = 0; - try { - num = std::stod(s.substr(0, idx)); - } catch (...) { - return 0; - } - - // Extract and normalize unit - std::string unit = s.substr(idx); - unit.erase(std::remove_if(unit.begin(), unit.end(), ::isspace), unit.end()); - std::transform(unit.begin(), unit.end(), unit.begin(), ::toupper); - - static const std::unordered_map unit_map = {{"", 1LL}, - {"B", 1LL}, - {"K", 1024LL}, - {"KB", 1024LL}, - {"M", 1024LL * 1024}, - {"MB", 1024LL * 1024}, - {"G", 1024LL * 1024 * 1024}, - {"GB", 1024LL * 1024 * 1024}, - {"T", 1024LL * 1024 * 1024 * 1024}, - {"TB", 1024LL * 1024 * 1024 * 1024}, - {"P", 1024LL * 1024 * 1024 * 1024 * 1024}, - {"PB", 1024LL * 1024 * 1024 * 1024 * 1024}}; - - auto it = unit_map.find(unit); - if (it == unit_map.end()) return 0; - - double result = num * static_cast(it->second); - if (result > static_cast(std::numeric_limits::max()) || - result < static_cast(std::numeric_limits::min())) { - return 0; - } - return static_cast(result); -} - } // namespace starrocks diff --git a/be/src/storage/utils.h b/be/src/storage/utils.h index 81b885890bca84..254f21fabe892c 100644 --- a/be/src/storage/utils.h +++ b/be/src/storage/utils.h @@ -177,6 +177,4 @@ bool is_tracker_hit_hard_limit(MemTracker* tracker, double hard_limit_ratio); int caculate_delta_writer_thread_num(int thread_num_from_config); -int64_t parse_data_size(const std::string& value_str); - } // namespace starrocks diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index afa363d266f8c8..e487f4830b1983 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -42,16 +42,6 @@ set(EXEC_FILES ./exec/pipeline/scan/split_morsel_ticket_checker_test.cpp ./exec/query_cache/query_cache_test.cpp ./exec/query_cache/transform_operator.cpp - ./exec/schema_columns_scanner_test.cpp - ./exec/schema_fe_tablet_schedules_scanner_test.cpp - ./exec/schema_loads_scanner_test.cpp - ./exec/schema_materialized_view_refresh_jobs_scanner_test.cpp - ./exec/schema_materialized_views_scanner_test.cpp - ./exec/schema_task_runs_scanner_test.cpp - ./exec/schema_scanner/schema_be_tablet_write_log_scanner_test.cpp - ./exec/schema_scanner/schema_scanner_test.cpp - ./exec/schema_scanner/schema_fe_metrics_scanner_test.cpp - ./exec/schema_scanner/schema_partitions_meta_scanner_test.cpp ./exec/sink/connector_sink_operator_test.cpp ./exec/sink/sink_io_buffer_test.cpp ./exec/agg_hash_map_test.cpp diff --git a/be/test/base/string/parse_util_test.cpp b/be/test/base/string/parse_util_test.cpp index 10207506aa7f76..e49a8ed692755c 100644 --- a/be/test/base/string/parse_util_test.cpp +++ b/be/test/base/string/parse_util_test.cpp @@ -80,4 +80,88 @@ TEST(TestParseMemSpec, Bad) { } } +TEST(TestParseDataSize, Normal) { + // Empty string should return 0 + ASSERT_EQ(0, ParseUtil::parse_data_size("")); + + // String with only spaces should return 0 + ASSERT_EQ(0, ParseUtil::parse_data_size(" ")); + + // String with invalid unit should return 0 + ASSERT_EQ(0, ParseUtil::parse_data_size("10Bytes")); + ASSERT_EQ(0, ParseUtil::parse_data_size("10X")); + ASSERT_EQ(0, ParseUtil::parse_data_size("abc")); + ASSERT_EQ(0, ParseUtil::parse_data_size("10.5XB")); + ASSERT_EQ(0, ParseUtil::parse_data_size(std::string(1, static_cast(0xFF)))); + ASSERT_EQ(0, ParseUtil::parse_data_size(std::string("10") + static_cast(0xFF))); + + // Only number, no unit, should be treated as bytes + ASSERT_EQ(10, ParseUtil::parse_data_size("10")); + ASSERT_EQ(123, ParseUtil::parse_data_size("123")); + + // Test with 'B' and spaces + ASSERT_EQ(10, ParseUtil::parse_data_size("10B")); + ASSERT_EQ(10, ParseUtil::parse_data_size(" 10B ")); + ASSERT_EQ(10, ParseUtil::parse_data_size("10 b")); + ASSERT_EQ(10, ParseUtil::parse_data_size("10 b ")); + + // Test with K/k/KB/kb + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size("10K")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size("10KB")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size("10k")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size("10kb")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size(" 10 KB ")); + + // Test with M/m/MB/mb + ASSERT_EQ(10 * 1024 * 1024, ParseUtil::parse_data_size("10M")); + ASSERT_EQ(10 * 1024 * 1024, ParseUtil::parse_data_size("10MB")); + ASSERT_EQ(10 * 1024 * 1024, ParseUtil::parse_data_size("10m")); + ASSERT_EQ(10 * 1024 * 1024, ParseUtil::parse_data_size("10mb")); + + // Test with G/g/GB/gb + ASSERT_EQ(10LL * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10G")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10GB")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10g")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10gb")); + + // Test with T/t/TB/tb + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10T")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10TB")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10t")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10tb")); + + // Test with P/p/PB/pb + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10P")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10PB")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10p")); + ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, ParseUtil::parse_data_size("10pb")); + + // Test with decimal numbers + ASSERT_EQ(1536, ParseUtil::parse_data_size("1.5K")); + ASSERT_EQ(1536, ParseUtil::parse_data_size("1.5KB")); + ASSERT_EQ(1572864, ParseUtil::parse_data_size("1.5M")); + ASSERT_EQ(1610612736, ParseUtil::parse_data_size("1.5G")); + ASSERT_EQ(0, ParseUtil::parse_data_size("1.5X")); // invalid unit + + // Test with leading/trailing spaces + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size(" 10K")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size("10K ")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size(" 10K ")); + + // Test with spaces between number and unit + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size("10 K")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size("10 K")); + ASSERT_EQ(10 * 1024, ParseUtil::parse_data_size(" 10 K ")); + + // Test with negative number (should return 0) + ASSERT_EQ(0, ParseUtil::parse_data_size("-10K")); + + // Test with value exceeding int64_t max (should return 0) + ASSERT_EQ(0, ParseUtil::parse_data_size("9223372036854775808")); + ASSERT_EQ(0, ParseUtil::parse_data_size("100000000000000000000P")); + + // Test with value below int64_t min (should return 0) + ASSERT_EQ(0, ParseUtil::parse_data_size("-100000000000000000000P")); +} + } // namespace starrocks diff --git a/be/test/exec/CMakeLists.txt b/be/test/exec/CMakeLists.txt index 8e82c6e196233b..ba558d6239a1ec 100644 --- a/be/test/exec/CMakeLists.txt +++ b/be/test/exec/CMakeLists.txt @@ -17,6 +17,9 @@ if (APPLE) list(APPEND BE_TEST_MACOS_LINK_STUBS ../base/macos_link_stubs.cpp) endif() +set(SCHEMA_SCANNER_TEST_DEPENDENCIES ${STARROCKS_DEPENDENCIES}) +list(REMOVE_ITEM SCHEMA_SCANNER_TEST_DEPENDENCIES arrow_testing) + set(EXEC_RUNTIME_TEST_FILES ${BE_TEST_MACOS_LINK_STUBS} ${BE_TEST_JNI_ENV_STUB} @@ -67,7 +70,7 @@ set(SCHEMA_SCANNER_CORE_TEST_FILES ) set(SCHEMA_SCANNER_CORE_TEST_LINK_LIBS - ExecSchemaScannerCore + SchemaScannerCore Expr Runtime ChunkCore @@ -77,7 +80,7 @@ set(SCHEMA_SCANNER_CORE_TEST_LINK_LIBS Base Gutil StarRocksGen - ${STARROCKS_DEPENDENCIES} + ${SCHEMA_SCANNER_TEST_DEPENDENCIES} ${STARROCKS_STATIC_LINK_LIBS} ${WL_LINK_DYNAMIC} ${STARROCKS_SYSTEM_LINK_LIBS} ${WRAP_LINKER_FLAGS} ${WL_START_GROUP} @@ -85,23 +88,21 @@ set(SCHEMA_SCANNER_CORE_TEST_LINK_LIBS gtest ${WL_END_GROUP} ) -list(REMOVE_ITEM SCHEMA_SCANNER_CORE_TEST_LINK_LIBS arrow_testing) - add_library(starrocks_schema_scanner_core_test_objs OBJECT ${SCHEMA_SCANNER_CORE_TEST_FILES}) target_link_libraries(starrocks_schema_scanner_core_test_objs PRIVATE StarRocksPCH) set_target_properties(starrocks_schema_scanner_core_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") add_executable(schema_scanner_core_test $) target_link_libraries(schema_scanner_core_test ${SCHEMA_SCANNER_CORE_TEST_LINK_LIBS} gtest_main) -set(EXEC_SCHEMA_SCANNERS_TEST_FILES +set(SCHEMA_SCANNER_LOCAL_TEST_FILES ${BE_TEST_MACOS_LINK_STUBS} ${BE_TEST_JNI_ENV_STUB} schema_scanner/exec_schema_scanners_test.cpp ) -set(EXEC_SCHEMA_SCANNERS_TEST_LINK_LIBS - ExecSchemaScanners - ExecSchemaScannerCore +set(SCHEMA_SCANNER_LOCAL_TEST_LINK_LIBS + SchemaScannerLocal + SchemaScannerCore Runtime ChunkCore ColumnCore @@ -110,7 +111,118 @@ set(EXEC_SCHEMA_SCANNERS_TEST_LINK_LIBS Base Gutil StarRocksGen - ${STARROCKS_DEPENDENCIES} + ${SCHEMA_SCANNER_TEST_DEPENDENCIES} + ${STARROCKS_STATIC_LINK_LIBS} + ${WL_LINK_DYNAMIC} ${STARROCKS_SYSTEM_LINK_LIBS} ${WRAP_LINKER_FLAGS} + ${WL_START_GROUP} + gmock + gtest + ${WL_END_GROUP} +) + +add_library(starrocks_schema_scanner_local_test_objs OBJECT ${SCHEMA_SCANNER_LOCAL_TEST_FILES}) +target_link_libraries(starrocks_schema_scanner_local_test_objs PRIVATE StarRocksPCH) +set_target_properties(starrocks_schema_scanner_local_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") +add_executable(schema_scanner_local_test $) +target_link_libraries(schema_scanner_local_test ${SCHEMA_SCANNER_LOCAL_TEST_LINK_LIBS} gtest_main) + +set(SCHEMA_SCANNER_FRONTEND_TEST_FILES + ${BE_TEST_MACOS_LINK_STUBS} + ${BE_TEST_JNI_ENV_STUB} + schema_scanner/schema_scanner_frontend_test_main.cpp + schema_columns_scanner_test.cpp + schema_fe_tablet_schedules_scanner_test.cpp + schema_loads_scanner_test.cpp + schema_materialized_view_refresh_jobs_scanner_test.cpp + schema_materialized_views_scanner_test.cpp + schema_task_runs_scanner_test.cpp + schema_scanner/schema_fe_metrics_scanner_test.cpp + schema_scanner/schema_partitions_meta_scanner_test.cpp +) + +set(SCHEMA_SCANNER_FRONTEND_TEST_LINK_LIBS + SchemaScannerFrontend + SchemaScannerLocal + SchemaScannerCore + Platform + Expr + Runtime + ChunkCore + ColumnCore + Types + Common + Base + Gutil + StarRocksGen + ${SCHEMA_SCANNER_TEST_DEPENDENCIES} + ${STARROCKS_STATIC_LINK_LIBS} + ${WL_LINK_DYNAMIC} ${STARROCKS_SYSTEM_LINK_LIBS} ${WRAP_LINKER_FLAGS} + ${WL_START_GROUP} + gmock + gtest + ${WL_END_GROUP} +) +add_library(starrocks_schema_scanner_frontend_test_objs OBJECT ${SCHEMA_SCANNER_FRONTEND_TEST_FILES}) +target_link_libraries(starrocks_schema_scanner_frontend_test_objs PRIVATE StarRocksPCH) +set_target_properties(starrocks_schema_scanner_frontend_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") +add_executable(schema_scanner_frontend_test $) +target_link_libraries(schema_scanner_frontend_test ${SCHEMA_SCANNER_FRONTEND_TEST_LINK_LIBS}) + +set(SCHEMA_SCANNER_STORAGE_TEST_FILES + ${BE_TEST_MACOS_LINK_STUBS} + ${BE_TEST_JNI_ENV_STUB} + schema_scanner/schema_be_tablet_write_log_scanner_test.cpp + schema_scanner/schema_scanner_storage_test.cpp +) + +set(SCHEMA_SCANNER_STORAGE_TEST_LINK_LIBS + SchemaScannerStorage + SchemaScannerFrontend + SchemaScannerLocal + SchemaScannerCore + Storage + StoragePrimitive + Runtime + ChunkCore + ColumnCore + Types + Common + Base + Gutil + StarRocksGen + ${SCHEMA_SCANNER_TEST_DEPENDENCIES} + ${STARROCKS_STATIC_LINK_LIBS} + ${WL_LINK_DYNAMIC} ${STARROCKS_SYSTEM_LINK_LIBS} ${WRAP_LINKER_FLAGS} + ${WL_START_GROUP} + gmock + gtest + ${WL_END_GROUP} +) +add_library(starrocks_schema_scanner_storage_test_objs OBJECT ${SCHEMA_SCANNER_STORAGE_TEST_FILES}) +target_link_libraries(starrocks_schema_scanner_storage_test_objs PRIVATE StarRocksPCH) +set_target_properties(starrocks_schema_scanner_storage_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") +add_executable(schema_scanner_storage_test $) +target_link_libraries(schema_scanner_storage_test ${SCHEMA_SCANNER_STORAGE_TEST_LINK_LIBS} gtest_main) + +set(SCHEMA_SCANNER_CACHE_TEST_FILES + ${BE_TEST_MACOS_LINK_STUBS} + ${BE_TEST_JNI_ENV_STUB} + schema_scanner/schema_scanner_cache_test.cpp +) + +set(SCHEMA_SCANNER_CACHE_TEST_LINK_LIBS + SchemaScannerCache + SchemaScannerCore + Cache + Runtime + ChunkCore + ColumnCore + Types + Common + Base + Gutil + StarRocksGen + ${SCHEMA_SCANNER_TEST_DEPENDENCIES} ${STARROCKS_STATIC_LINK_LIBS} ${WL_LINK_DYNAMIC} ${STARROCKS_SYSTEM_LINK_LIBS} ${WRAP_LINKER_FLAGS} ${WL_START_GROUP} @@ -118,12 +230,51 @@ set(EXEC_SCHEMA_SCANNERS_TEST_LINK_LIBS gtest ${WL_END_GROUP} ) +add_library(starrocks_schema_scanner_cache_test_objs OBJECT ${SCHEMA_SCANNER_CACHE_TEST_FILES}) +target_link_libraries(starrocks_schema_scanner_cache_test_objs PRIVATE StarRocksPCH) +set_target_properties(starrocks_schema_scanner_cache_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") +add_executable(schema_scanner_cache_test $) +target_link_libraries(schema_scanner_cache_test ${SCHEMA_SCANNER_CACHE_TEST_LINK_LIBS} gtest_main) -add_library(starrocks_exec_schema_scanners_test_objs OBJECT ${EXEC_SCHEMA_SCANNERS_TEST_FILES}) -target_link_libraries(starrocks_exec_schema_scanners_test_objs PRIVATE StarRocksPCH) -set_target_properties(starrocks_exec_schema_scanners_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") -add_executable(exec_schema_scanners_test $) -target_link_libraries(exec_schema_scanners_test ${EXEC_SCHEMA_SCANNERS_TEST_LINK_LIBS} gtest_main) +set(SCHEMA_SCANNER_BUILTIN_TEST_FILES + ${BE_TEST_MACOS_LINK_STUBS} + ${BE_TEST_JNI_ENV_STUB} + schema_scanner/schema_scanner_test.cpp +) + +set(SCHEMA_SCANNER_BUILTIN_TEST_LINK_LIBS + SchemaScannerBuiltin + SchemaScannerStorage + SchemaScannerCache + SchemaScannerFrontend + SchemaScannerLocal + SchemaScannerCore + Storage + StoragePrimitive + Cache + Platform + Expr + Runtime + ChunkCore + ColumnCore + Types + Common + Base + Gutil + StarRocksGen + ${SCHEMA_SCANNER_TEST_DEPENDENCIES} + ${STARROCKS_STATIC_LINK_LIBS} + ${WL_LINK_DYNAMIC} ${STARROCKS_SYSTEM_LINK_LIBS} ${WRAP_LINKER_FLAGS} + ${WL_START_GROUP} + gmock + gtest + ${WL_END_GROUP} +) +add_library(starrocks_schema_scanner_builtin_test_objs OBJECT ${SCHEMA_SCANNER_BUILTIN_TEST_FILES}) +target_link_libraries(starrocks_schema_scanner_builtin_test_objs PRIVATE StarRocksPCH) +set_target_properties(starrocks_schema_scanner_builtin_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control") +add_executable(schema_scanner_builtin_test $) +target_link_libraries(schema_scanner_builtin_test ${SCHEMA_SCANNER_BUILTIN_TEST_LINK_LIBS} gtest_main) set(JOIN_CORE_TEST_FILES ${BE_TEST_MACOS_LINK_STUBS} diff --git a/be/test/exec/schema_scanner/schema_scanner_cache_test.cpp b/be/test/exec/schema_scanner/schema_scanner_cache_test.cpp new file mode 100644 index 00000000000000..505c781f5b8394 --- /dev/null +++ b/be/test/exec/schema_scanner/schema_scanner_cache_test.cpp @@ -0,0 +1,41 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "base/testutil/assert.h" +#include "common/object_pool.h" +#include "exec/schema_scanner/schema_be_datacache_metrics_scanner.h" + +namespace starrocks { + +TEST(SchemaScannerCacheTest, InitializesNineColumnSchemaWithoutCache) { + SchemaBeDataCacheMetricsScanner scanner; + SchemaScannerParam params; + ObjectPool pool; + + EXPECT_OK(scanner.init(¶ms, &pool)); + + const auto& slots = scanner.get_slot_descs(); + ASSERT_EQ(9, slots.size()); + + constexpr const char* expected_names[] = {"BE_ID", "STATUS", "DISK_QUOTA_BYTES", + "DISK_USED_BYTES", "MEM_QUOTA_BYTES", "MEM_USED_BYTES", + "META_USED_BYTES", "DIR_SPACES", "USED_BYTES_DETAIL"}; + for (size_t i = 0; i < slots.size(); ++i) { + EXPECT_EQ(expected_names[i], slots[i]->col_name()); + } +} + +} // namespace starrocks diff --git a/be/test/exec/schema_scanner/schema_scanner_frontend_test_main.cpp b/be/test/exec/schema_scanner/schema_scanner_frontend_test_main.cpp new file mode 100644 index 00000000000000..3e853b5a68b8a4 --- /dev/null +++ b/be/test/exec/schema_scanner/schema_scanner_frontend_test_main.cpp @@ -0,0 +1,23 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "types/time_types.h" + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + starrocks::date::init_date_cache(); + return RUN_ALL_TESTS(); +} diff --git a/be/test/exec/schema_scanner/schema_scanner_storage_test.cpp b/be/test/exec/schema_scanner/schema_scanner_storage_test.cpp new file mode 100644 index 00000000000000..221a041211dc96 --- /dev/null +++ b/be/test/exec/schema_scanner/schema_scanner_storage_test.cpp @@ -0,0 +1,49 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "base/testutil/assert.h" +#include "common/object_pool.h" +#include "exec/schema_scanner/schema_be_cloud_native_compactions_scanner.h" +#include "exec/schema_scanner/schema_be_compactions_scanner.h" +#include "exec/schema_scanner/schema_be_tablet_write_log_scanner.h" +#include "exec/schema_scanner/schema_be_tablets_scanner.h" +#include "exec/schema_scanner/schema_be_txns_scanner.h" + +namespace starrocks { + +namespace { + +template +void expect_scanner_schema_initializes() { + Scanner scanner; + SchemaScannerParam params; + ObjectPool pool; + + EXPECT_OK(scanner.init(¶ms, &pool)); + EXPECT_FALSE(scanner.get_slot_descs().empty()); +} + +} // namespace + +TEST(SchemaScannerStorageTest, AllScannerSchemasInitialize) { + expect_scanner_schema_initializes(); + expect_scanner_schema_initializes(); + expect_scanner_schema_initializes(); + expect_scanner_schema_initializes(); + expect_scanner_schema_initializes(); +} + +} // namespace starrocks diff --git a/be/test/exec/schema_scanner/schema_scanner_test.cpp b/be/test/exec/schema_scanner/schema_scanner_test.cpp index 875f3aabfe4084..f897921583001a 100644 --- a/be/test/exec/schema_scanner/schema_scanner_test.cpp +++ b/be/test/exec/schema_scanner/schema_scanner_test.cpp @@ -15,31 +15,163 @@ #include #include "exec/builtin_schema_scanner_factory.h" +#include "exec/schema_scanner/schema_analyze_status.h" +#include "exec/schema_scanner/schema_applicable_roles_scanner.h" +#include "exec/schema_scanner/schema_be_bvars_scanner.h" +#include "exec/schema_scanner/schema_be_cloud_native_compactions_scanner.h" +#include "exec/schema_scanner/schema_be_compactions_scanner.h" +#include "exec/schema_scanner/schema_be_configs_scanner.h" +#include "exec/schema_scanner/schema_be_datacache_metrics_scanner.h" +#include "exec/schema_scanner/schema_be_logs_scanner.h" +#include "exec/schema_scanner/schema_be_metrics_scanner.h" +#include "exec/schema_scanner/schema_be_tablet_write_log_scanner.h" +#include "exec/schema_scanner/schema_be_tablets_scanner.h" +#include "exec/schema_scanner/schema_be_threads_scanner.h" +#include "exec/schema_scanner/schema_be_txns_scanner.h" +#include "exec/schema_scanner/schema_charsets_scanner.h" +#include "exec/schema_scanner/schema_cluster_snapshot_jobs_scanner.h" +#include "exec/schema_scanner/schema_cluster_snapshots_scanner.h" +#include "exec/schema_scanner/schema_collation_character_set_applicability_scanner.h" +#include "exec/schema_scanner/schema_collations_scanner.h" +#include "exec/schema_scanner/schema_column_stats_usage_scanner.h" +#include "exec/schema_scanner/schema_columns_scanner.h" +#include "exec/schema_scanner/schema_dummy_scanner.h" +#include "exec/schema_scanner/schema_fe_metrics_scanner.h" +#include "exec/schema_scanner/schema_fe_tablet_schedules_scanner.h" +#include "exec/schema_scanner/schema_fe_threads_scanner.h" +#include "exec/schema_scanner/schema_keywords_scanner.h" +#include "exec/schema_scanner/schema_load_tracking_logs_scanner.h" +#include "exec/schema_scanner/schema_loads_scanner.h" +#include "exec/schema_scanner/schema_materialized_view_refresh_jobs_scanner.h" +#include "exec/schema_scanner/schema_materialized_views_scanner.h" +#include "exec/schema_scanner/schema_partitions_meta_scanner.h" +#include "exec/schema_scanner/schema_pipe_files.h" +#include "exec/schema_scanner/schema_pipes.h" +#include "exec/schema_scanner/schema_recyclebin_catalogs.h" +#include "exec/schema_scanner/schema_routine_load_jobs_scanner.h" +#include "exec/schema_scanner/schema_schema_privileges_scanner.h" +#include "exec/schema_scanner/schema_schemata_scanner.h" +#include "exec/schema_scanner/schema_stream_loads_scanner.h" +#include "exec/schema_scanner/schema_table_privileges_scanner.h" +#include "exec/schema_scanner/schema_tables_config_scanner.h" +#include "exec/schema_scanner/schema_tables_scanner.h" #include "exec/schema_scanner/schema_tablet_reshard_jobs_scanner.h" +#include "exec/schema_scanner/schema_task_runs_scanner.h" +#include "exec/schema_scanner/schema_tasks_scanner.h" +#include "exec/schema_scanner/schema_temp_tables_scanner.h" +#include "exec/schema_scanner/schema_user_privileges_scanner.h" +#include "exec/schema_scanner/schema_variables_scanner.h" +#include "exec/schema_scanner/schema_views_scanner.h" +#include "exec/schema_scanner/schema_warehouse_metrics.h" +#include "exec/schema_scanner/schema_warehouse_queries.h" +#include "exec/schema_scanner/starrocks_grants_to_scanner.h" +#include "exec/schema_scanner/starrocks_role_edges_scanner.h" +#include "exec/schema_scanner/sys_fe_locks.h" +#include "exec/schema_scanner/sys_fe_memory_usage.h" +#include "exec/schema_scanner/sys_object_dependencies.h" #include "gen_cpp/Descriptors_types.h" namespace starrocks { -class SchemaScannerTest : public ::testing::Test {}; +namespace { -TEST_F(SchemaScannerTest, test_create) { +template +void expect_scanner_type(const SchemaScannerFactory& factory, TSchemaTableType::type type) { + auto scanner = factory.create(type); + ASSERT_NE(nullptr, scanner); + EXPECT_NE(nullptr, dynamic_cast(scanner.get())); +} + +void expect_variable_scanner(const SchemaScannerFactory& factory, TSchemaTableType::type table_type, + TVarType::type variable_type) { + auto scanner = factory.create(table_type); + ASSERT_NE(nullptr, scanner); + auto* variables = dynamic_cast(scanner.get()); + ASSERT_NE(nullptr, variables); + EXPECT_EQ(variable_type, variables->_type); +} + +void expect_grants_scanner(const SchemaScannerFactory& factory, TSchemaTableType::type table_type, + TGrantsToType::type grants_type) { + auto scanner = factory.create(table_type); + ASSERT_NE(nullptr, scanner); + auto* grants = dynamic_cast(scanner.get()); + ASSERT_NE(nullptr, grants); + EXPECT_EQ(grants_type, grants->_type); +} + +} // namespace + +TEST(SchemaScannerBuiltinTest, CreatesExpectedConcreteScannerForEveryTableType) { + auto factory = create_builtin_schema_scanner_factory(); + ASSERT_NE(nullptr, factory); + + expect_scanner_type(*factory, TSchemaTableType::SCH_TABLES); + expect_scanner_type(*factory, TSchemaTableType::SCH_SCHEMATA); + expect_scanner_type(*factory, TSchemaTableType::SCH_COLUMNS); + expect_scanner_type(*factory, TSchemaTableType::SCH_CHARSETS); + expect_scanner_type(*factory, TSchemaTableType::SCH_COLLATIONS); + expect_scanner_type( + *factory, TSchemaTableType::SCH_COLLATION_CHARACTER_SET_APPLICABILITY); + expect_variable_scanner(*factory, TSchemaTableType::SCH_GLOBAL_VARIABLES, TVarType::GLOBAL); + expect_variable_scanner(*factory, TSchemaTableType::SCH_SESSION_VARIABLES, TVarType::SESSION); + expect_variable_scanner(*factory, TSchemaTableType::SCH_VARIABLES, TVarType::SESSION); + expect_scanner_type(*factory, TSchemaTableType::SCH_USER_PRIVILEGES); + expect_scanner_type(*factory, TSchemaTableType::SCH_SCHEMA_PRIVILEGES); + expect_scanner_type(*factory, TSchemaTableType::SCH_TABLE_PRIVILEGES); + expect_scanner_type(*factory, TSchemaTableType::SCH_VIEWS); + expect_scanner_type(*factory, TSchemaTableType::SCH_TASKS); + expect_scanner_type(*factory, TSchemaTableType::SCH_TASK_RUNS); + expect_scanner_type(*factory, TSchemaTableType::SCH_MATERIALIZED_VIEWS); + expect_scanner_type(*factory, + TSchemaTableType::SCH_MATERIALIZED_VIEW_REFRESH_JOBS); + expect_scanner_type(*factory, TSchemaTableType::SCH_LOADS); + expect_scanner_type(*factory, TSchemaTableType::SCH_LOAD_TRACKING_LOGS); + expect_scanner_type(*factory, TSchemaTableType::SCH_TABLES_CONFIG); + expect_variable_scanner(*factory, TSchemaTableType::SCH_VERBOSE_SESSION_VARIABLES, TVarType::VERBOSE); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_TABLETS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_METRICS); + expect_scanner_type(*factory, TSchemaTableType::SCH_FE_METRICS); + expect_scanner_type(*factory, TSchemaTableType::SCH_FE_THREADS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_TXNS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_CONFIGS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_THREADS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_LOGS); + expect_scanner_type(*factory, TSchemaTableType::SCH_FE_TABLET_SCHEDULES); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_COMPACTIONS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_BVARS); + expect_scanner_type(*factory, + TSchemaTableType::SCH_BE_CLOUD_NATIVE_COMPACTIONS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_TABLET_WRITE_LOG); + expect_scanner_type(*factory, TSchemaTableType::STARROCKS_ROLE_EDGES); + expect_grants_scanner(*factory, TSchemaTableType::STARROCKS_GRANT_TO_ROLES, TGrantsToType::ROLE); + expect_grants_scanner(*factory, TSchemaTableType::STARROCKS_GRANT_TO_USERS, TGrantsToType::USER); + expect_scanner_type(*factory, TSchemaTableType::STARROCKS_OBJECT_DEPENDENCIES); + expect_scanner_type(*factory, TSchemaTableType::SCH_ROUTINE_LOAD_JOBS); + expect_scanner_type(*factory, TSchemaTableType::SCH_STREAM_LOADS); + expect_scanner_type(*factory, TSchemaTableType::SCH_PIPE_FILES); + expect_scanner_type(*factory, TSchemaTableType::SCH_PIPES); + expect_scanner_type(*factory, TSchemaTableType::SYS_FE_LOCKS); + expect_scanner_type(*factory, TSchemaTableType::SCH_BE_DATACACHE_METRICS); + expect_scanner_type(*factory, TSchemaTableType::SCH_PARTITIONS_META); + expect_scanner_type(*factory, TSchemaTableType::SYS_FE_MEMORY_USAGE); + expect_scanner_type(*factory, TSchemaTableType::SCH_TEMP_TABLES); + expect_scanner_type(*factory, TSchemaTableType::SCH_RECYCLEBIN_CATALOGS); + expect_scanner_type(*factory, TSchemaTableType::SCH_COLUMN_STATS_USAGE); + expect_scanner_type(*factory, TSchemaTableType::SCH_ANALYZE_STATUS); + expect_scanner_type(*factory, TSchemaTableType::SCH_CLUSTER_SNAPSHOTS); + expect_scanner_type(*factory, TSchemaTableType::SCH_CLUSTER_SNAPSHOT_JOBS); + expect_scanner_type(*factory, TSchemaTableType::SCH_APPLICABLE_ROLES); + expect_scanner_type(*factory, TSchemaTableType::SCH_KEYWORDS); + expect_scanner_type(*factory, TSchemaTableType::SCH_WAREHOUSE_METRICS); + expect_scanner_type(*factory, TSchemaTableType::SCH_WAREHOUSE_QUERIES); + expect_scanner_type(*factory, TSchemaTableType::SCH_TABLET_RESHARD_JOBS); +} + +TEST(SchemaScannerBuiltinTest, UnknownTableTypeCreatesDummyScanner) { auto factory = create_builtin_schema_scanner_factory(); - { - auto scanner = factory->create(TSchemaTableType::SCH_TABLET_RESHARD_JOBS); - ASSERT_NE(scanner, nullptr); - auto* reshard_jobs_scanner = dynamic_cast(scanner.get()); - ASSERT_NE(reshard_jobs_scanner, nullptr); - } - { - // Test an existing one to ensure it still works - auto scanner = factory->create(TSchemaTableType::SCH_TABLES); - ASSERT_NE(scanner, nullptr); - } - { - // Test default case - auto scanner = factory->create(static_cast(-1)); - ASSERT_NE(scanner, nullptr); - } + ASSERT_NE(nullptr, factory); + expect_scanner_type(*factory, static_cast(-1)); } } // namespace starrocks diff --git a/be/test/storage/utils_test.cpp b/be/test/storage/utils_test.cpp index e034214148fd1f..766a733b4e55b3 100644 --- a/be/test/storage/utils_test.cpp +++ b/be/test/storage/utils_test.cpp @@ -66,85 +66,4 @@ TEST_F(TestUtils, test_valid_datetime) { ASSERT_FALSE(valid_datetime("2020-01-01 00:00:00.0123456")); } -TEST_F(TestUtils, test_parse_data_size) { - // Empty string should return 0 - ASSERT_EQ(0, parse_data_size("")); - - // String with only spaces should return 0 - ASSERT_EQ(0, parse_data_size(" ")); - - // String with invalid unit should return 0 - ASSERT_EQ(0, parse_data_size("10Bytes")); - ASSERT_EQ(0, parse_data_size("10X")); - ASSERT_EQ(0, parse_data_size("abc")); - ASSERT_EQ(0, parse_data_size("10.5XB")); - - // Only number, no unit, should be treated as bytes - ASSERT_EQ(10, parse_data_size("10")); - ASSERT_EQ(123, parse_data_size("123")); - - // Test with 'B' and spaces - ASSERT_EQ(10, parse_data_size("10B")); - ASSERT_EQ(10, parse_data_size(" 10B ")); - ASSERT_EQ(10, parse_data_size("10 b")); - ASSERT_EQ(10, parse_data_size("10 b ")); - - // Test with K/k/KB/kb - ASSERT_EQ(10 * 1024, parse_data_size("10K")); - ASSERT_EQ(10 * 1024, parse_data_size("10KB")); - ASSERT_EQ(10 * 1024, parse_data_size("10k")); - ASSERT_EQ(10 * 1024, parse_data_size("10kb")); - ASSERT_EQ(10 * 1024, parse_data_size(" 10 KB ")); - - // Test with M/m/MB/mb - ASSERT_EQ(10 * 1024 * 1024, parse_data_size("10M")); - ASSERT_EQ(10 * 1024 * 1024, parse_data_size("10MB")); - ASSERT_EQ(10 * 1024 * 1024, parse_data_size("10m")); - ASSERT_EQ(10 * 1024 * 1024, parse_data_size("10mb")); - - // Test with G/g/GB/gb - ASSERT_EQ(10LL * 1024 * 1024 * 1024, parse_data_size("10G")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024, parse_data_size("10GB")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024, parse_data_size("10g")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024, parse_data_size("10gb")); - - // Test with T/t/TB/tb - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, parse_data_size("10T")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, parse_data_size("10TB")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, parse_data_size("10t")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024, parse_data_size("10tb")); - - // Test with P/p/PB/pb - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, parse_data_size("10P")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, parse_data_size("10PB")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, parse_data_size("10p")); - ASSERT_EQ(10LL * 1024 * 1024 * 1024 * 1024 * 1024, parse_data_size("10pb")); - - // Test with decimal numbers - ASSERT_EQ(1536, parse_data_size("1.5K")); - ASSERT_EQ(1536, parse_data_size("1.5KB")); - ASSERT_EQ(1572864, parse_data_size("1.5M")); - ASSERT_EQ(1610612736, parse_data_size("1.5G")); - ASSERT_EQ(0, parse_data_size("1.5X")); // invalid unit - - // Test with leading/trailing spaces - ASSERT_EQ(10 * 1024, parse_data_size(" 10K")); - ASSERT_EQ(10 * 1024, parse_data_size("10K ")); - ASSERT_EQ(10 * 1024, parse_data_size(" 10K ")); - - // Test with spaces between number and unit - ASSERT_EQ(10 * 1024, parse_data_size("10 K")); - ASSERT_EQ(10 * 1024, parse_data_size("10 K")); - ASSERT_EQ(10 * 1024, parse_data_size(" 10 K ")); - - // Test with negative number (should return 0) - ASSERT_EQ(0, parse_data_size("-10K")); - - // Test with value exceeding int64_t max (should return 0) - ASSERT_EQ(0, parse_data_size("100000000000000000000P")); - - // Test with value below int64_t min (should return 0) - ASSERT_EQ(0, parse_data_size("-100000000000000000000P")); -} - } // namespace starrocks diff --git a/build-support/check_be_module_boundaries.py b/build-support/check_be_module_boundaries.py index c7b5204a56836a..94bd4814adf868 100644 --- a/build-support/check_be_module_boundaries.py +++ b/build-support/check_be_module_boundaries.py @@ -108,6 +108,15 @@ class ModuleSpec: summary: str = "" +@dataclass(frozen=True) +class TargetDependencyGuard: + id: str + target: str + forbidden_dependency_prefixes: tuple[str, ...] + allowed_dependencies: tuple[str, ...] + remediation: str + + @dataclass class CMakeState: target_sources: dict[str, list[str]] @@ -161,7 +170,21 @@ def load_manifest(path: Path) -> dict: summary=raw.get("summary", ""), ) ) - return {"modules": modules} + target_dependency_guards = [] + for raw in payload.get("target_dependency_guards", []): + target_dependency_guards.append( + TargetDependencyGuard( + id=raw.get("id", f"{raw['target'].lower()}targetdependencyguard"), + target=raw["target"], + forbidden_dependency_prefixes=tuple(raw.get("forbidden_dependency_prefixes", [])), + allowed_dependencies=tuple(raw.get("allowed_dependencies", [])), + remediation=raw.get( + "remediation", + "Remove the forbidden direct target dependency or move composition to an allowed layer.", + ), + ) + ) + return {"modules": modules, "target_dependency_guards": target_dependency_guards} def load_baseline(path: Path) -> dict[str, set[tuple[str, str, str]]]: @@ -300,6 +323,10 @@ def collect_violations( target_link_violations.extend(check_target_links_for_module(module, cmake_state)) test_link_violations.extend(check_test_links_for_module(module, cmake_state)) + target_link_violations.extend( + check_target_dependency_guards(manifest.get("target_dependency_guards", []), cmake_state) + ) + return Violations( include_violations=sorted(include_violations, key=lambda item: (item.module, item.path, item.edge)), target_link_violations=sorted(target_link_violations, key=lambda item: (item.module, item.path, item.edge)), @@ -455,6 +482,29 @@ def check_target_links_for_module(module: ModuleSpec, cmake_state: CMakeState) - return violations +def check_target_dependency_guards( + guards: Iterable[TargetDependencyGuard], cmake_state: CMakeState +) -> list[Violation]: + violations: list[Violation] = [] + for guard in guards: + allowed_dependencies = set(guard.allowed_dependencies) + for dep in cmake_state.target_links.get(guard.target, []): + if dep in allowed_dependencies: + continue + if not _matches_any_prefix(dep, guard.forbidden_dependency_prefixes): + continue + violations.append( + Violation( + module=guard.id, + path=f"target:{guard.target}", + edge=dep, + detail=f"direct target dependency is forbidden by the guard for {guard.target}", + remediation=guard.remediation, + ) + ) + return violations + + def check_test_links_for_module(module: ModuleSpec, cmake_state: CMakeState) -> list[Violation]: violations: list[Violation] = [] allowed_test_links = set(module.allowed_test_link_deps) diff --git a/build-support/test_check_be_module_boundaries.py b/build-support/test_check_be_module_boundaries.py index 1c62c4f5aff7cf..7585f341ed8ce0 100644 --- a/build-support/test_check_be_module_boundaries.py +++ b/build-support/test_check_be_module_boundaries.py @@ -316,6 +316,71 @@ def test_parses_target_sources_and_test_link_deps(self) -> None: cmake_state.target_links["ColumnCore"], ) + def test_load_manifest_parses_target_dependency_guards(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + manifest_path = Path(tmpdir) / "module_boundary_manifest.json" + manifest_path.write_text( + json.dumps( + { + "modules": [], + "target_dependency_guards": [ + { + "id": "execschemascannerdependencyguard", + "target": "Exec", + "forbidden_dependency_prefixes": ["SchemaScanner"], + "allowed_dependencies": ["SchemaScannerCore"], + "remediation": "Compose concrete schema scanners above Exec.", + } + ], + } + ) + ) + + manifest = MODULE.load_manifest(manifest_path) + + self.assertEqual( + [ + MODULE.TargetDependencyGuard( + id="execschemascannerdependencyguard", + target="Exec", + forbidden_dependency_prefixes=("SchemaScanner",), + allowed_dependencies=("SchemaScannerCore",), + remediation="Compose concrete schema scanners above Exec.", + ) + ], + manifest["target_dependency_guards"], + ) + + def test_target_dependency_guard_allows_core_and_rejects_concrete_scanner_layer(self) -> None: + guard = MODULE.TargetDependencyGuard( + id="execschemascannerdependencyguard", + target="Exec", + forbidden_dependency_prefixes=("SchemaScanner",), + allowed_dependencies=("SchemaScannerCore",), + remediation="Compose concrete schema scanners above Exec.", + ) + cmake_state = MODULE.CMakeState( + target_sources={}, + target_definition_paths={}, + target_links={"Exec": ["SchemaScannerCore", "SchemaScannerFrontend"]}, + test_target_links={}, + ) + + violations = MODULE.collect_violations( + repo_root=Path("."), + manifest={"modules": [], "target_dependency_guards": [guard]}, + cmake_state=cmake_state, + selected_modules={"unrelated-changed-module"}, + ) + + self.assertEqual([], violations.include_violations) + self.assertEqual([], violations.test_link_violations) + self.assertEqual(1, len(violations.target_link_violations)) + violation = violations.target_link_violations[0] + self.assertEqual("execschemascannerdependencyguard", violation.module) + self.assertEqual("target:Exec", violation.path) + self.assertEqual("SchemaScannerFrontend", violation.edge) + def test_changed_paths_limit_checked_modules(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo = Path(tmpdir)