Skip to content

Commit 06882fe

Browse files
dantengskydantengsky
committed
feat: introduce block meta cache (#17360)
* feat: introduce BlockMetaCache * tweak sys tbl `system.cache` and configs * cargo fmt * update ut gloden file * fix: set cache cap of `memory_cache_block_meta`
1 parent 254d5b9 commit 06882fe

File tree

9 files changed

+80
-11
lines changed

9 files changed

+80
-11
lines changed

‎src/query/config/src/config.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2701,6 +2701,15 @@ pub struct CacheConfig {
27012701
)]
27022702
pub block_meta_count: u64,
27032703

2704+
/// Max number of **segment** which all of its block meta will be cached.
2705+
/// Note that a segment may contain multiple block metadata entries.
2706+
#[clap(
2707+
long = "cache-segment-block-metas-count",
2708+
value_name = "VALUE",
2709+
default_value = "0"
2710+
)]
2711+
pub segment_block_metas_count: u64,
2712+
27042713
/// Max number of cached table statistic meta
27052714
#[clap(
27062715
long = "cache-table-meta-statistic-count",
@@ -2999,6 +3008,7 @@ mod cache_config_converters {
29993008
table_meta_snapshot_count: value.table_meta_snapshot_count,
30003009
table_meta_segment_bytes: value.table_meta_segment_bytes,
30013010
block_meta_count: value.block_meta_count,
3011+
segment_block_metas_count: value.segment_block_metas_count,
30023012
table_meta_statistic_count: value.table_meta_statistic_count,
30033013
enable_table_index_bloom: value.enable_table_bloom_index_cache,
30043014
table_bloom_index_meta_count: value.table_bloom_index_meta_count,
@@ -3043,6 +3053,7 @@ mod cache_config_converters {
30433053
table_data_deserialized_data_bytes: value.table_data_deserialized_data_bytes,
30443054
table_data_deserialized_memory_ratio: value.table_data_deserialized_memory_ratio,
30453055
table_meta_segment_count: None,
3056+
segment_block_metas_count: value.segment_block_metas_count,
30463057
}
30473058
}
30483059
}

‎src/query/config/src/inner.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,10 @@ pub struct CacheConfig {
536536
/// Max number of cached table block meta
537537
pub block_meta_count: u64,
538538

539+
/// Max number of **segment** which all of its block meta will be cached.
540+
/// Note that a segment may contain multiple block metadata entries.
541+
pub segment_block_metas_count: u64,
542+
539543
/// Max number of cached table segment
540544
pub table_meta_statistic_count: u64,
541545

@@ -683,6 +687,7 @@ impl Default for CacheConfig {
683687
table_meta_snapshot_count: 256,
684688
table_meta_segment_bytes: 1073741824,
685689
block_meta_count: 0,
690+
segment_block_metas_count: 0,
686691
table_meta_statistic_count: 256,
687692
enable_table_index_bloom: true,
688693
table_bloom_index_meta_count: 3000,

‎src/query/service/tests/it/storages/testdata/configs_table_basic.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ DB.Table: 'system'.'configs', Table: configs-table_id:1, ver:0, Engine: SystemCo
1515
| 'cache' | 'inverted_index_filter_memory_ratio' | '0' | '' |
1616
| 'cache' | 'inverted_index_filter_size' | '2147483648' | '' |
1717
| 'cache' | 'inverted_index_meta_count' | '3000' | '' |
18+
| 'cache' | 'segment_block_metas_count' | '0' | '' |
1819
| 'cache' | 'table_bloom_index_filter_count' | '0' | '' |
1920
| 'cache' | 'table_bloom_index_filter_size' | '2147483648' | '' |
2021
| 'cache' | 'table_bloom_index_meta_count' | '3000' | '' |

‎src/query/storages/common/cache/src/caches.rs

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,15 @@ use crate::InMemoryLruCache;
3535
/// In memory object cache of SegmentInfo
3636
pub type CompactSegmentInfoCache = InMemoryLruCache<CompactSegmentInfo>;
3737

38-
pub type BlockMetaCache = InMemoryLruCache<Vec<Arc<BlockMeta>>>;
38+
/// In-memory cache for all the block metadata of individual segments.
39+
///
40+
/// Note that this cache may be memory-intensive, as each item of this cache
41+
/// contains ALL the BlockMeta of a segment, for well-compacted segment, the
42+
/// number of BlockMeta might be 1000 ~ 2000.
43+
pub type SegmentBlockMetasCache = InMemoryLruCache<Vec<Arc<BlockMeta>>>;
44+
45+
/// In-memory cache of individual BlockMeta.
46+
pub type BlockMetaCache = InMemoryLruCache<Arc<BlockMeta>>;
3947

4048
/// In memory object cache of TableSnapshot
4149
pub type TableSnapshotCache = InMemoryLruCache<TableSnapshot>;
@@ -95,9 +103,9 @@ impl CachedObject<TableSnapshot> for TableSnapshot {
95103
}
96104

97105
impl CachedObject<Vec<Arc<BlockMeta>>> for Vec<Arc<BlockMeta>> {
98-
type Cache = BlockMetaCache;
106+
type Cache = SegmentBlockMetasCache;
99107
fn cache() -> Option<Self::Cache> {
100-
CacheManager::instance().get_block_meta_cache()
108+
CacheManager::instance().get_segment_block_metas_cache()
101109
}
102110
}
103111

@@ -187,6 +195,15 @@ impl From<Vec<Arc<BlockMeta>>> for CacheValue<Vec<Arc<BlockMeta>>> {
187195
}
188196
}
189197

198+
impl From<Arc<BlockMeta>> for CacheValue<Arc<BlockMeta>> {
199+
fn from(value: Arc<BlockMeta>) -> Self {
200+
CacheValue {
201+
inner: Arc::new(value),
202+
mem_bytes: 0,
203+
}
204+
}
205+
}
206+
190207
impl From<TableSnapshot> for CacheValue<TableSnapshot> {
191208
fn from(value: TableSnapshot) -> Self {
192209
CacheValue {

‎src/query/storages/common/cache/src/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ mod read;
2323

2424
pub use cache::CacheAccessor;
2525
pub use cache::Unit;
26-
pub use caches::BlockMetaCache;
2726
pub use caches::CacheValue;
2827
pub use caches::CachedObject;
28+
pub use caches::SegmentBlockMetasCache;
2929
pub use caches::SizedColumnArray;
3030
pub use manager::CacheManager;
3131
pub use providers::DiskCacheError;

‎src/query/storages/common/cache/src/manager.rs

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ use crate::caches::FileMetaDataCache;
3434
use crate::caches::InvertedIndexFileCache;
3535
use crate::caches::InvertedIndexMetaCache;
3636
use crate::caches::PrunePartitionsCache;
37+
use crate::caches::SegmentBlockMetasCache;
3738
use crate::caches::TableSnapshotCache;
3839
use crate::caches::TableSnapshotStatisticCache;
3940
use crate::InMemoryLruCache;
@@ -78,6 +79,7 @@ pub struct CacheManager {
7879
parquet_meta_data_cache: CacheSlot<FileMetaDataCache>,
7980
table_data_cache: CacheSlot<TableDataCache>,
8081
in_memory_table_data_cache: CacheSlot<ColumnArrayCache>,
82+
segment_block_metas_cache: CacheSlot<SegmentBlockMetasCache>,
8183
block_meta_cache: CacheSlot<BlockMetaCache>,
8284
}
8385

@@ -151,6 +153,7 @@ impl CacheManager {
151153
table_statistic_cache: CacheSlot::new(None),
152154
table_data_cache,
153155
in_memory_table_data_cache,
156+
segment_block_metas_cache: CacheSlot::new(None),
154157
block_meta_cache: CacheSlot::new(None),
155158
}));
156159
} else {
@@ -201,8 +204,14 @@ impl CacheManager {
201204
DEFAULT_PARQUET_META_DATA_CACHE_ITEMS,
202205
);
203206

207+
let segment_block_metas_cache = Self::new_items_cache_slot(
208+
MEMORY_CACHE_SEGMENT_BLOCK_METAS,
209+
config.block_meta_count as usize,
210+
);
211+
204212
let block_meta_cache = Self::new_items_cache_slot(
205213
MEMORY_CACHE_BLOCK_META,
214+
// TODO replace this config
206215
config.block_meta_count as usize,
207216
);
208217

@@ -217,8 +226,9 @@ impl CacheManager {
217226
table_statistic_cache,
218227
table_data_cache,
219228
in_memory_table_data_cache,
220-
block_meta_cache,
229+
segment_block_metas_cache,
221230
parquet_meta_data_cache,
231+
block_meta_cache,
222232
}));
223233
}
224234

@@ -270,6 +280,9 @@ impl CacheManager {
270280
MEMORY_CACHE_TABLE_SNAPSHOT => {
271281
Self::set_items_capacity(&self.table_snapshot_cache, new_capacity, name);
272282
}
283+
MEMORY_CACHE_SEGMENT_BLOCK_METAS => {
284+
Self::set_items_capacity(&self.segment_block_metas_cache, new_capacity, name);
285+
}
273286
MEMORY_CACHE_BLOCK_META => {
274287
Self::set_items_capacity(&self.block_meta_cache, new_capacity, name);
275288
}
@@ -311,6 +324,10 @@ impl CacheManager {
311324
}
312325
}
313326

327+
pub fn get_segment_block_metas_cache(&self) -> Option<SegmentBlockMetasCache> {
328+
self.segment_block_metas_cache.get()
329+
}
330+
314331
pub fn get_block_meta_cache(&self) -> Option<BlockMetaCache> {
315332
self.block_meta_cache.get()
316333
}
@@ -426,4 +443,6 @@ const MEMORY_CACHE_BLOOM_INDEX_FILTER: &str = "memory_cache_bloom_index_filter";
426443
const MEMORY_CACHE_COMPACT_SEGMENT_INFO: &str = "memory_cache_compact_segment_info";
427444
const MEMORY_CACHE_TABLE_STATISTICS: &str = "memory_cache_table_statistics";
428445
const MEMORY_CACHE_TABLE_SNAPSHOT: &str = "memory_cache_table_snapshot";
446+
const MEMORY_CACHE_SEGMENT_BLOCK_METAS: &str = "memory_cache_segment_block_metas";
447+
429448
const MEMORY_CACHE_BLOCK_META: &str = "memory_cache_block_meta";

‎src/query/storages/fuse/src/pruning/fuse_pruner.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ use databend_common_expression::SEGMENT_NAME_COL_NAME;
2626
use databend_common_functions::BUILTIN_FUNCTIONS;
2727
use databend_common_sql::field_default_value;
2828
use databend_common_sql::BloomIndexColumns;
29-
use databend_storages_common_cache::BlockMetaCache;
3029
use databend_storages_common_cache::CacheAccessor;
3130
use databend_storages_common_cache::CacheManager;
31+
use databend_storages_common_cache::SegmentBlockMetasCache;
3232
use databend_storages_common_index::RangeIndex;
3333
use databend_storages_common_pruner::BlockMetaIndex;
3434
use databend_storages_common_pruner::InternalColumnPruner;
@@ -195,7 +195,7 @@ pub struct FusePruner {
195195
pub push_down: Option<PushDownInfo>,
196196
pub inverse_range_index: Option<RangeIndex>,
197197
pub deleted_segments: Vec<DeletedSegmentInfo>,
198-
pub block_meta_cache: Option<BlockMetaCache>,
198+
pub block_meta_cache: Option<SegmentBlockMetasCache>,
199199
}
200200

201201
impl FusePruner {
@@ -265,7 +265,7 @@ impl FusePruner {
265265
pruning_ctx,
266266
inverse_range_index: None,
267267
deleted_segments: vec![],
268-
block_meta_cache: CacheManager::instance().get_block_meta_cache(),
268+
block_meta_cache: CacheManager::instance().get_segment_block_metas_cache(),
269269
})
270270
}
271271

@@ -406,7 +406,7 @@ impl FusePruner {
406406
segment: &CompactSegmentInfo,
407407
populate_cache: bool,
408408
) -> Result<Arc<Vec<Arc<BlockMeta>>>> {
409-
if let Some(cache) = CacheManager::instance().get_block_meta_cache() {
409+
if let Some(cache) = CacheManager::instance().get_segment_block_metas_cache() {
410410
if let Some(metas) = cache.get(segment_path) {
411411
Ok(metas)
412412
} else {

‎src/query/storages/system/src/caches_table.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ impl SyncSystemTable for CachesTable {
7676
let segment_info_cache = cache_manager.get_table_segment_cache();
7777
let bloom_index_filter_cache = cache_manager.get_bloom_index_filter_cache();
7878
let bloom_index_meta_cache = cache_manager.get_bloom_index_meta_cache();
79+
let segment_block_metas_cache = cache_manager.get_segment_block_metas_cache();
7980
let block_meta_cache = cache_manager.get_block_meta_cache();
8081
let inverted_index_meta_cache = cache_manager.get_inverted_index_meta_cache();
8182
let inverted_index_file_cache = cache_manager.get_inverted_index_file_cache();
@@ -105,6 +106,10 @@ impl SyncSystemTable for CachesTable {
105106
Self::append_row(&bloom_index_meta_cache, &local_node, &mut columns);
106107
}
107108

109+
if let Some(segment_block_metas_cache) = segment_block_metas_cache {
110+
Self::append_row(&segment_block_metas_cache, &local_node, &mut columns);
111+
}
112+
108113
if let Some(block_meta_cache) = block_meta_cache {
109114
Self::append_row(&block_meta_cache, &local_node, &mut columns);
110115
}

‎tests/sqllogictests/suites/base/09_fuse_engine/09_0043_set_cache_cap.sql

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,22 @@ use db_09_0041;
77
# By default, memory_cache_block_meta is disabled,
88
# let's enable it by setting a non-zero capacity
99
statement ok
10-
call system$set_cache_capacity('memory_cache_block_meta', 1000);
10+
call system$set_cache_capacity('memory_cache_block_meta', 1);
1111

1212
# check cache "memory_cache_block_meta" exists
1313

1414
query II
15-
select count()>=1 from system.caches where name = 'memory_cache_block_meta' and capacity = 1000;
15+
select count()>=1 from system.caches where name = 'memory_cache_block_meta' and capacity = 1;
16+
----
17+
1
18+
19+
20+
statement ok
21+
call system$set_cache_capacity('memory_cache_segment_block_metas', 3);
22+
23+
# check cache "memory_cache_segment_block_metas" exists
24+
25+
query II
26+
select count()>=1 from system.caches where name = 'memory_cache_segment_block_metas' and capacity = 3;
1627
----
1728
1

0 commit comments

Comments
 (0)