Skip to content

Commit 0fc03e1

Browse files
committed
refactor(serve): move disk usage check to startup
1 parent b691c08 commit 0fc03e1

6 files changed

Lines changed: 139 additions & 77 deletions

File tree

quickwit/quickwit-cli/src/lib.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,8 +232,8 @@ pub fn start_actor_runtimes(
232232
/// Loads a node config located at `config_uri` with the default storage configuration.
233233
///
234234
/// When `service_override` is provided, it is applied before config validation so that
235-
/// service-dependent validation (e.g. the disk-usage check) reflects the actual runtime service
236-
/// set (e.g. `--service searcher`).
235+
/// service-dependent validation reflects the actual runtime service set (e.g. `--service
236+
/// searcher`).
237237
async fn load_node_config(
238238
config_uri: &Uri,
239239
service_override: Option<&HashSet<QuickwitService>>,

quickwit/quickwit-common/src/fs.rs

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,20 @@ pub fn get_cache_directory_path(data_dir_path: &Path) -> PathBuf {
3636
data_dir_path.join("indexer-split-cache").join("splits")
3737
}
3838

39-
/// Get the total size of the disk containing the given directory, or `None` if
40-
/// it couldn't be determined.
41-
pub fn get_disk_size(dir_path: &Path) -> Option<ByteSize> {
39+
/// Information about the disk containing a directory.
40+
#[derive(Clone, Debug, Eq, PartialEq)]
41+
pub struct DiskInfo {
42+
/// Name reported by the operating system for the backing device.
43+
pub device_name: String,
44+
/// Canonical mount point of the backing device.
45+
pub mount_point: PathBuf,
46+
/// Total capacity of the backing device.
47+
pub total_space: ByteSize,
48+
}
49+
50+
/// Returns information about the disk containing the given directory, or `None` if it couldn't be
51+
/// determined.
52+
pub fn get_disk_info(dir_path: &Path) -> Option<DiskInfo> {
4253
let disks = sysinfo::Disks::new_with_refreshed_list_specifics(
4354
DiskRefreshKind::nothing().with_storage(),
4455
);
@@ -65,7 +76,11 @@ pub fn get_disk_size(dir_path: &Path) -> Option<ByteSize> {
6576
return None;
6677
}
6778
}
68-
best_match.map(|(disk, _)| ByteSize::b(disk.total_space()))
79+
best_match.map(|(disk, mount_point)| DiskInfo {
80+
device_name: disk.name().to_string_lossy().into_owned(),
81+
mount_point,
82+
total_space: ByteSize::b(disk.total_space()),
83+
})
6984
}
7085

7186
#[cfg(test)]

quickwit/quickwit-config/src/node_config/mod.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -915,9 +915,8 @@ impl NodeConfig {
915915
}
916916

917917
/// Parses and validates a [`NodeConfig`], applying `service_override` before validation so
918-
/// service-dependent validation (e.g. the disk-usage check) uses the actual runtime service
919-
/// set rather than the pre-override default (e.g. when `--service searcher` is passed via the
920-
/// CLI).
918+
/// service-dependent validation uses the actual runtime service set rather than the
919+
/// pre-override default (e.g. when `--service searcher` is passed via the CLI).
921920
pub async fn load_with_service_override(
922921
config_format: ConfigFormat,
923922
config_content: &[u8],

quickwit/quickwit-config/src/node_config/serialize.rs

Lines changed: 3 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,7 @@ use std::str::FromStr;
1818
use std::time::Duration;
1919

2020
use anyhow::{Context, bail, ensure};
21-
use bytesize::ByteSize;
2221
use http::HeaderMap;
23-
use quickwit_common::fs::get_disk_size;
2422
use quickwit_common::net::{Host, find_private_ip, get_short_hostname};
2523
use quickwit_common::new_coolid;
2624
use quickwit_common::uri::Uri;
@@ -394,61 +392,9 @@ fn validate(node_config: &NodeConfig) -> anyhow::Result<()> {
394392
merge pipeline, the compactor service must not be enabled."
395393
);
396394
}
397-
validate_disk_usage(node_config);
398395
Ok(())
399396
}
400397

401-
/// A list of all the known disk budgets
402-
///
403-
/// External disk usage and unbounded disk usages, e.g the indexing workbench
404-
/// (indexing/) and the delete task workbench (delete_task_service/) are not included.
405-
#[derive(Default, Debug)]
406-
struct ExpectedDiskUsage {
407-
// indexer / ingester
408-
split_store_max_num_bytes: Option<ByteSize>,
409-
max_queue_disk_usage: Option<ByteSize>,
410-
// searcher
411-
split_cache: Option<ByteSize>,
412-
}
413-
414-
impl ExpectedDiskUsage {
415-
fn from_config(node_config: &NodeConfig) -> Self {
416-
let mut expected = Self::default();
417-
if node_config.is_service_enabled(QuickwitService::Indexer) {
418-
expected.max_queue_disk_usage =
419-
Some(node_config.ingest_api_config.max_queue_disk_usage);
420-
expected.split_store_max_num_bytes =
421-
Some(node_config.indexer_config.split_store_max_num_bytes);
422-
}
423-
if node_config.is_service_enabled(QuickwitService::Searcher) {
424-
expected.split_cache = node_config
425-
.searcher_config
426-
.split_cache
427-
.map(|limits| limits.max_num_bytes);
428-
}
429-
expected
430-
}
431-
432-
fn total(&self) -> ByteSize {
433-
self.split_store_max_num_bytes.unwrap_or_default()
434-
+ self.max_queue_disk_usage.unwrap_or_default()
435-
+ self.split_cache.unwrap_or_default()
436-
}
437-
}
438-
439-
fn validate_disk_usage(node_config: &NodeConfig) {
440-
if let Some(volume_size) = get_disk_size(&node_config.data_dir_path) {
441-
let expected_disk_usage = ExpectedDiskUsage::from_config(node_config);
442-
if expected_disk_usage.total() > volume_size {
443-
warn!(
444-
?volume_size,
445-
?expected_disk_usage,
446-
"data dir volume too small"
447-
);
448-
}
449-
}
450-
}
451-
452398
#[cfg(test)]
453399
impl Default for NodeConfigBuilder {
454400
fn default() -> Self {
@@ -1626,22 +1572,16 @@ mod tests {
16261572

16271573
#[tokio::test]
16281574
async fn test_service_override_applied_before_validation() {
1629-
// A searcher started with `--service searcher` must not warn about indexer split-store or
1630-
// ingest queue disk budgets. The service override has to be applied before validation so
1631-
// the disk-usage check runs against the real runtime service set, not the default set.
1575+
// The config's compactor service is invalid without `enable_standalone_compactors`. A CLI
1576+
// searcher override must replace it before service-dependent validation runs.
16321577
let data_dir = env::temp_dir().join(new_coolid("node-config"));
16331578
std::fs::create_dir_all(&data_dir).unwrap();
16341579
let config_yaml = format!(
16351580
r#"
16361581
version: 0.8
16371582
data_dir: "{}"
16381583
enabled_services:
1639-
- indexer
1640-
- searcher
1641-
indexer:
1642-
split_store_max_num_bytes: 100GiB
1643-
ingest_api:
1644-
max_queue_disk_usage: 24GiB
1584+
- compactor
16451585
"#,
16461586
data_dir.display()
16471587
);
@@ -1655,11 +1595,6 @@ mod tests {
16551595
.unwrap();
16561596

16571597
assert_eq!(config.enabled_services, services_override);
1658-
1659-
// With only the searcher enabled and no split_cache configured, the expected disk usage
1660-
// must be zero: no indexer/ingest budgets are counted.
1661-
let expected_disk_usage = ExpectedDiskUsage::from_config(&config);
1662-
assert_eq!(expected_disk_usage.total(), ByteSize::default());
16631598
std::fs::remove_dir_all(data_dir).unwrap();
16641599
}
16651600
}
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Copyright 2021-Present Datadog, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
use bytesize::ByteSize;
16+
use quickwit_common::fs::get_disk_info;
17+
use quickwit_config::NodeConfig;
18+
use quickwit_config::service::QuickwitService;
19+
use tracing::warn;
20+
21+
/// A list of all the known disk budgets.
22+
///
23+
/// External and unbounded disk usage, e.g. the indexing workbench (`indexing/`) and the delete
24+
/// task workbench (`delete_task_service/`), are not included.
25+
#[derive(Default, Debug, Eq, PartialEq)]
26+
struct ExpectedDiskUsage {
27+
// indexer / ingester
28+
split_store_max_num_bytes: Option<ByteSize>,
29+
max_queue_disk_usage: Option<ByteSize>,
30+
// searcher
31+
split_cache: Option<ByteSize>,
32+
}
33+
34+
impl ExpectedDiskUsage {
35+
fn from_config(node_config: &NodeConfig) -> Self {
36+
let mut expected = Self::default();
37+
if node_config.is_service_enabled(QuickwitService::Indexer) {
38+
expected.max_queue_disk_usage =
39+
Some(node_config.ingest_api_config.max_queue_disk_usage);
40+
expected.split_store_max_num_bytes =
41+
Some(node_config.indexer_config.split_store_max_num_bytes);
42+
}
43+
if node_config.is_service_enabled(QuickwitService::Searcher) {
44+
expected.split_cache = node_config
45+
.searcher_config
46+
.split_cache
47+
.map(|limits| limits.max_num_bytes);
48+
}
49+
expected
50+
}
51+
52+
fn total(&self) -> ByteSize {
53+
self.split_store_max_num_bytes.unwrap_or_default()
54+
+ self.max_queue_disk_usage.unwrap_or_default()
55+
+ self.split_cache.unwrap_or_default()
56+
}
57+
}
58+
59+
pub(super) fn check_data_dir_disk_usage(node_config: &NodeConfig) {
60+
let Some(disk_info) = get_disk_info(&node_config.data_dir_path) else {
61+
return;
62+
};
63+
let expected_disk_usage = ExpectedDiskUsage::from_config(node_config);
64+
if expected_disk_usage.total() > disk_info.total_space {
65+
warn!(
66+
data_dir = %node_config.data_dir_path.display(),
67+
device = %disk_info.device_name,
68+
mount_point = %disk_info.mount_point.display(),
69+
volume_size = ?disk_info.total_space,
70+
?expected_disk_usage,
71+
"data dir volume too small"
72+
);
73+
}
74+
}
75+
76+
#[cfg(test)]
77+
mod tests {
78+
use std::collections::HashSet;
79+
80+
use super::*;
81+
82+
#[test]
83+
fn test_searcher_without_split_cache_has_no_expected_disk_usage() {
84+
let mut node_config = NodeConfig::for_test();
85+
node_config.enabled_services = HashSet::from([QuickwitService::Searcher]);
86+
node_config.searcher_config.split_cache = None;
87+
88+
let expected_disk_usage = ExpectedDiskUsage::from_config(&node_config);
89+
90+
assert_eq!(expected_disk_usage, ExpectedDiskUsage::default());
91+
}
92+
93+
#[test]
94+
fn test_expected_disk_usage_aggregates_enabled_service_budgets() {
95+
let mut node_config = NodeConfig::for_test();
96+
node_config.enabled_services =
97+
HashSet::from([QuickwitService::Indexer, QuickwitService::Searcher]);
98+
99+
let expected_disk_usage = ExpectedDiskUsage::from_config(&node_config);
100+
let expected_total = node_config.indexer_config.split_store_max_num_bytes
101+
+ node_config.ingest_api_config.max_queue_disk_usage
102+
+ node_config
103+
.searcher_config
104+
.split_cache
105+
.map(|limits| limits.max_num_bytes)
106+
.unwrap_or_default();
107+
108+
assert_eq!(expected_disk_usage.total(), expected_total);
109+
}
110+
}

quickwit/quickwit-serve/src/lib.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ mod datafusion_api;
2121
mod decompression;
2222
mod delete_task_api;
2323
mod developer_api;
24+
mod disk_usage;
2425
mod elasticsearch_api;
2526
mod format;
2627
mod grpc;
@@ -564,6 +565,8 @@ pub async fn serve_quickwit(
564565
shutdown_signal: BoxFutureInfaillible<()>,
565566
env_filter_reload_fn: EnvFilterReloadFn,
566567
) -> anyhow::Result<HashMap<String, ActorExitStatus>> {
568+
disk_usage::check_data_dir_disk_usage(&node_config);
569+
567570
let cluster = start_cluster_service(&node_config)
568571
.await
569572
.context("failed to start cluster service")?;

0 commit comments

Comments
 (0)