From f19516e648e49ce541ef490bd9714ad8e450bca9 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Wed, 2 Aug 2023 14:48:59 +0300 Subject: [PATCH 01/19] update spaceInfo with disks metrics (WIP) --- bob/src/api/mod.rs | 61 ++++++++++----- bob/src/hw_metrics_collector.rs | 128 +++++++++++++++++++++----------- 2 files changed, 127 insertions(+), 62 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index d46c7af6b..dd69fd822 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -19,10 +19,10 @@ use bob_access::{Authenticator, CredentialsHolder}; use bob_backend::pearl::{Group as PearlGroup, Holder, NoopHooks}; use bob_common::{ configs::node::TLSConfig, + core_types::{DiskName, NodeDisk, VDisk as DataVDisk}, data::{BobData, BobKey, BobMeta, BOB_KEY_SIZE}, - core_types::{VDisk as DataVDisk, NodeDisk}, - operation_options::{BobPutOptions, BobGetOptions, BobDeleteOptions}, error::Error as BobError, + operation_options::{BobDeleteOptions, BobGetOptions, BobPutOptions}, }; use bytes::Bytes; use futures::{future::BoxFuture, stream::FuturesUnordered, FutureExt, StreamExt}; @@ -145,12 +145,18 @@ pub(crate) struct NodeConfiguration { } #[derive(Debug, Serialize)] -pub(crate) struct SpaceInfo { +struct Space { total_disk_space_bytes: u64, free_disk_space_bytes: u64, used_disk_space_bytes: u64, occupied_disk_space_bytes: u64, - occupied_disk_space_by_disk: HashMap, +} + +#[derive(Debug, Serialize)] +pub(crate) struct SpaceInfo { + #[serde(flatten)] + space: Space, + disk_space_by_disk: HashMap, } async fn tls_server(tls_config: &TLSConfig, addr: SocketAddr) -> AxumServer { @@ -313,31 +319,50 @@ async fn status(bob: Extension>) -> Json { async fn get_space_info( bob: Extension>, ) -> Result, StatusExt> { - let DiskSpaceMetrics { - total_space, - used_space, - free_space, - } = bob.grinder().hw_counter().update_space_metrics(); + let mut disk_metrics = bob.grinder().hw_counter().update_space_metrics(); + let summed_metrics: DiskSpaceMetrics = disk_metrics.values().sum(); let backend = bob.grinder().backend().inner(); let (dcs, adc) = backend .disk_controllers() .ok_or_else(not_acceptable_backend)?; let mut map = HashMap::new(); - for dc in dcs.iter() { - map.insert(dc.disk().name().to_string(), dc.disk_used().await); + for dc in dcs { + map.insert(dc.disk().name(), dc.disk_used().await); } let adc_space = adc.disk_used().await; - map.entry(adc.disk().name().to_string()) - .and_modify(|s| *s = *s + adc_space) + map.entry(adc.disk().name()) + .and_modify(|s| *s += adc_space) .or_insert(adc_space); Ok(Json(SpaceInfo { - total_disk_space_bytes: total_space, - used_disk_space_bytes: used_space, - free_disk_space_bytes: free_space, - occupied_disk_space_bytes: map.values().sum(), - occupied_disk_space_by_disk: map, + space: Space { + total_disk_space_bytes: summed_metrics.total_space, + used_disk_space_bytes: summed_metrics.used_space, + free_disk_space_bytes: summed_metrics.free_space, + occupied_disk_space_bytes: map.values().sum(), + }, + disk_space_by_disk: map + .iter() + .map(|(&name, &occup_space)| { + (name.clone(), { + let DiskSpaceMetrics { + total_space, + used_space, + free_space, + } = disk_metrics + .remove_entry(name) + .map(|(_, space)| space) + .unwrap_or_default(); + Space { + total_disk_space_bytes: total_space, + free_disk_space_bytes: free_space, + used_disk_space_bytes: used_space, + occupied_disk_space_bytes: occup_space, + } + }) + }) + .collect(), })) } diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 8e4c11cb4..e9a154bec 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -1,21 +1,21 @@ use crate::prelude::*; use bob_common::core_types::DiskName; use bob_common::metrics::{ - BOB_RAM, BOB_VIRTUAL_RAM, BOB_CPU_LOAD, DESCRIPTORS_AMOUNT, CPU_IOWAIT, - TOTAL_RAM, AVAILABLE_RAM, USED_RAM, USED_SWAP, - TOTAL_SPACE, FREE_SPACE, USED_SPACE, HW_DISKS_FOLDER + AVAILABLE_RAM, BOB_CPU_LOAD, BOB_RAM, BOB_VIRTUAL_RAM, CPU_IOWAIT, DESCRIPTORS_AMOUNT, + FREE_SPACE, HW_DISKS_FOLDER, TOTAL_RAM, TOTAL_SPACE, USED_RAM, USED_SPACE, USED_SWAP, }; use libc::statvfs; +use std::fs::read_to_string; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::process::{self, Command}; -use std::fs::read_to_string; use sysinfo::{DiskExt, ProcessExt, ProcessRefreshKind, RefreshKind, System, SystemExt}; const DESCRS_DIR: &str = "/proc/self/fd/"; const CPU_STAT_FILE: &str = "/proc/stat"; const DISK_STAT_FILE: &str = "/proc/diskstats"; +#[derive(Debug, Serialize, Default, Clone)] pub(crate) struct DiskSpaceMetrics { pub(crate) total_space: u64, pub(crate) used_space: u64, @@ -63,7 +63,7 @@ impl HWMetricsCollector { tokio::spawn(Self::task(self.interval_time, self.disks.clone())); } - pub(crate) fn update_space_metrics(&self) -> DiskSpaceMetrics { + pub(crate) fn update_space_metrics(&self) -> HashMap { Self::update_space_metrics_from_disks(&self.disks) } @@ -91,10 +91,10 @@ impl HWMetricsCollector { match cpu_s_c.iowait() { Ok(iowait) => { gauge!(CPU_IOWAIT, iowait); - }, + } Err(CommandError::Primary(e)) => { warn!("Error while collecting cpu iowait: {}", e); - }, + } Err(CommandError::Unavailable) => (), } @@ -112,7 +112,10 @@ impl HWMetricsCollector { let available_mem = sys.available_memory(); let used_mem = total_mem - available_mem; let used_swap = sys.used_swap(); - debug!("used mem in bytes: {} | available mem in bytes: {} | used swap: {}", used_mem, available_mem, used_swap); + debug!( + "used mem in bytes: {} | available mem in bytes: {} | used swap: {}", + used_mem, available_mem, used_swap + ); gauge!(USED_RAM, used_mem as f64); gauge!(AVAILABLE_RAM, available_mem as f64); gauge!(USED_SWAP, used_swap as f64); @@ -124,11 +127,14 @@ impl HWMetricsCollector { } } - fn update_space_metrics_from_disks(disks: &HashMap) -> DiskSpaceMetrics { + fn update_space_metrics_from_disks( + disks: &HashMap, + ) -> HashMap { let disks_metrics = Self::space(disks); - gauge!(TOTAL_SPACE, bytes_to_mb(disks_metrics.total_space) as f64); - gauge!(USED_SPACE, bytes_to_mb(disks_metrics.used_space) as f64); - gauge!(FREE_SPACE, bytes_to_mb(disks_metrics.free_space) as f64); + let summed_space: DiskSpaceMetrics = disks_metrics.values().sum(); + gauge!(TOTAL_SPACE, bytes_to_mb(summed_space.total_space) as f64); + gauge!(USED_SPACE, bytes_to_mb(summed_space.used_space) as f64); + gauge!(FREE_SPACE, bytes_to_mb(summed_space.free_space) as f64); disks_metrics } @@ -157,12 +163,9 @@ impl HWMetricsCollector { // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big, // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one - fn space(disks: &HashMap) -> DiskSpaceMetrics { - let mut total = 0; - let mut used = 0; - let mut free = 0; - - for mount_point in disks.keys() { + fn space(disks: &HashMap) -> HashMap { + let mut res = HashMap::new(); + for (mount_point, disk_name) in disks { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); if let Some(stat) = stat { @@ -170,17 +173,18 @@ impl HWMetricsCollector { let blocks = stat.f_blocks as u64; let bavail = stat.f_bavail as u64; let bfree = stat.f_bfree as u64; - total += bsize * blocks; - free += bsize * bavail; - used += (blocks - bfree) * bsize; + res.insert( + disk_name.clone(), + DiskSpaceMetrics { + total_space: bsize * blocks, + used_space: bsize * bavail, + free_space: (blocks - bfree) * bsize, + }, + ); } } - DiskSpaceMetrics { - total_space: total, - used_space: used, - free_space: free, - } + res } } @@ -194,9 +198,7 @@ struct CPUStatCollector { impl CPUStatCollector { fn new() -> CPUStatCollector { - CPUStatCollector { - procfs_avl: true - } + CPUStatCollector { procfs_avl: true } } fn stat_cpu_line() -> Result { @@ -229,7 +231,7 @@ impl CPUStatCollector { if i == CPU_IOWAIT_COLUMN { f_iowait = val; } - }, + } Err(_) => { let msg = format!("Can't parse {}", CPU_STAT_FILE); err = Some(msg); @@ -244,7 +246,7 @@ impl CPUStatCollector { let msg = format!("CPU stat format in {} changed", CPU_STAT_FILE); err = Some(msg); } - }, + } Err(e) => { err = Some(e); } @@ -261,13 +263,11 @@ struct DiffContainer { } impl DiffContainer -where T: std::ops::Sub + - Copy, +where + T: std::ops::Sub + Copy, { fn new() -> Self { - Self { - last: None, - } + Self { last: None } } fn diff(&mut self, new: T) -> Option { @@ -307,7 +307,7 @@ struct DiskStatsContainer { stats: DiffContainer, } -impl DiskStatsContainer { +impl DiskStatsContainer { fn new(prefix: String) -> Self { Self { prefix, @@ -355,7 +355,7 @@ impl DiskStatCollector { parts[7].parse::(), ) { new_ds.reads = r_ios; - new_ds.writes = w_ios; + new_ds.writes = w_ios; if new_ds.extended { // time spend doing i/o operations is in 12th column if let Ok(io_time) = parts[12].parse::() { @@ -385,7 +385,7 @@ impl DiskStatCollector { // compare device name from 2nd column with disk device names if let Some(ds) = self.disk_metric_data.get_mut(lsp[2]) { let new_ds = Self::parse_stat_line(lsp)?; - + if let Some(diff) = ds.stats.diff(new_ds) { let iops = (diff.reads + diff.writes) as f64 / elapsed; let gauge_name = format!("{}_iops", ds.prefix); @@ -401,7 +401,10 @@ impl DiskStatCollector { } } else { self.procfs_avl = false; - let msg = format!("Not enough diskstat info in {} for metrics calculation", DISK_STAT_FILE); + let msg = format!( + "Not enough diskstat info in {} for metrics calculation", + DISK_STAT_FILE + ); return Err(CommandError::Primary(msg)); } } @@ -490,7 +493,10 @@ impl DescrCounter { let pid_arg = process::id().to_string(); let cmd_lsof = Command::new("lsof") - .args(["-a", "-p", &pid_arg, "-d", "^mem", "-d", "^cwd", "-d", "^rtd", "-d", "^txt", "-d", "^DEL"]) + .args([ + "-a", "-p", &pid_arg, "-d", "^mem", "-d", "^cwd", "-d", "^rtd", "-d", "^txt", "-d", + "^DEL", + ]) .stdout(std::process::Stdio::piped()) .spawn(); match cmd_lsof { @@ -507,17 +513,17 @@ impl DescrCounter { debug!("failed to parse lsof result: {}", e); } } - }, + } Err(e) => { debug!("can't use lsof, wc error (fs /proc will be used): {}", e); } } - }, + } None => { debug!("lsof has no stdout (fs /proc will be used)"); } } - }, + } Err(e) => { debug!("can't use lsof (fs /proc will be used): {}", e); } @@ -589,3 +595,37 @@ fn parse_command_output(command: &mut Command) -> Result { } } } + +/// Traits Impls +/// +use std::iter::Sum; +use std::ops::{Add, AddAssign}; + +impl AddAssign for DiskSpaceMetrics { + fn add_assign(&mut self, rhs: Self) { + self.used_space += rhs.used_space; + self.free_space += rhs.free_space; + self.total_space += rhs.total_space; + } +} +impl Add for DiskSpaceMetrics { + type Output = Self; + + fn add(mut self, rhs: Self) -> Self::Output { + self += rhs; + self + } +} + +impl<'a> Sum<&'a Self> for DiskSpaceMetrics { + fn sum>(iter: I) -> Self { + iter.fold( + Self { + total_space: 0, + used_space: 0, + free_space: 0, + }, + |a, b| a + b.clone(), + ) + } +} From 2b35c1425f62e6b4ac91698de52711c09f6ca1cb Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Wed, 2 Aug 2023 23:15:10 +0300 Subject: [PATCH 02/19] make it more conscious --- bob/src/api/mod.rs | 78 ++++++++++++++++++++------------- bob/src/hw_metrics_collector.rs | 17 ++++--- 2 files changed, 56 insertions(+), 39 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index dd69fd822..8b551d26b 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -320,49 +320,67 @@ async fn get_space_info( bob: Extension>, ) -> Result, StatusExt> { let mut disk_metrics = bob.grinder().hw_counter().update_space_metrics(); - let summed_metrics: DiskSpaceMetrics = disk_metrics.values().sum(); + let DiskSpaceMetrics { + total_space, + used_space, + free_space, + } = disk_metrics.values().sum(); let backend = bob.grinder().backend().inner(); let (dcs, adc) = backend .disk_controllers() .ok_or_else(not_acceptable_backend)?; let mut map = HashMap::new(); + let mut total_occupied = 0; for dc in dcs { - map.insert(dc.disk().name(), dc.disk_used().await); + let occupied_space = dc.disk_used().await; + total_occupied += occupied_space; + let DiskSpaceMetrics { + total_space, + used_space, + free_space, + } = disk_metrics + .remove_entry(dc.disk().name()) + .map(|(_, space)| space) + .unwrap_or_default(); + map.insert( + dc.disk().name().clone(), + Space { + total_disk_space_bytes: total_space, + free_disk_space_bytes: free_space, + used_disk_space_bytes: used_space, + occupied_disk_space_bytes: occupied_space, + }, + ); } let adc_space = adc.disk_used().await; - map.entry(adc.disk().name()) - .and_modify(|s| *s += adc_space) - .or_insert(adc_space); + map.entry(adc.disk().name().clone()) + .and_modify(|s| s.occupied_disk_space_bytes += adc_space) + .or_insert({ + let DiskSpaceMetrics { + total_space, + used_space, + free_space, + } = disk_metrics + .remove_entry(adc.disk().name()) + .map(|(_, space)| space) + .unwrap_or_default(); + Space { + total_disk_space_bytes: total_space, + free_disk_space_bytes: free_space, + used_disk_space_bytes: used_space, + occupied_disk_space_bytes: adc_space, + } + }); Ok(Json(SpaceInfo { space: Space { - total_disk_space_bytes: summed_metrics.total_space, - used_disk_space_bytes: summed_metrics.used_space, - free_disk_space_bytes: summed_metrics.free_space, - occupied_disk_space_bytes: map.values().sum(), + total_disk_space_bytes: total_space, + used_disk_space_bytes: used_space, + free_disk_space_bytes: free_space, + occupied_disk_space_bytes: total_occupied, }, - disk_space_by_disk: map - .iter() - .map(|(&name, &occup_space)| { - (name.clone(), { - let DiskSpaceMetrics { - total_space, - used_space, - free_space, - } = disk_metrics - .remove_entry(name) - .map(|(_, space)| space) - .unwrap_or_default(); - Space { - total_disk_space_bytes: total_space, - free_disk_space_bytes: free_space, - used_disk_space_bytes: used_space, - occupied_disk_space_bytes: occup_space, - } - }) - }) - .collect(), + disk_space_by_disk: map, })) } diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index e9a154bec..5eabbe8f6 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -6,6 +6,8 @@ use bob_common::metrics::{ }; use libc::statvfs; use std::fs::read_to_string; +use std::iter::Sum; +use std::ops::{Add, AddAssign}; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::process::{self, Command}; @@ -596,22 +598,19 @@ fn parse_command_output(command: &mut Command) -> Result { } } -/// Traits Impls -/// -use std::iter::Sum; -use std::ops::{Add, AddAssign}; +// Std Traits Impls -impl AddAssign for DiskSpaceMetrics { - fn add_assign(&mut self, rhs: Self) { +impl<'a> AddAssign<&'a Self> for DiskSpaceMetrics { + fn add_assign(&mut self, rhs: &Self) { self.used_space += rhs.used_space; self.free_space += rhs.free_space; self.total_space += rhs.total_space; } } -impl Add for DiskSpaceMetrics { +impl<'a> Add<&'a Self> for DiskSpaceMetrics { type Output = Self; - fn add(mut self, rhs: Self) -> Self::Output { + fn add(mut self, rhs: &Self) -> Self::Output { self += rhs; self } @@ -625,7 +624,7 @@ impl<'a> Sum<&'a Self> for DiskSpaceMetrics { used_space: 0, free_space: 0, }, - |a, b| a + b.clone(), + |a, b| a + b, ) } } From 34141fc8ef5cc84a9ec01b17fdb4504b4f986163 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Thu, 3 Aug 2023 01:29:26 +0300 Subject: [PATCH 03/19] Update Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42f89a8de..3dc60702e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Bob versions changelog - Added mimalloc allocator for musl target (#688) - Added jemalloc-profile for memory profiling (#797) - Proper support for GetSource::ALL requests (#723) +- Added detailed information about total, used and free space on every disk (#823) #### Changed - BobClient clone overhead reduced (#774) From b43ee7cff5429c3329b7b95de7b520b8a513fbce Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Thu, 3 Aug 2023 01:39:45 +0300 Subject: [PATCH 04/19] naming + clippy fixes --- bob/src/api/mod.rs | 13 +++++++------ bob/src/hw_metrics_collector.rs | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 8b551d26b..6c46b976e 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -155,7 +155,7 @@ struct Space { #[derive(Debug, Serialize)] pub(crate) struct SpaceInfo { #[serde(flatten)] - space: Space, + total_space: Space, disk_space_by_disk: HashMap, } @@ -330,7 +330,7 @@ async fn get_space_info( let (dcs, adc) = backend .disk_controllers() .ok_or_else(not_acceptable_backend)?; - let mut map = HashMap::new(); + let mut disk_space_by_disk = HashMap::new(); let mut total_occupied = 0; for dc in dcs { let occupied_space = dc.disk_used().await; @@ -343,7 +343,7 @@ async fn get_space_info( .remove_entry(dc.disk().name()) .map(|(_, space)| space) .unwrap_or_default(); - map.insert( + disk_space_by_disk.insert( dc.disk().name().clone(), Space { total_disk_space_bytes: total_space, @@ -354,7 +354,8 @@ async fn get_space_info( ); } let adc_space = adc.disk_used().await; - map.entry(adc.disk().name().clone()) + disk_space_by_disk + .entry(adc.disk().name().clone()) .and_modify(|s| s.occupied_disk_space_bytes += adc_space) .or_insert({ let DiskSpaceMetrics { @@ -374,13 +375,13 @@ async fn get_space_info( }); Ok(Json(SpaceInfo { - space: Space { + total_space: Space { total_disk_space_bytes: total_space, used_disk_space_bytes: used_space, free_disk_space_bytes: free_space, occupied_disk_space_bytes: total_occupied, }, - disk_space_by_disk: map, + disk_space_by_disk, })) } diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 5eabbe8f6..3ec264689 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -171,10 +171,10 @@ impl HWMetricsCollector { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); if let Some(stat) = stat { - let bsize = stat.f_bsize as u64; - let blocks = stat.f_blocks as u64; - let bavail = stat.f_bavail as u64; - let bfree = stat.f_bfree as u64; + let bsize = stat.f_bsize; + let blocks = stat.f_blocks; + let bavail = stat.f_bavail; + let bfree = stat.f_bfree; res.insert( disk_name.clone(), DiskSpaceMetrics { From 5ec7e17912fad0615959e12a2243dd95e1fda76d Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Tue, 15 Aug 2023 15:52:16 +0300 Subject: [PATCH 05/19] change bsize to frsize --- bob/src/hw_metrics_collector.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 3ec264689..cc54c7014 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -171,16 +171,16 @@ impl HWMetricsCollector { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); if let Some(stat) = stat { - let bsize = stat.f_bsize; + let frsize = stat.f_frsize; let blocks = stat.f_blocks; let bavail = stat.f_bavail; let bfree = stat.f_bfree; res.insert( disk_name.clone(), DiskSpaceMetrics { - total_space: bsize * blocks, - used_space: bsize * bavail, - free_space: (blocks - bfree) * bsize, + total_space: frsize * blocks, + used_space: frsize * bavail, + free_space: (blocks - bfree) * frsize, }, ); } From 188fbf5bdc89cfcf5f8be66556a42be95899ad06 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Sun, 20 Aug 2023 15:41:26 +0300 Subject: [PATCH 06/19] review fixes --- bob/src/api/mod.rs | 117 +++++++++++++++----------------- bob/src/hw_metrics_collector.rs | 93 +++++++++++++------------ 2 files changed, 106 insertions(+), 104 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 6c46b976e..897e67d37 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -19,10 +19,10 @@ use bob_access::{Authenticator, CredentialsHolder}; use bob_backend::pearl::{Group as PearlGroup, Holder, NoopHooks}; use bob_common::{ configs::node::TLSConfig, - core_types::{DiskName, NodeDisk, VDisk as DataVDisk}, data::{BobData, BobKey, BobMeta, BOB_KEY_SIZE}, + core_types::{VDisk as DataVDisk, NodeDisk, DiskName}, + operation_options::{BobPutOptions, BobGetOptions, BobDeleteOptions}, error::Error as BobError, - operation_options::{BobDeleteOptions, BobGetOptions, BobPutOptions}, }; use bytes::Bytes; use futures::{future::BoxFuture, stream::FuturesUnordered, FutureExt, StreamExt}; @@ -149,14 +149,18 @@ struct Space { total_disk_space_bytes: u64, free_disk_space_bytes: u64, used_disk_space_bytes: u64, - occupied_disk_space_bytes: u64, } #[derive(Debug, Serialize)] pub(crate) struct SpaceInfo { #[serde(flatten)] total_space: Space, - disk_space_by_disk: HashMap, + + /// Key - Bob disk name + occupied_disk_space_by_disk: HashMap, + + /// Key - mount point + disk_space_by_disk: HashMap, } async fn tls_server(tls_config: &TLSConfig, addr: SocketAddr) -> AxumServer { @@ -319,72 +323,54 @@ async fn status(bob: Extension>) -> Json { async fn get_space_info( bob: Extension>, ) -> Result, StatusExt> { - let mut disk_metrics = bob.grinder().hw_counter().update_space_metrics(); - let DiskSpaceMetrics { - total_space, - used_space, - free_space, - } = disk_metrics.values().sum(); - + let disk_metrics = bob.grinder().hw_counter().update_space_metrics(); + let total_disks_metrics: DiskSpaceMetrics = disk_metrics.values().sum(); let backend = bob.grinder().backend().inner(); let (dcs, adc) = backend .disk_controllers() .ok_or_else(not_acceptable_backend)?; - let mut disk_space_by_disk = HashMap::new(); - let mut total_occupied = 0; - for dc in dcs { - let occupied_space = dc.disk_used().await; - total_occupied += occupied_space; - let DiskSpaceMetrics { - total_space, - used_space, - free_space, - } = disk_metrics - .remove_entry(dc.disk().name()) - .map(|(_, space)| space) - .unwrap_or_default(); - disk_space_by_disk.insert( - dc.disk().name().clone(), - Space { - total_disk_space_bytes: total_space, - free_disk_space_bytes: free_space, - used_disk_space_bytes: used_space, - occupied_disk_space_bytes: occupied_space, - }, - ); - } + + let mut occupied_disk_space_by_disk: HashMap = futures::future::join_all( + dcs.iter().map(|dc| async { ( dc.disk().name().clone(), dc.disk_used().await ) }) + ).await.into_iter().collect(); + + let disk_space_by_disk = + dcs + .iter() + .map(|dc| { + let mount_point = PathBuf::from(dc.disk().path()); + let space = get_space(&disk_metrics, &mount_point); + (mount_point, space) + }).collect(); + let adc_space = adc.disk_used().await; - disk_space_by_disk - .entry(adc.disk().name().clone()) - .and_modify(|s| s.occupied_disk_space_bytes += adc_space) - .or_insert({ - let DiskSpaceMetrics { - total_space, - used_space, - free_space, - } = disk_metrics - .remove_entry(adc.disk().name()) - .map(|(_, space)| space) - .unwrap_or_default(); - Space { - total_disk_space_bytes: total_space, - free_disk_space_bytes: free_space, - used_disk_space_bytes: used_space, - occupied_disk_space_bytes: adc_space, - } - }); + occupied_disk_space_by_disk.entry(adc.disk().name().clone()) + .and_modify(|s| *s += adc_space) + .or_insert(adc_space); Ok(Json(SpaceInfo { total_space: Space { - total_disk_space_bytes: total_space, - used_disk_space_bytes: used_space, - free_disk_space_bytes: free_space, - occupied_disk_space_bytes: total_occupied, + total_disk_space_bytes: total_disks_metrics.total_space, + used_disk_space_bytes: total_disks_metrics.used_space, + free_disk_space_bytes: total_disks_metrics.free_space, }, disk_space_by_disk, + occupied_disk_space_by_disk, })) } +fn get_space(metrics: &HashMap<&PathBuf, DiskSpaceMetrics>, mount_point: &PathBuf) -> Space { + metrics.get(&mount_point).map_or(Space { + total_disk_space_bytes: 0, + free_disk_space_bytes: 0, + used_disk_space_bytes: 0, + }, |metrics| Space { + total_disk_space_bytes: metrics.total_space, + free_disk_space_bytes: metrics.free_space, + used_disk_space_bytes: metrics.used_space, + }) +} + fn not_acceptable_backend() -> StatusExt { let status = StatusExt::new( StatusCode::NOT_ACCEPTABLE, @@ -410,13 +396,22 @@ async fn find_group( .iter() .find(|dc| dc.vdisks().iter().any(|&vd| vd == vdisk_id)) .ok_or_else(|| { - let err = format!("Disk Controller with vdisk #{} not found", vdisk_id); + let dcs = dcs.iter() + .map(|dc| format!("DC: {}, vdisks: {}", + dc.disk().name(), + dc.vdisks().iter().map(|v| format!("#{}", v)).collect::>().join(", "))) + .collect::>() + .join(", "); + let err = format!("Disk Controller with vdisk #{} not found, available dcs: {}", vdisk_id, dcs); warn!("{}", err); StatusExt::new(StatusCode::NOT_FOUND, false, err) })?; - needed_dc.vdisk_group(vdisk_id).await.map_err(|_| { - let err = format!("Disk Controller with vdisk #{} not found", vdisk_id); - warn!("{}", err); + needed_dc.vdisk_group(vdisk_id).await.map_err(|e| { + let err = format!("VDiskGroup #{} is missing on disk controller '{}', available vdisks: {}", + vdisk_id, + needed_dc.disk().name(), + needed_dc.vdisks().iter().map(|v| format!("#{}", v)).collect::>().join(", ")); + warn!("{}. Error: {:?}", err, e); StatusExt::new(StatusCode::NOT_FOUND, false, err) }) } diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index cc54c7014..0332b92ee 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -1,24 +1,26 @@ use crate::prelude::*; use bob_common::core_types::DiskName; use bob_common::metrics::{ - AVAILABLE_RAM, BOB_CPU_LOAD, BOB_RAM, BOB_VIRTUAL_RAM, CPU_IOWAIT, DESCRIPTORS_AMOUNT, - FREE_SPACE, HW_DISKS_FOLDER, TOTAL_RAM, TOTAL_SPACE, USED_RAM, USED_SPACE, USED_SWAP, + BOB_RAM, BOB_VIRTUAL_RAM, BOB_CPU_LOAD, DESCRIPTORS_AMOUNT, CPU_IOWAIT, + TOTAL_RAM, AVAILABLE_RAM, USED_RAM, USED_SWAP, + TOTAL_SPACE, FREE_SPACE, USED_SPACE, HW_DISKS_FOLDER }; use libc::statvfs; -use std::fs::read_to_string; +use std::ops::{AddAssign, Add}; use std::iter::Sum; -use std::ops::{Add, AddAssign}; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::process::{self, Command}; +use std::fs::read_to_string; use sysinfo::{DiskExt, ProcessExt, ProcessRefreshKind, RefreshKind, System, SystemExt}; const DESCRS_DIR: &str = "/proc/self/fd/"; const CPU_STAT_FILE: &str = "/proc/stat"; const DISK_STAT_FILE: &str = "/proc/diskstats"; -#[derive(Debug, Serialize, Default, Clone)] +#[derive(Debug, Clone)] pub(crate) struct DiskSpaceMetrics { + pub(crate) disk_name: DiskName, pub(crate) total_space: u64, pub(crate) used_space: u64, pub(crate) free_space: u64, @@ -65,7 +67,7 @@ impl HWMetricsCollector { tokio::spawn(Self::task(self.interval_time, self.disks.clone())); } - pub(crate) fn update_space_metrics(&self) -> HashMap { + pub(crate) fn update_space_metrics(&self) -> HashMap<&PathBuf, DiskSpaceMetrics> { Self::update_space_metrics_from_disks(&self.disks) } @@ -93,10 +95,10 @@ impl HWMetricsCollector { match cpu_s_c.iowait() { Ok(iowait) => { gauge!(CPU_IOWAIT, iowait); - } + }, Err(CommandError::Primary(e)) => { warn!("Error while collecting cpu iowait: {}", e); - } + }, Err(CommandError::Unavailable) => (), } @@ -114,10 +116,7 @@ impl HWMetricsCollector { let available_mem = sys.available_memory(); let used_mem = total_mem - available_mem; let used_swap = sys.used_swap(); - debug!( - "used mem in bytes: {} | available mem in bytes: {} | used swap: {}", - used_mem, available_mem, used_swap - ); + debug!("used mem in bytes: {} | available mem in bytes: {} | used swap: {}", used_mem, available_mem, used_swap); gauge!(USED_RAM, used_mem as f64); gauge!(AVAILABLE_RAM, available_mem as f64); gauge!(USED_SWAP, used_swap as f64); @@ -131,7 +130,7 @@ impl HWMetricsCollector { fn update_space_metrics_from_disks( disks: &HashMap, - ) -> HashMap { + ) -> HashMap<&PathBuf, DiskSpaceMetrics> { let disks_metrics = Self::space(disks); let summed_space: DiskSpaceMetrics = disks_metrics.values().sum(); gauge!(TOTAL_SPACE, bytes_to_mb(summed_space.total_space) as f64); @@ -165,22 +164,23 @@ impl HWMetricsCollector { // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big, // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one - fn space(disks: &HashMap) -> HashMap { + fn space(disks: &HashMap) -> HashMap<&PathBuf, DiskSpaceMetrics> { let mut res = HashMap::new(); for (mount_point, disk_name) in disks { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); if let Some(stat) = stat { - let frsize = stat.f_frsize; + let bsize = stat.f_bsize; let blocks = stat.f_blocks; let bavail = stat.f_bavail; let bfree = stat.f_bfree; res.insert( - disk_name.clone(), + mount_point, DiskSpaceMetrics { - total_space: frsize * blocks, - used_space: frsize * bavail, - free_space: (blocks - bfree) * frsize, + total_space: bsize * blocks, + used_space: bsize * bavail, + free_space: (blocks - bfree) * bsize, + disk_name: disk_name.clone(), }, ); } @@ -200,7 +200,9 @@ struct CPUStatCollector { impl CPUStatCollector { fn new() -> CPUStatCollector { - CPUStatCollector { procfs_avl: true } + CPUStatCollector { + procfs_avl: true + } } fn stat_cpu_line() -> Result { @@ -233,7 +235,7 @@ impl CPUStatCollector { if i == CPU_IOWAIT_COLUMN { f_iowait = val; } - } + }, Err(_) => { let msg = format!("Can't parse {}", CPU_STAT_FILE); err = Some(msg); @@ -248,7 +250,7 @@ impl CPUStatCollector { let msg = format!("CPU stat format in {} changed", CPU_STAT_FILE); err = Some(msg); } - } + }, Err(e) => { err = Some(e); } @@ -265,11 +267,13 @@ struct DiffContainer { } impl DiffContainer -where - T: std::ops::Sub + Copy, +where T: std::ops::Sub + + Copy, { fn new() -> Self { - Self { last: None } + Self { + last: None, + } } fn diff(&mut self, new: T) -> Option { @@ -309,7 +313,7 @@ struct DiskStatsContainer { stats: DiffContainer, } -impl DiskStatsContainer { +impl DiskStatsContainer { fn new(prefix: String) -> Self { Self { prefix, @@ -357,7 +361,7 @@ impl DiskStatCollector { parts[7].parse::(), ) { new_ds.reads = r_ios; - new_ds.writes = w_ios; + new_ds.writes = w_ios; if new_ds.extended { // time spend doing i/o operations is in 12th column if let Ok(io_time) = parts[12].parse::() { @@ -387,7 +391,7 @@ impl DiskStatCollector { // compare device name from 2nd column with disk device names if let Some(ds) = self.disk_metric_data.get_mut(lsp[2]) { let new_ds = Self::parse_stat_line(lsp)?; - + if let Some(diff) = ds.stats.diff(new_ds) { let iops = (diff.reads + diff.writes) as f64 / elapsed; let gauge_name = format!("{}_iops", ds.prefix); @@ -403,10 +407,7 @@ impl DiskStatCollector { } } else { self.procfs_avl = false; - let msg = format!( - "Not enough diskstat info in {} for metrics calculation", - DISK_STAT_FILE - ); + let msg = format!("Not enough diskstat info in {} for metrics calculation", DISK_STAT_FILE); return Err(CommandError::Primary(msg)); } } @@ -495,10 +496,7 @@ impl DescrCounter { let pid_arg = process::id().to_string(); let cmd_lsof = Command::new("lsof") - .args([ - "-a", "-p", &pid_arg, "-d", "^mem", "-d", "^cwd", "-d", "^rtd", "-d", "^txt", "-d", - "^DEL", - ]) + .args(["-a", "-p", &pid_arg, "-d", "^mem", "-d", "^cwd", "-d", "^rtd", "-d", "^txt", "-d", "^DEL"]) .stdout(std::process::Stdio::piped()) .spawn(); match cmd_lsof { @@ -515,17 +513,17 @@ impl DescrCounter { debug!("failed to parse lsof result: {}", e); } } - } + }, Err(e) => { debug!("can't use lsof, wc error (fs /proc will be used): {}", e); } } - } + }, None => { debug!("lsof has no stdout (fs /proc will be used)"); } } - } + }, Err(e) => { debug!("can't use lsof (fs /proc will be used): {}", e); } @@ -617,13 +615,22 @@ impl<'a> Add<&'a Self> for DiskSpaceMetrics { } impl<'a> Sum<&'a Self> for DiskSpaceMetrics { - fn sum>(iter: I) -> Self { - iter.fold( - Self { + /// Summarize [`DiskSpaceMetrics`] over an iterator. + /// NOTE: the `disk_name` field will chosen from the first appeared disk in iterator if there + /// is any. Otherwise 'None' will be passed + fn sum>(mut iter: I) -> Self { + let init = if let Some(metrics) = iter.next() { + metrics.clone() + } else { + return Self { + disk_name: "None".into(), total_space: 0, used_space: 0, free_space: 0, - }, + } + }; + iter.fold( + init, |a, b| a + b, ) } From 50356376cb7bd7dc054034cec4e0896cd745dfac Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Sun, 20 Aug 2023 15:53:23 +0300 Subject: [PATCH 07/19] return occupied_disk_space_by_disk to SpaceInfo --- bob/src/api/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 897e67d37..f1b3d0893 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -156,6 +156,9 @@ pub(crate) struct SpaceInfo { #[serde(flatten)] total_space: Space, + /// Total occupied disk space by Bob + occupied_disk_space_bytes: u64, + /// Key - Bob disk name occupied_disk_space_by_disk: HashMap, @@ -354,8 +357,9 @@ async fn get_space_info( used_disk_space_bytes: total_disks_metrics.used_space, free_disk_space_bytes: total_disks_metrics.free_space, }, - disk_space_by_disk, + occupied_disk_space_bytes: occupied_disk_space_by_disk.values().sum(), occupied_disk_space_by_disk, + disk_space_by_disk, })) } From f9ef9e931c3d50da8f0ba640876282c37b31b9db Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 21 Aug 2023 13:59:59 +0300 Subject: [PATCH 08/19] map metrics to the f_fsid --- bob/src/api/mod.rs | 32 ++++++++++---------------------- bob/src/hw_metrics_collector.rs | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index f1b3d0893..08e98f14c 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -163,7 +163,7 @@ pub(crate) struct SpaceInfo { occupied_disk_space_by_disk: HashMap, /// Key - mount point - disk_space_by_disk: HashMap, + disk_space_by_disk: HashMap, } async fn tls_server(tls_config: &TLSConfig, addr: SocketAddr) -> AxumServer { @@ -326,7 +326,7 @@ async fn status(bob: Extension>) -> Json { async fn get_space_info( bob: Extension>, ) -> Result, StatusExt> { - let disk_metrics = bob.grinder().hw_counter().update_space_metrics(); + let disk_metrics = dbg!(bob.grinder().hw_counter().update_space_metrics()); let total_disks_metrics: DiskSpaceMetrics = disk_metrics.values().sum(); let backend = bob.grinder().backend().inner(); let (dcs, adc) = backend @@ -337,14 +337,14 @@ async fn get_space_info( dcs.iter().map(|dc| async { ( dc.disk().name().clone(), dc.disk_used().await ) }) ).await.into_iter().collect(); - let disk_space_by_disk = - dcs - .iter() - .map(|dc| { - let mount_point = PathBuf::from(dc.disk().path()); - let space = get_space(&disk_metrics, &mount_point); - (mount_point, space) - }).collect(); + let disk_space_by_disk = disk_metrics.into_values().map(|disk| ( + disk.disk_name, + Space { + total_disk_space_bytes: disk.total_space, + free_disk_space_bytes: disk.free_space, + used_disk_space_bytes: disk.used_space + } + )).collect(); let adc_space = adc.disk_used().await; occupied_disk_space_by_disk.entry(adc.disk().name().clone()) @@ -363,18 +363,6 @@ async fn get_space_info( })) } -fn get_space(metrics: &HashMap<&PathBuf, DiskSpaceMetrics>, mount_point: &PathBuf) -> Space { - metrics.get(&mount_point).map_or(Space { - total_disk_space_bytes: 0, - free_disk_space_bytes: 0, - used_disk_space_bytes: 0, - }, |metrics| Space { - total_disk_space_bytes: metrics.total_space, - free_disk_space_bytes: metrics.free_space, - used_disk_space_bytes: metrics.used_space, - }) -} - fn not_acceptable_backend() -> StatusExt { let status = StatusExt::new( StatusCode::NOT_ACCEPTABLE, diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 0332b92ee..3d77ca06e 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -67,7 +67,10 @@ impl HWMetricsCollector { tokio::spawn(Self::task(self.interval_time, self.disks.clone())); } - pub(crate) fn update_space_metrics(&self) -> HashMap<&PathBuf, DiskSpaceMetrics> { + /// Returns the updated space metrics of this [`HWMetricsCollector`]. + /// + /// Key -- `f_fsid` of underlying [`statvfs`] function, represents file system ID + pub(crate) fn update_space_metrics(&self) -> HashMap { Self::update_space_metrics_from_disks(&self.disks) } @@ -130,7 +133,7 @@ impl HWMetricsCollector { fn update_space_metrics_from_disks( disks: &HashMap, - ) -> HashMap<&PathBuf, DiskSpaceMetrics> { + ) -> HashMap { let disks_metrics = Self::space(disks); let summed_space: DiskSpaceMetrics = disks_metrics.values().sum(); gauge!(TOTAL_SPACE, bytes_to_mb(summed_space.total_space) as f64); @@ -164,7 +167,10 @@ impl HWMetricsCollector { // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big, // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one - fn space(disks: &HashMap) -> HashMap<&PathBuf, DiskSpaceMetrics> { + /// Maps [`DiskSpaceMetrics`] to the corresponding `f_fsid` of [`statvfs`] function result + /// + /// `f_fsid` represents file system ID. + fn space(disks: &HashMap) -> HashMap { let mut res = HashMap::new(); for (mount_point, disk_name) in disks { let cm_p = Self::to_cpath(mount_point.as_path()); @@ -175,12 +181,12 @@ impl HWMetricsCollector { let bavail = stat.f_bavail; let bfree = stat.f_bfree; res.insert( - mount_point, + stat.f_fsid, DiskSpaceMetrics { + disk_name: disk_name.clone(), total_space: bsize * blocks, used_space: bsize * bavail, free_space: (blocks - bfree) * bsize, - disk_name: disk_name.clone(), }, ); } @@ -616,7 +622,7 @@ impl<'a> Add<&'a Self> for DiskSpaceMetrics { impl<'a> Sum<&'a Self> for DiskSpaceMetrics { /// Summarize [`DiskSpaceMetrics`] over an iterator. - /// NOTE: the `disk_name` field will chosen from the first appeared disk in iterator if there + /// NOTE: the `disk_name` field will be chosen from the first appeared disk in iterator if there /// is any. Otherwise 'None' will be passed fn sum>(mut iter: I) -> Self { let init = if let Some(metrics) = iter.next() { From c91108d5a449f529aada1877e69ee1cb81df2d3c Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 21 Aug 2023 14:20:14 +0300 Subject: [PATCH 09/19] change DiskName to mount points --- bob/src/api/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 08e98f14c..9f48786c1 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -162,8 +162,8 @@ pub(crate) struct SpaceInfo { /// Key - Bob disk name occupied_disk_space_by_disk: HashMap, - /// Key - mount point - disk_space_by_disk: HashMap, + /// Key - Mount point or Bob disk name if no mount point was found + disk_space_by_disk: HashMap, } async fn tls_server(tls_config: &TLSConfig, addr: SocketAddr) -> AxumServer { @@ -337,8 +337,10 @@ async fn get_space_info( dcs.iter().map(|dc| async { ( dc.disk().name().clone(), dc.disk_used().await ) }) ).await.into_iter().collect(); + let disk_path: HashMap<_, _> = dcs.iter().map(|dc| (dc.disk().name().to_string(), dc.disk().path().to_string())).collect(); + let disk_space_by_disk = disk_metrics.into_values().map(|disk| ( - disk.disk_name, + disk_path.get(&disk.disk_name.to_string()).unwrap_or(&disk.disk_name.to_string()).clone(), Space { total_disk_space_bytes: disk.total_space, free_disk_space_bytes: disk.free_space, From 462d9f704bcdc6a96465767a1f172a5912bb5c7d Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 21 Aug 2023 14:58:17 +0300 Subject: [PATCH 10/19] delete dbg! --- bob/src/api/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 9f48786c1..f4804de76 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -326,7 +326,7 @@ async fn status(bob: Extension>) -> Json { async fn get_space_info( bob: Extension>, ) -> Result, StatusExt> { - let disk_metrics = dbg!(bob.grinder().hw_counter().update_space_metrics()); + let disk_metrics = bob.grinder().hw_counter().update_space_metrics(); let total_disks_metrics: DiskSpaceMetrics = disk_metrics.values().sum(); let backend = bob.grinder().backend().inner(); let (dcs, adc) = backend From 4dece524823c018756a3fc4aa25424482adc337d Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Fri, 25 Aug 2023 19:22:53 +0300 Subject: [PATCH 11/19] mount point as key --- bob/src/hw_metrics_collector.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 3d77ca06e..deca333a8 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -70,7 +70,7 @@ impl HWMetricsCollector { /// Returns the updated space metrics of this [`HWMetricsCollector`]. /// /// Key -- `f_fsid` of underlying [`statvfs`] function, represents file system ID - pub(crate) fn update_space_metrics(&self) -> HashMap { + pub(crate) fn update_space_metrics(&self) -> HashMap { Self::update_space_metrics_from_disks(&self.disks) } @@ -133,7 +133,7 @@ impl HWMetricsCollector { fn update_space_metrics_from_disks( disks: &HashMap, - ) -> HashMap { + ) -> HashMap { let disks_metrics = Self::space(disks); let summed_space: DiskSpaceMetrics = disks_metrics.values().sum(); gauge!(TOTAL_SPACE, bytes_to_mb(summed_space.total_space) as f64); @@ -167,11 +167,12 @@ impl HWMetricsCollector { // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big, // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one - /// Maps [`DiskSpaceMetrics`] to the corresponding `f_fsid` of [`statvfs`] function result + /// Maps [`DiskSpaceMetrics`] to the corresponding mount point. /// - /// `f_fsid` represents file system ID. - fn space(disks: &HashMap) -> HashMap { + /// `f_fsid` field of [`statvfs`] is used to ensure that disks are unique. + fn space(disks: &HashMap) -> HashMap { let mut res = HashMap::new(); + let mut fs_ids = HashSet::new(); for (mount_point, disk_name) in disks { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); @@ -180,14 +181,16 @@ impl HWMetricsCollector { let blocks = stat.f_blocks; let bavail = stat.f_bavail; let bfree = stat.f_bfree; - res.insert( - stat.f_fsid, - DiskSpaceMetrics { - disk_name: disk_name.clone(), - total_space: bsize * blocks, - used_space: bsize * bavail, - free_space: (blocks - bfree) * bsize, - }, + fs_ids.insert(stat.f_fsid).then(|| + res.insert( + mount_point.clone(), + DiskSpaceMetrics { + disk_name: disk_name.clone(), + total_space: bsize * blocks, + used_space: bsize * bavail, + free_space: (blocks - bfree) * bsize, + }, + ) ); } } From a3a7393c4a2d22217960d6e50aec0609f0587871 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 29 Jan 2024 10:48:00 +0300 Subject: [PATCH 12/19] post review fixes --- bob/src/api/mod.rs | 6 +++--- bob/src/hw_metrics_collector.rs | 24 ++++++++++-------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index f4804de76..9e1a82859 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -336,11 +336,12 @@ async fn get_space_info( let mut occupied_disk_space_by_disk: HashMap = futures::future::join_all( dcs.iter().map(|dc| async { ( dc.disk().name().clone(), dc.disk_used().await ) }) ).await.into_iter().collect(); + let adc_space = adc.disk_used().await; let disk_path: HashMap<_, _> = dcs.iter().map(|dc| (dc.disk().name().to_string(), dc.disk().path().to_string())).collect(); - let disk_space_by_disk = disk_metrics.into_values().map(|disk| ( - disk_path.get(&disk.disk_name.to_string()).unwrap_or(&disk.disk_name.to_string()).clone(), + let disk_space_by_disk = disk_metrics.into_iter().map(|(mount_point, disk)| ( + mount_point.to_string_lossy().to_string(), Space { total_disk_space_bytes: disk.total_space, free_disk_space_bytes: disk.free_space, @@ -348,7 +349,6 @@ async fn get_space_info( } )).collect(); - let adc_space = adc.disk_used().await; occupied_disk_space_by_disk.entry(adc.disk().name().clone()) .and_modify(|s| *s += adc_space) .or_insert(adc_space); diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 9b297c132..85aefea86 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -20,7 +20,6 @@ const DISK_STAT_FILE: &str = "/proc/diskstats"; #[derive(Debug, Clone)] pub(crate) struct DiskSpaceMetrics { - pub(crate) disk_name: DiskName, pub(crate) total_space: u64, pub(crate) used_space: u64, pub(crate) free_space: u64, @@ -69,7 +68,8 @@ impl HWMetricsCollector { /// Returns the updated space metrics of this [`HWMetricsCollector`]. /// - /// Key -- `f_fsid` of underlying [`statvfs`] function, represents file system ID + /// Key -- mount points of used disks + /// Value -- Updated [`DiskSpaceMetrics`] pub(crate) fn update_space_metrics(&self) -> HashMap { Self::update_space_metrics_from_disks(&self.disks) } @@ -168,9 +168,7 @@ impl HWMetricsCollector { // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big, // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one - /// Maps [`DiskSpaceMetrics`] to the corresponding mount point. - /// - /// `f_fsid` field of [`statvfs`] is used to ensure that disks are unique. + /// Maps mount point to the corresponding [`DiskSpaceMetrics`]. fn space(disks: &HashMap) -> HashMap { let mut res = HashMap::new(); let mut fs_ids = HashSet::new(); @@ -178,21 +176,20 @@ impl HWMetricsCollector { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); if let Some(stat) = stat { - let bsize = stat.f_bsize; - let blocks = stat.f_blocks; - let bavail = stat.f_bavail; - let bfree = stat.f_bfree; - fs_ids.insert(stat.f_fsid).then(|| + let bsize: u64 = stat.f_bsize; + let blocks: u64 = stat.f_blocks; + let bavail: u64 = stat.f_bavail; + let bfree: u64 = stat.f_bfree; + if fs_ids.insert(stat.f_fsid) { res.insert( mount_point.clone(), DiskSpaceMetrics { - disk_name: disk_name.clone(), total_space: bsize * blocks, used_space: bsize * bavail, free_space: (blocks - bfree) * bsize, }, - ) - ); + ); + } } } @@ -646,7 +643,6 @@ impl<'a> Sum<&'a Self> for DiskSpaceMetrics { metrics.clone() } else { return Self { - disk_name: "None".into(), total_space: 0, used_space: 0, free_space: 0, From 65a3c1a629a788e987ca5e3b2d1c365927e03bf4 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 29 Jan 2024 11:41:54 +0300 Subject: [PATCH 13/19] remove unused vars --- bob/src/api/mod.rs | 2 -- bob/src/hw_metrics_collector.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 9e1a82859..25a3e6ab5 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -338,8 +338,6 @@ async fn get_space_info( ).await.into_iter().collect(); let adc_space = adc.disk_used().await; - let disk_path: HashMap<_, _> = dcs.iter().map(|dc| (dc.disk().name().to_string(), dc.disk().path().to_string())).collect(); - let disk_space_by_disk = disk_metrics.into_iter().map(|(mount_point, disk)| ( mount_point.to_string_lossy().to_string(), Space { diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 85aefea86..b20d127e9 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -172,7 +172,7 @@ impl HWMetricsCollector { fn space(disks: &HashMap) -> HashMap { let mut res = HashMap::new(); let mut fs_ids = HashSet::new(); - for (mount_point, disk_name) in disks { + for (mount_point, _) in disks { let cm_p = Self::to_cpath(mount_point.as_path()); let stat = Self::statvfs_wrap(&cm_p); if let Some(stat) = stat { From 582b6177914a91d849102c46c06d3a333b449c8d Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 29 Jan 2024 14:20:49 +0300 Subject: [PATCH 14/19] update openapi --- config-examples/openapi.yaml | 46 +++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/config-examples/openapi.yaml b/config-examples/openapi.yaml index 899bf9974..958944e5c 100644 --- a/config-examples/openapi.yaml +++ b/config-examples/openapi.yaml @@ -750,21 +750,51 @@ components: type: string root_dir_name: type: string - SpaceInfo: + Space: type: object + required: + - total_disk_space_bytes + - free_disk_space_bytes + - used_disk_space_bytes properties: - total_disk_space_bytes: - type: integer free_disk_space_bytes: type: integer - used_disk_space_bytes: + format: int64 + minimum: 0 + total_disk_space_bytes: type: integer - occupied_disk_space_bytes: + format: int64 + minimum: 0 + used_disk_space_bytes: type: integer - occupied_disk_space_by_disk: - type: object - additionalProperties: + format: int64 + minimum: 0 + SpaceInfo: + allOf: + - $ref: '#/components/schemas/Space' + - type: object + required: + - occupied_disk_space_bytes + - occupied_disk_space_by_disk + - disk_space_by_disk + properties: + disk_space_by_disk: + type: object + description: Key - Mount point or Bob disk name if no mount point was found + additionalProperties: + $ref: '#/components/schemas/Space' + occupied_disk_space_by_disk: + type: object + description: Key - Bob disk name + additionalProperties: + type: integer + format: int64 + minimum: 0 + occupied_disk_space_bytes: type: integer + format: int64 + description: Total occupied disk space by Bob + minimum: 0 MetricsEntryModel: type: object properties: From abb93580c268da01f1c8bce70ebe4a99f1027b81 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 29 Jan 2024 14:48:46 +0300 Subject: [PATCH 15/19] add total space info in metrics --- bob/src/api/mod.rs | 11 +++++------ bob/src/hw_metrics_collector.rs | 34 ++++++++++++++++++++------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 25a3e6ab5..11939a962 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -1,5 +1,5 @@ use crate::{ - build_info::BuildInfo, hw_metrics_collector::DiskSpaceMetrics, server::Server as BobServer, + build_info::BuildInfo, server::Server as BobServer, }; use axum::{ body::{self, BoxBody}, @@ -327,7 +327,6 @@ async fn get_space_info( bob: Extension>, ) -> Result, StatusExt> { let disk_metrics = bob.grinder().hw_counter().update_space_metrics(); - let total_disks_metrics: DiskSpaceMetrics = disk_metrics.values().sum(); let backend = bob.grinder().backend().inner(); let (dcs, adc) = backend .disk_controllers() @@ -338,7 +337,7 @@ async fn get_space_info( ).await.into_iter().collect(); let adc_space = adc.disk_used().await; - let disk_space_by_disk = disk_metrics.into_iter().map(|(mount_point, disk)| ( + let disk_space_by_disk = disk_metrics.per_disk.into_iter().map(|(mount_point, disk)| ( mount_point.to_string_lossy().to_string(), Space { total_disk_space_bytes: disk.total_space, @@ -353,9 +352,9 @@ async fn get_space_info( Ok(Json(SpaceInfo { total_space: Space { - total_disk_space_bytes: total_disks_metrics.total_space, - used_disk_space_bytes: total_disks_metrics.used_space, - free_disk_space_bytes: total_disks_metrics.free_space, + total_disk_space_bytes: disk_metrics.total_space, + used_disk_space_bytes: disk_metrics.used_space, + free_disk_space_bytes: disk_metrics.free_space, }, occupied_disk_space_bytes: occupied_disk_space_by_disk.values().sum(), occupied_disk_space_by_disk, diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index b20d127e9..b0e1a1db6 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -23,6 +23,14 @@ pub(crate) struct DiskSpaceMetrics { pub(crate) total_space: u64, pub(crate) used_space: u64, pub(crate) free_space: u64, + pub(crate) per_disk: HashMap +} + +#[derive(Debug, Clone)] +pub(crate) struct SpaceMetrics { + pub(crate) total_space: u64, + pub(crate) used_space: u64, + pub(crate) free_space: u64, } pub(crate) struct HWMetricsCollector { @@ -70,7 +78,7 @@ impl HWMetricsCollector { /// /// Key -- mount points of used disks /// Value -- Updated [`DiskSpaceMetrics`] - pub(crate) fn update_space_metrics(&self) -> HashMap { + pub(crate) fn update_space_metrics(&self) -> DiskSpaceMetrics { Self::update_space_metrics_from_disks(&self.disks) } @@ -134,13 +142,13 @@ impl HWMetricsCollector { fn update_space_metrics_from_disks( disks: &HashMap, - ) -> HashMap { + ) -> DiskSpaceMetrics { let disks_metrics = Self::space(disks); - let summed_space: DiskSpaceMetrics = disks_metrics.values().sum(); - gauge!(TOTAL_SPACE, bytes_to_mb(summed_space.total_space) as f64); - gauge!(USED_SPACE, bytes_to_mb(summed_space.used_space) as f64); - gauge!(FREE_SPACE, bytes_to_mb(summed_space.free_space) as f64); - disks_metrics + let SpaceMetrics { total_space, used_space, free_space } = disks_metrics.values().sum(); + gauge!(TOTAL_SPACE, bytes_to_mb(total_space) as f64); + gauge!(USED_SPACE, bytes_to_mb(used_space) as f64); + gauge!(FREE_SPACE, bytes_to_mb(free_space) as f64); + DiskSpaceMetrics { total_space, used_space, free_space, per_disk: disks_metrics } } fn to_cpath(path: &Path) -> Vec { @@ -168,8 +176,8 @@ impl HWMetricsCollector { // NOTE: HashMap contains only needed mount points of used disks, so it won't be really big, // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one - /// Maps mount point to the corresponding [`DiskSpaceMetrics`]. - fn space(disks: &HashMap) -> HashMap { + /// Maps mount point to the corresponding [`SpaceMetrics`]. + fn space(disks: &HashMap) -> HashMap { let mut res = HashMap::new(); let mut fs_ids = HashSet::new(); for (mount_point, _) in disks { @@ -183,7 +191,7 @@ impl HWMetricsCollector { if fs_ids.insert(stat.f_fsid) { res.insert( mount_point.clone(), - DiskSpaceMetrics { + SpaceMetrics { total_space: bsize * blocks, used_space: bsize * bavail, free_space: (blocks - bfree) * bsize, @@ -618,14 +626,14 @@ async fn parse_command_output(command: &mut Command) -> Result { // Std Traits Impls -impl<'a> AddAssign<&'a Self> for DiskSpaceMetrics { +impl<'a> AddAssign<&'a Self> for SpaceMetrics { fn add_assign(&mut self, rhs: &Self) { self.used_space += rhs.used_space; self.free_space += rhs.free_space; self.total_space += rhs.total_space; } } -impl<'a> Add<&'a Self> for DiskSpaceMetrics { +impl<'a> Add<&'a Self> for SpaceMetrics { type Output = Self; fn add(mut self, rhs: &Self) -> Self::Output { @@ -634,7 +642,7 @@ impl<'a> Add<&'a Self> for DiskSpaceMetrics { } } -impl<'a> Sum<&'a Self> for DiskSpaceMetrics { +impl<'a> Sum<&'a Self> for SpaceMetrics { /// Summarize [`DiskSpaceMetrics`] over an iterator. /// NOTE: the `disk_name` field will be chosen from the first appeared disk in iterator if there /// is any. Otherwise 'None' will be passed From 5c9c28fd6b87abfdb821b0c0d0f693a9a263faf2 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 29 Jan 2024 14:53:32 +0300 Subject: [PATCH 16/19] update doc-strings --- bob/src/hw_metrics_collector.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index b0e1a1db6..ecc7cfe3c 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -75,9 +75,6 @@ impl HWMetricsCollector { } /// Returns the updated space metrics of this [`HWMetricsCollector`]. - /// - /// Key -- mount points of used disks - /// Value -- Updated [`DiskSpaceMetrics`] pub(crate) fn update_space_metrics(&self) -> DiskSpaceMetrics { Self::update_space_metrics_from_disks(&self.disks) } @@ -177,6 +174,9 @@ impl HWMetricsCollector { // but maybe it's more efficient to store disks (instead of mount_points) and update them one by one /// Maps mount point to the corresponding [`SpaceMetrics`]. + /// + /// Key -- mount points of used disks + /// Value -- Up-to-date [`SpaceMetrics`] fn space(disks: &HashMap) -> HashMap { let mut res = HashMap::new(); let mut fs_ids = HashSet::new(); @@ -643,7 +643,7 @@ impl<'a> Add<&'a Self> for SpaceMetrics { } impl<'a> Sum<&'a Self> for SpaceMetrics { - /// Summarize [`DiskSpaceMetrics`] over an iterator. + /// Summarize [`SpaceMetrics`] over an iterator. /// NOTE: the `disk_name` field will be chosen from the first appeared disk in iterator if there /// is any. Otherwise 'None' will be passed fn sum>(mut iter: I) -> Self { From baeb407055ebe9dfbba2d4eff6e5217424009d65 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Mon, 29 Jan 2024 15:39:35 +0300 Subject: [PATCH 17/19] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c28240d2f..391d9eca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Bob versions changelog ## [Unreleased] #### Added - Blob performes fsync if buffered bytes are larger than max_dirty_bytes_before_sync config param (#748) +- Added detailed information about total, used and free space on every disk (#823) #### Changed - Use cargo workspace to declare dependencies to avoid their duplication (#821) @@ -46,7 +47,6 @@ Bob versions changelog - Added mimalloc allocator for musl target (#688) - Added jemalloc-profile for memory profiling (#797) - Proper support for GetSource::ALL requests (#723) -- Added detailed information about total, used and free space on every disk (#823) #### Changed - BobClient clone overhead reduced (#774) From e9a21cf518f24d7a62c81ac0dace4b045661c531 Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Wed, 31 Jan 2024 05:45:52 +0300 Subject: [PATCH 18/19] re-use space metrics --- bob/src/api/mod.rs | 6 +++--- bob/src/hw_metrics_collector.rs | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 11939a962..1879f0e53 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -352,9 +352,9 @@ async fn get_space_info( Ok(Json(SpaceInfo { total_space: Space { - total_disk_space_bytes: disk_metrics.total_space, - used_disk_space_bytes: disk_metrics.used_space, - free_disk_space_bytes: disk_metrics.free_space, + total_disk_space_bytes: disk_metrics.total.total_space, + used_disk_space_bytes: disk_metrics.total.used_space, + free_disk_space_bytes: disk_metrics.total.free_space, }, occupied_disk_space_bytes: occupied_disk_space_by_disk.values().sum(), occupied_disk_space_by_disk, diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index ecc7cfe3c..8a6e0efd2 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -20,9 +20,7 @@ const DISK_STAT_FILE: &str = "/proc/diskstats"; #[derive(Debug, Clone)] pub(crate) struct DiskSpaceMetrics { - pub(crate) total_space: u64, - pub(crate) used_space: u64, - pub(crate) free_space: u64, + pub(crate) total: SpaceMetrics, pub(crate) per_disk: HashMap } @@ -145,7 +143,14 @@ impl HWMetricsCollector { gauge!(TOTAL_SPACE, bytes_to_mb(total_space) as f64); gauge!(USED_SPACE, bytes_to_mb(used_space) as f64); gauge!(FREE_SPACE, bytes_to_mb(free_space) as f64); - DiskSpaceMetrics { total_space, used_space, free_space, per_disk: disks_metrics } + DiskSpaceMetrics { + total: SpaceMetrics { + total_space, + used_space, + free_space, + }, + per_disk: disks_metrics, + } } fn to_cpath(path: &Path) -> Vec { From 44f7db82f0423ba7e9ba0d364da81651e6ea00aa Mon Sep 17 00:00:00 2001 From: Simeon Romanov Date: Wed, 31 Jan 2024 15:56:22 +0300 Subject: [PATCH 19/19] used_space and free_space calculation swap --- bob/src/hw_metrics_collector.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bob/src/hw_metrics_collector.rs b/bob/src/hw_metrics_collector.rs index 8a6e0efd2..b0fd1808f 100644 --- a/bob/src/hw_metrics_collector.rs +++ b/bob/src/hw_metrics_collector.rs @@ -198,8 +198,8 @@ impl HWMetricsCollector { mount_point.clone(), SpaceMetrics { total_space: bsize * blocks, - used_space: bsize * bavail, - free_space: (blocks - bfree) * bsize, + used_space: (blocks - bfree) * bsize, + free_space: bsize * bavail, }, ); }