Merge pull request #65 from silicon-heaven/unify-configs

Unify configs

Author: fvacek

Cargo.lock

@@ -3,7 +3,7 @@ name = "historyprovider"
 description = "historyprovider-rs"
 license = "MIT"
 repository = "https://github.com/silicon-heaven/historyprovider-rs"
-version = "2.4.1"
+version = "2.5.0"
 edition = "2024"
 
 [[bin]]

Cargo.toml

@@ -11,9 +11,6 @@ struct Opts {
     /// Config file path
     #[arg(long)]
     config: Option<String>,
-    /// Historyprovider config file
-    #[arg(long, default_value = "config.yaml")]
-    hp_config: String,
     /// Create default config file if one specified by --config is not found
     #[arg(short, long)]
     create_default_config: bool,
@@ -39,6 +36,24 @@ struct Opts {
     /// Print version and exit
     #[arg(short = 'V', long)]
     version: bool,
+
+    // HP specific options
+    //
+    /// Path to journal directory
+    #[arg(long)]
+    journal_dir: Option<String>,
+    /// Maximum number of parallel sync tasks
+    #[arg(long)]
+    max_sync_tasks: Option<usize>,
+    /// Maximum size of journal directory in bytes
+    #[arg(long)]
+    max_journal_dir_size: Option<usize>,
+    /// Periodic sync interval in seconds
+    #[arg(long)]
+    periodic_sync_interval: Option<u64>,
+    /// Number of days to keep history data
+    #[arg(long)]
+    days_to_keep: Option<i64>,
 }
 
 fn init_logger(cli_opts: &Opts) {
@@ -65,24 +80,34 @@ fn print_banner(text: impl AsRef<str>) {
     info!("{banner_line}");
 }
 
-fn load_client_config(cli_opts: Opts) -> shvrpc::Result<ClientConfig> {
-    let mut config = if let Some(config_file) = &cli_opts.config {
-        ClientConfig::from_file_or_default(config_file, cli_opts.create_default_config)?
+fn load_client_config(cli_opts: Opts) -> shvrpc::Result<(ClientConfig, HpConfig)> {
+    let (mut client_config, mut hp_config) = if let Some(config_file) = &cli_opts.config {
+        (
+            ClientConfig::from_file_or_default(config_file, cli_opts.create_default_config)?,
+            HpConfig::load(config_file)?,
+        )
     } else {
-        Default::default()
+        (Default::default(), Default::default())
     };
-    config.url = cli_opts.url.map_or_else(|| Ok(config.url), |url_str| Url::parse(&url_str))?;
-    config.device_id = cli_opts.device_id.or(config.device_id);
-    config.mount = cli_opts.mount.or(config.mount);
-    config.reconnect_interval = cli_opts.reconnect_interval.map_or_else(
-        || Ok(config.reconnect_interval),
+    client_config.url = cli_opts.url.map_or_else(|| Ok(client_config.url), |url_str| Url::parse(&url_str))?;
+    client_config.device_id = cli_opts.device_id.or(client_config.device_id);
+    client_config.mount = cli_opts.mount.or(client_config.mount);
+    client_config.reconnect_interval = cli_opts.reconnect_interval.map_or_else(
+        || Ok(client_config.reconnect_interval),
         |interval_str| duration_str::parse(&interval_str).map(Some)
     )?;
-    config.heartbeat_interval = cli_opts.heartbeat_interval.map_or_else(
-        || Ok(config.heartbeat_interval),
+    client_config.heartbeat_interval = cli_opts.heartbeat_interval.map_or_else(
+        || Ok(client_config.heartbeat_interval),
         |interval_str| duration_str::parse(&interval_str)
     )?;
-    Ok(config)
+
+    hp_config.journal_dir = cli_opts.journal_dir.unwrap_or(hp_config.journal_dir);
+    hp_config.max_sync_tasks = cli_opts.max_sync_tasks.unwrap_or(hp_config.max_sync_tasks);
+    hp_config.max_journal_dir_size = cli_opts.max_journal_dir_size.unwrap_or(hp_config.max_journal_dir_size);
+    hp_config.periodic_sync_interval = cli_opts.periodic_sync_interval.unwrap_or(hp_config.periodic_sync_interval);
+    hp_config.days_to_keep = cli_opts.days_to_keep.unwrap_or(hp_config.days_to_keep);
+
+    Ok((client_config, hp_config))
 }
 
 #[tokio::main]
@@ -97,10 +122,7 @@ pub(crate) async fn main() -> shvrpc::Result<()> {
 
     init_logger(&cli_opts);
     print_banner(format!("{} {} starting", std::module_path!(), PKG_VERSION));
-
-    let hp_config = cli_opts.hp_config.clone();
-    let client_config = load_client_config(cli_opts).expect("Invalid config");
-    let hp_config = HpConfig::load(hp_config)?;
+    let (client_config, hp_config) = load_client_config(cli_opts).expect("Invalid config");
 
     historyprovider::run(&hp_config, &client_config).await
 }

src/bin/hp.rs

@@ -27,18 +27,38 @@ mod util;
 pub mod typeinfo;
 pub mod alarm;
 
-fn default_journal_dir() -> String {
+const fn max_sync_tasks_default() -> usize { 8 }
+const fn max_journal_dir_size_default() -> usize { 30 * 1_000_000_000 } // 30 GB
+const fn periodic_sync_interval_default() -> u64 { 60 * 60 } // 1 hour
+
+fn journal_dir_default() -> String {
     "/tmp/hp-rs/shvjournal".into()
 }
 
 #[derive(Clone, Deserialize)]
 pub struct HpConfig {
-    #[serde(default = "default_journal_dir")]
-    journal_dir: String,
-    max_sync_tasks: Option<usize>,
-    max_journal_dir_size: Option<usize>,
-    periodic_sync_interval: Option<u64>,
-    days_to_keep: Option<i64>,
+    #[serde(default = "journal_dir_default")]
+    pub journal_dir: String,
+    #[serde(default = "max_sync_tasks_default")]
+    pub max_sync_tasks: usize,
+    #[serde(default = "max_journal_dir_size_default")]
+    pub max_journal_dir_size: usize,
+    #[serde(default = "periodic_sync_interval_default")]
+    pub periodic_sync_interval: u64,
+    #[serde(default)]
+    pub days_to_keep: i64,
+}
+
+impl Default for HpConfig {
+    fn default() -> Self {
+        Self {
+            journal_dir: journal_dir_default(),
+            max_sync_tasks: max_sync_tasks_default(),
+            max_journal_dir_size: max_journal_dir_size_default(),
+            periodic_sync_interval: periodic_sync_interval_default(),
+            days_to_keep: Default::default(),
+        }
+    }
 }
 
 impl HpConfig {

src/lib.rs

@@ -353,8 +353,7 @@ pub(crate) async fn sites_task(
     {
         let app_state = app_state.clone();
         tokio::spawn(async move {
-            const PERIODIC_SYNC_INTERVAL_DEFAULT: u64 = 60 * 60;
-            let periodic_sync_interval = app_state.config.periodic_sync_interval.unwrap_or(PERIODIC_SYNC_INTERVAL_DEFAULT);
+            let periodic_sync_interval = app_state.config.periodic_sync_interval;
             let mut interval: Option<tokio::time::Interval> = None;
             loop {
                 tokio::select! {

src/sites.rs

@@ -25,7 +25,7 @@ use crate::journalrw::{GetLog2Params, GetLog2Since, JournalReaderLog2, JournalWr
 use crate::sites::{SitesData, SubHpInfo};
 use crate::tree::{FileType, LsFilesEntry, METH_READ};
 use crate::util::{get_files, is_log2_file, msec_to_log2_filename, DedupReceiver};
-use crate::{HpConfig, State};
+use crate::State;
 
 #[derive(Default)]
 pub(crate) struct SyncInfo {
@@ -705,13 +705,6 @@ async fn sync_site_legacy(
     Ok(())
 }
 
-const MAX_SYNC_TASKS_DEFAULT: usize = 8;
-
-pub fn log_size_limit(config: &HpConfig) -> u64 {
-    const MAX_JOURNAL_DIR_SIZE_DEFAULT: usize = 30 * 1_000_000_000;
-    config.max_journal_dir_size.unwrap_or(MAX_JOURNAL_DIR_SIZE_DEFAULT) as u64
-}
-
 pub(crate) async fn sync_task(
     client_cmd_tx: ClientCommandSender,
     _client_evt_rx: ClientEventsReceiver,
@@ -736,8 +729,8 @@ pub(crate) async fn sync_task(
 
     // The download size limit should be lower than the max_journal_dir_size, because it doesn't
     // count in the files synced by getLog.
-    let max_journal_dir_size = log_size_limit(&app_state.config);
-    let days_to_keep = app_state.config.days_to_keep.unwrap_or(0);
+    let max_journal_dir_size = app_state.config.max_journal_dir_size as u64;
+    let days_to_keep = app_state.config.days_to_keep;
 
     while let Some(cmd) = sync_cmd_rx.next().await {
         match cmd {
@@ -749,7 +742,7 @@ pub(crate) async fn sync_task(
                     app_state.sites_data.read().await.clone(),
                     max_journal_dir_size
                 ).await;
-                let max_sync_tasks = app_state.config.max_sync_tasks.unwrap_or(MAX_SYNC_TASKS_DEFAULT);
+                let max_sync_tasks = app_state.config.max_sync_tasks;
                 let semaphore = Arc::new(Semaphore::new(max_sync_tasks));
                 let mut sync_tasks = vec![];
                 let sync_start = tokio::time::Instant::now();

src/sync.rs

@@ -26,7 +26,6 @@ use crate::getlog::getlog_handler;
 use crate::journalrw::{journal_entries_to_rpcvalue, GetLog2Params, Log2Header, Log2Reader};
 use crate::pushlog::pushlog_impl;
 use crate::sites::SubHpInfo;
-use crate::sync::log_size_limit;
 use crate::{AlarmWithTimestamp, HpConfig, State};
 
 // History site node methods
@@ -236,15 +235,15 @@ async fn shvjournal_request_handler(
                 METH_LS_FILES => return m.resolve(METHODS, async || {
                     journaldir_lsfiles_handler(get_journaldir_entries(path).await?).await
                 }),
-                METH_LOG_SIZE_LIMIT => return m.resolve(METHODS, async move || Ok(log_size_limit(&app_state.config))),
+                METH_LOG_SIZE_LIMIT => return m.resolve(METHODS, async move || Ok(app_state.config.max_journal_dir_size as i64)),
                 METH_TOTAL_LOG_SIZE => return m.resolve(METHODS, async move || total_log_size(&app_state.config)
                     .await
                     .map(RpcValue::from)
                     .map_err(rpc_error_filesystem)
                 ),
                 METH_LOG_USAGE => return m.resolve(METHODS, async move || total_log_size(&app_state.config)
                         .await
-                        .map(|size| 100. * (size as f64) / (log_size_limit(&app_state.config) as f64))
+                        .map(|size| 100. * (size as f64) / (app_state.config.max_journal_dir_size as f64))
                         .map_err(rpc_error_filesystem)
                 ),
                 METH_SYNC_LOG => return m.resolve(METHODS, async move || sync_log_request_handler(&param, app_state).await),

src/tree.rs

@@ -144,7 +144,7 @@ pub(crate) fn msec_to_log2_filename(msec: i64) -> String {
 
 #[cfg(test)]
 pub mod testing {
-    use crate::{State, dirtylog::DirtyLogCommand, sites::{SiteInfo, SitesData, SubHpInfo}, sync::SyncCommand, util::{DedupReceiver, dedup_channel}};
+    use crate::{State, dirtylog::DirtyLogCommand, max_journal_dir_size_default, max_sync_tasks_default, sites::{SiteInfo, SitesData, SubHpInfo}, sync::SyncCommand, util::{DedupReceiver, dedup_channel}};
     use futures::{channel::mpsc::{unbounded, UnboundedReceiver, UnboundedSender}, StreamExt};
     use log::debug;
     use shvclient::{clientapi::{ClientCommand, ClientEventsReceiver}, ClientCommandSender};
@@ -404,10 +404,10 @@ pub mod testing {
             start_time: std::time::Instant::now(),
             config: crate::HpConfig {
                 journal_dir: journal_dir.path().to_str().expect("path must work").to_string(),
-                max_sync_tasks: None,
-                max_journal_dir_size: None,
-                days_to_keep: None,
-                periodic_sync_interval: Some(3),
+                max_sync_tasks: max_sync_tasks_default(),
+                max_journal_dir_size: max_journal_dir_size_default(),
+                days_to_keep: 0,
+                periodic_sync_interval: 3,
             },
             dirtylog_cmd_tx,
             sync_cmd_tx: dedup_sender.clone(),