Move config to the top level args, start to allow override at source level

Author: Auspicus

martin/benches/bench.rs

@@ -1,6 +1,7 @@
 use async_trait::async_trait;
 use criterion::async_executor::FuturesExecutor;
 use criterion::{Criterion, criterion_group, criterion_main};
+use martin::file_config::ValidationLevel;
 use martin::srv::DynTileSource;
 use martin::{CatalogSourceEntry, MartinResult, Source, TileData, TileSources, UrlQuery};
 use martin_tile_utils::{Encoding, Format, TileCoord, TileInfo};
@@ -42,6 +43,10 @@ impl Source for NullSource {
         false
     }
 
+    async fn validate(&self, _validation_level: ValidationLevel) -> MartinResult<()> {
+        MartinResult::Ok(())
+    }
+
     async fn get_tile(
         &self,
         _xyz: TileCoord,
@@ -63,7 +68,7 @@ async fn process_tile(sources: &TileSources) {
 }
 
 fn bench_null_source(c: &mut Criterion) {
-    let sources = TileSources::new(vec![vec![Box::new(NullSource::new())]]);
+    let sources = TileSources::new(vec![Box::new(NullSource::new())]);
     c.bench_function("get_table_source_tile", |b| {
         b.to_async(FuturesExecutor).iter(|| process_tile(&sources));
     });

martin/src/args/mbtiles.rs

@@ -9,11 +9,6 @@ mod pg;
 #[cfg(feature = "postgres")]
 pub use pg::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT, PgArgs};
 
-#[cfg(feature = "mbtiles")]
-mod mbtiles;
-#[cfg(feature = "mbtiles")]
-pub use mbtiles::MbtArgs;
-
 mod root;
 pub use root::{Args, ExtraArgs, MetaArgs};
 

martin/src/args/mod.rs

@@ -35,9 +35,6 @@ pub struct Args {
     #[cfg(feature = "postgres")]
     #[command(flatten)]
     pub pg: Option<crate::args::pg::PgArgs>,
-    #[cfg(feature = "mbtiles")]
-    #[command(flatten)]
-    pub mbtiles: Option<crate::args::mbtiles::MbtArgs>,
 }
 
 // None of these params will be transferred to the config
@@ -118,17 +115,6 @@ impl Args {
             config.pmtiles = parse_file_args(&mut cli_strings, &["pmtiles"], true);
         }
 
-        #[cfg(feature = "mbtiles")]
-        {
-            if !cli_strings.is_empty() {
-                config.mbtiles = parse_file_args(&mut cli_strings, &["mbtiles"], false);
-            }
-            let mbt_args = self.mbtiles.unwrap_or_default();
-            if let FileConfigEnum::Config(c) = &mut config.mbtiles {
-                mbt_args.override_config(&mut c.custom);
-            }
-        }
-
         #[cfg(feature = "cog")]
         if !cli_strings.is_empty() {
             config.cog = parse_file_args(&mut cli_strings, &["tif", "tiff"], false);

martin/src/args/root.rs

@@ -1,7 +1,10 @@
 use clap::ValueEnum;
 use serde::{Deserialize, Serialize};
 
-use crate::srv::{KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT, SrvConfig};
+use crate::{
+    file_config::{OnInvalid, ValidationLevel},
+    srv::{KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT, SrvConfig},
+};
 
 #[allow(clippy::doc_markdown)]
 #[derive(clap::Args, Debug, PartialEq, Default)]
@@ -34,6 +37,12 @@ pub struct SrvArgs {
     #[arg(short = 'u', long = "webui")]
     #[cfg(feature = "webui")]
     pub web_ui: Option<WebUiMode>,
+    /// Level of validation to apply to sources
+    #[arg(long)]
+    pub validate: Option<ValidationLevel>,
+    /// How to handle invalid source
+    #[arg(long)]
+    pub on_invalid: Option<OnInvalid>,
 }
 
 #[cfg(feature = "webui")]
@@ -81,6 +90,12 @@ impl SrvArgs {
         if self.preferred_encoding.is_some() {
             srv_config.preferred_encoding = self.preferred_encoding;
         }
+        if let Some(v) = self.validate {
+            srv_config.validate = v
+        }
+        if let Some(v) = self.on_invalid {
+            srv_config.on_invalid = v;
+        }
         #[cfg(feature = "webui")]
         if self.web_ui.is_some() {
             srv_config.web_ui = self.web_ui;

martin/src/args/srv.rs

@@ -49,9 +49,6 @@ pub struct CopierArgs {
     #[cfg(feature = "postgres")]
     #[command(flatten)]
     pub pg: Option<martin::args::PgArgs>,
-    #[cfg(feature = "mbtiles")]
-    #[command(flatten)]
-    pub mbtiles: Option<martin::args::MbtArgs>,
 }
 
 #[serde_with::serde_as]
@@ -144,8 +141,6 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> {
         srv: SrvArgs::default(),
         #[cfg(feature = "postgres")]
         pg: copy_args.pg,
-        #[cfg(feature = "mbtiles")]
-        mbtiles: copy_args.mbtiles,
     };
 
     args.merge_into_config(&mut config, &env)?;

martin/src/bin/martin-cp.rs

@@ -13,7 +13,7 @@ use tiff::tags::Tag::{self, GdalNodata};
 use tilejson::{TileJSON, tilejson};
 
 use super::CogError;
-use crate::file_config::{FileError, FileResult};
+use crate::file_config::{FileError, FileResult, ValidationLevel};
 use crate::{MartinResult, Source, TileData, UrlQuery};
 
 #[derive(Clone, Debug)]
@@ -151,6 +151,10 @@ impl Source for CogSource {
         Box::new(self.clone())
     }
 
+    async fn validate(&self, _validation_level: ValidationLevel) -> MartinResult<()> {
+        MartinResult::Ok(())
+    }
+
     async fn get_tile(
         &self,
         xyz: TileCoord,

martin/src/cog/source.rs

@@ -6,10 +6,11 @@ use std::path::{Path, PathBuf};
 use std::pin::Pin;
 
 use futures::future::try_join_all;
-use log::info;
+use log::{info, warn};
 use serde::{Deserialize, Serialize};
 use subst::VariableMap;
 
+use crate::file_config::OnInvalid;
 use crate::MartinError::{ConfigLoadError, ConfigParseError, ConfigWriteError, NoSources};
 #[cfg(any(feature = "fonts", feature = "postgres"))]
 use crate::OptOneMany;
@@ -209,7 +210,34 @@ impl Config {
             sources.push(Box::pin(val));
         }
 
-        Ok(TileSources::new(try_join_all(sources).await?))
+        let resolved_sources = try_join_all(sources).await?
+            .into_iter()
+            .flatten()
+            .collect::<TileInfoSources>();
+        
+        let mut sources_to_prune: Vec<usize> = vec![];
+        for (idx, source) in resolved_sources.iter().enumerate() {
+            let validation_result = source.validate(self.srv.validate).await;
+            if let Err(e) = validation_result {
+                match self.srv.on_invalid {
+                    OnInvalid::Abort => {
+                        return MartinResult::Err(e)
+                    },
+                    OnInvalid::Warn => {
+                        warn!(
+                            "Source {} failed validation, this may cause performance issues: {}",
+                            source.get_id(),
+                            e.to_string()
+                        );
+                    },
+                    OnInvalid::Ignore => {
+                        sources_to_prune.push(idx)
+                    },
+                }
+            }
+        }
+
+        Ok(TileSources::new(resolved_sources))
     }
 
     pub fn save_to_file(&self, file_name: PathBuf) -> MartinResult<()> {

martin/src/config.rs

@@ -3,9 +3,9 @@ use std::fmt::Debug;
 use std::mem;
 use std::path::{Path, PathBuf};
 
+use clap::ValueEnum;
 use futures::TryFutureExt;
 use log::{info, warn};
-use martin_tile_utils::TileInfo;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
@@ -62,6 +62,31 @@ pub enum FileError {
     CogError(#[from] crate::cog::CogError),
 }
 
+#[derive(PartialEq, Eq, Debug, Clone, Copy, Default, Serialize, Deserialize, ValueEnum)]
+#[serde(rename_all = "lowercase")]
+pub enum ValidationLevel {
+    /// Quickly check the source
+    #[default]
+    Fast,
+
+    /// Do a slow check of everything
+    Thorough,
+}
+
+#[derive(PartialEq, Eq, Debug, Clone, Copy, Default, Serialize, Deserialize, ValueEnum)]
+#[serde(rename_all = "lowercase")]
+pub enum OnInvalid {
+    /// Print warning message, and abort if the error is critical
+    #[default]
+    Warn,
+
+    /// Skip this source
+    Ignore,
+
+    /// Do not start Martin on any warnings
+    Abort,
+}
+
 pub trait ConfigExtras: Clone + Debug + Default + PartialEq + Send {
     fn init_parsing(&mut self, _cache: OptMainCache) -> FileResult<()> {
         Ok(())
@@ -130,6 +155,8 @@ impl<T: ConfigExtras> FileConfigEnum<T> {
                 } else {
                     Some(configs)
                 },
+                validate: None,
+                on_invalid: None,
                 custom,
             })
         }
@@ -187,6 +214,10 @@ pub struct FileConfig<T> {
     pub paths: OptOneMany<PathBuf>,
     /// A map of source IDs to file paths or config objects
     pub sources: Option<BTreeMap<String, FileConfigSrc>>,
+    #[serde(default)]
+    pub validate: Option<ValidationLevel>,
+    #[serde(default)]
+    pub on_invalid: Option<OnInvalid>,
     /// Any customizations related to the specifics of the configuration section
     #[serde(flatten)]
     pub custom: T,
@@ -275,10 +306,7 @@ async fn resolve_int<T: SourceConfigExtras>(
                 let dup = if dup { "duplicate " } else { "" };
                 let id = idr.resolve(&id, url.to_string());
                 configs.insert(id.clone(), source);
-                maybe_add_source(
-                    &mut results,
-                    cfg.custom.new_sources_url(id.clone(), url.clone()).await,
-                )?;
+                results.push(cfg.custom.new_sources_url(id.clone(), url.clone()).await?);
                 info!("Configured {dup}source {id} from {}", sanitize_url(&url));
             } else {
                 let can = source.abs_path()?;
@@ -292,10 +320,7 @@ async fn resolve_int<T: SourceConfigExtras>(
                 let id = idr.resolve(&id, can.to_string_lossy().to_string());
                 info!("Configured {dup}source {id} from {}", can.display());
                 configs.insert(id.clone(), source.clone());
-                maybe_add_source(
-                    &mut results,
-                    cfg.custom.new_sources(id, source.into_path()).await,
-                )?;
+                results.push(cfg.custom.new_sources(id, source.into_path()).await?);
             }
         }
     }
@@ -319,10 +344,7 @@ async fn resolve_int<T: SourceConfigExtras>(
 
             let id = idr.resolve(id, url.to_string());
             configs.insert(id.clone(), FileConfigSrc::Path(path));
-            maybe_add_source(
-                &mut results,
-                cfg.custom.new_sources_url(id.clone(), url.clone()).await,
-            )?;
+            results.push(cfg.custom.new_sources_url(id.clone(), url.clone()).await?);
             info!("Configured source {id} from URL {}", sanitize_url(&url));
         } else {
             let is_dir = path.is_dir();
@@ -351,7 +373,7 @@ async fn resolve_int<T: SourceConfigExtras>(
                 info!("Configured source {id} from {}", can.display());
                 files.insert(can);
                 configs.insert(id.clone(), FileConfigSrc::Path(path.clone()));
-                maybe_add_source(&mut results, cfg.custom.new_sources(id, path).await)?;
+                results.push(cfg.custom.new_sources(id, path).await?);
             }
         }
     }