chore(core) final push in the pg-move (#2192)

CommanderStorm · web-flow · commit 6e870e510725 · 2025-09-16T17:55:29.000Z
This is the final push in the postgres move to core.

For the function/table resolver, this needs a bit of an refactoring
before these would be publicly usefull. I am clasifying them as
"configuration"-stuff and moving them to the config-file area.

I am not sure if this is the final resting place, but for now this seems
an OK place to push this to.

This PR also fixes two unrelated issues:
- some of our constants are `static` instead of `const`
- the `postgres` configuration module (in the config file "postgres") is
named `pg`
diff --git a/martin/src/config/args/mod.rs b/martin/src/config/args/mod.rs
@@ -2,9 +2,9 @@ mod connections;
 pub use connections::State;
 
 #[cfg(feature = "postgres")]
-mod pg;
+mod postgres;
 #[cfg(feature = "postgres")]
-pub use pg::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT, PgArgs};
+pub use postgres::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT, PgArgs};
 
 mod root;
 pub use root::*;
diff --git a/martin/src/config/args/postgres.rs b/martin/src/config/args/postgres.rs
@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use super::connections::Arguments;
 use super::connections::State::{Ignore, Take};
 use crate::config::file::UnrecognizedValues;
-use crate::config::file::pg::{POOL_SIZE_DEFAULT, PgConfig, PgSslCerts};
+use crate::config::file::postgres::{POOL_SIZE_DEFAULT, PgConfig, PgSslCerts};
 // Must match the help string for BoundsType::Quick
 pub const DEFAULT_BOUNDS_TIMEOUT: Duration = Duration::from_secs(5);
 
diff --git a/martin/src/config/args/root.rs b/martin/src/config/args/root.rs
@@ -10,6 +10,8 @@ use super::connections::Arguments;
 use super::srv::SrvArgs;
 use crate::MartinError::ConfigAndConnectionsError;
 use crate::MartinResult;
+#[cfg(feature = "postgres")]
+use crate::config::args::PgArgs;
 use crate::config::file::Config;
 #[cfg(any(
     feature = "cog",
@@ -45,7 +47,7 @@ pub struct Args {
     pub srv: SrvArgs,
     #[cfg(feature = "postgres")]
     #[command(flatten)]
-    pub pg: Option<super::pg::PgArgs>,
+    pub pg: Option<PgArgs>,
 }
 
 // None of these params will be transferred to the config
@@ -231,6 +233,8 @@ mod tests {
     fn cli_with_config() {
         use martin_core::config::OptOneMany;
 
+        use crate::config::file::postgres::PgConfig;
+
         let args = parse(&["martin", "--config", "c.toml"]).unwrap();
         let meta = MetaArgs {
             config: Some(PathBuf::from("c.toml")),
@@ -248,7 +252,7 @@ mod tests {
 
         let args = parse(&["martin", "postgres://connection"]).unwrap();
         let cfg = Config {
-            postgres: OptOneMany::One(crate::config::file::pg::PgConfig {
+            postgres: OptOneMany::One(PgConfig {
                 connection_string: Some("postgres://connection".to_string()),
                 ..Default::default()
             }),
diff --git a/martin/src/config/file/main.rs b/martin/src/config/file/main.rs
@@ -50,7 +50,7 @@ pub struct Config {
 
     #[cfg(feature = "postgres")]
     #[serde(default, skip_serializing_if = "OptOneMany::is_none")]
-    pub postgres: OptOneMany<super::pg::PgConfig>,
+    pub postgres: OptOneMany<super::postgres::PgConfig>,
 
     #[cfg(feature = "pmtiles")]
     #[serde(default, skip_serializing_if = "FileConfigEnum::is_none")]
diff --git a/martin/src/config/file/mod.rs b/martin/src/config/file/mod.rs
@@ -12,10 +12,10 @@ pub mod cog;
 pub mod fonts;
 #[cfg(feature = "mbtiles")]
 pub mod mbtiles;
-#[cfg(feature = "postgres")]
-pub mod pg;
 #[cfg(feature = "pmtiles")]
 pub mod pmtiles;
+#[cfg(feature = "postgres")]
+pub mod postgres;
 #[cfg(feature = "sprites")]
 pub mod sprites;
 #[cfg(feature = "styles")]
diff --git a/martin/src/config/file/pg/utils.rs b/martin/src/config/file/pg/utils.rs
diff --git a/martin/src/config/file/postgres/builder.rs b/martin/src/config/file/postgres/builder.rs
@@ -1,5 +1,5 @@
 use std::cmp::Ordering;
-use std::collections::{BTreeMap, HashSet};
+use std::collections::HashSet;
 
 use futures::future::join_all;
 use itertools::Itertools as _;
@@ -10,20 +10,16 @@ use martin_core::tiles::BoxedSource;
 use martin_core::tiles::postgres::{PgError, PgPool, PgResult, PgSource, PgSqlInfo};
 
 use crate::config::args::BoundsCalcType;
-use crate::config::file::pg::{
-    FuncInfoSources, FunctionInfo, POOL_SIZE_DEFAULT, PgCfgPublish, PgCfgPublishFuncs, PgConfig,
-    PgInfo, TableInfo, TableInfoSources,
+use crate::config::file::postgres::resolver::{
+    query_available_function, query_available_tables, table_to_query,
+};
+use crate::config::file::postgres::utils::{find_info, find_kv_ignore_case, normalize_key};
+use crate::config::file::postgres::{
+    FuncInfoSources, POOL_SIZE_DEFAULT, PgCfgPublish, PgCfgPublishFuncs, PgConfig, PgInfo,
+    TableInfo, TableInfoSources,
 };
-use crate::pg::query_functions::query_available_function;
-use crate::pg::query_tables::{query_available_tables, table_to_query};
-use crate::pg::utils::{find_info, find_kv_ignore_case, normalize_key};
 use crate::utils::IdResolver;
 
-/// Map of `PostgreSQL` functions organized by schema and function name.
-pub type SqlFuncInfoMapMap = BTreeMap<String, BTreeMap<String, (PgSqlInfo, FunctionInfo)>>;
-/// Map of `PostgreSQL` tables organized by schema, table, and geometry column.
-pub type SqlTableInfoMapMapMap = BTreeMap<String, BTreeMap<String, BTreeMap<String, TableInfo>>>;
-
 /// Builder for auto-discovering `PostgreSQL` tile sources.
 #[derive(Debug)]
 pub struct PgBuilder {
@@ -118,11 +114,13 @@ impl PgBuilder {
     }
 
     /// Returns the bounds calculation type for this builder.
+    #[must_use]
     pub fn auto_bounds(&self) -> BoundsCalcType {
         self.auto_bounds
     }
 
     /// ID under which this [`PgBuilder`] is identified externally
+    #[must_use]
     pub fn get_id(&self) -> &str {
         self.pool.get_id()
     }
@@ -131,7 +129,6 @@ impl PgBuilder {
     #[allow(clippy::too_many_lines)]
     pub async fn instantiate_tables(&self) -> PgResult<(Vec<BoxedSource>, TableInfoSources)> {
         // FIXME: this function has gotten too long due to the new formatting rules, need to be refactored
-        //
         let mut db_tables_info = query_available_tables(&self.pool).await?;
 
         // Match configured sources with the discovered ones and add them to the pending list.
diff --git a/martin/src/config/file/postgres/config.rs b/martin/src/config/file/postgres/config.rs
@@ -14,10 +14,10 @@ use tokio::time::timeout;
 use super::{FuncInfoSources, TableInfoSources};
 use crate::MartinResult;
 use crate::config::args::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT};
+use crate::config::file::postgres::PgBuilder;
 use crate::config::file::{
     ConfigExtras, UnrecognizedKeys, UnrecognizedValues, copy_unrecognized_keys_from_config,
 };
-use crate::pg::builder::PgBuilder;
 use crate::utils::IdResolver;
 
 pub trait PgInfo {
@@ -324,7 +324,7 @@ mod tests {
     use tilejson::Bounds;
 
     use super::*;
-    use crate::config::file::pg::{FunctionInfo, TableInfo};
+    use crate::config::file::postgres::{FunctionInfo, TableInfo};
     use crate::config::file::{Config, parse_config};
 
     pub fn parse_cfg(yaml: &str) -> Config {
diff --git a/martin/src/config/file/postgres/config_function.rs b/martin/src/config/file/postgres/config_function.rs
@@ -5,7 +5,7 @@ use tilejson::{Bounds, TileJSON};
 
 use super::config::PgInfo;
 use crate::config::file::UnrecognizedValues;
-use crate::config::file::pg::utils::patch_json;
+use crate::config::file::postgres::utils::patch_json;
 
 pub type FuncInfoSources = BTreeMap<String, FunctionInfo>;
 
diff --git a/martin/src/config/file/postgres/config_table.rs b/martin/src/config/file/postgres/config_table.rs
@@ -6,8 +6,7 @@ use tilejson::{Bounds, TileJSON, VectorLayer};
 
 use super::PgInfo;
 use crate::config::file::UnrecognizedValues;
-use crate::config::file::pg::utils::patch_json;
-use crate::pg::utils::normalize_key;
+use crate::config::file::postgres::utils::{normalize_key, patch_json};
 
 pub type TableInfoSources = BTreeMap<String, TableInfo>;
 
diff --git a/martin/src/config/file/postgres/mod.rs b/martin/src/config/file/postgres/mod.rs
@@ -7,4 +7,9 @@ pub use config_function::*;
 mod config_table;
 pub use config_table::*;
 
-mod utils;
+pub(crate) mod utils;
+
+mod builder;
+pub use builder::PgBuilder;
+
+pub mod resolver;
diff --git a/martin/src/config/file/postgres/resolver/mod.rs b/martin/src/config/file/postgres/resolver/mod.rs
@@ -0,0 +1,5 @@
+mod query_functions;
+mod query_tables;
+
+pub use query_functions::query_available_function;
+pub use query_tables::{query_available_tables, table_to_query};
diff --git a/martin/src/config/file/postgres/resolver/query_functions.rs b/martin/src/config/file/postgres/resolver/query_functions.rs
@@ -1,5 +1,6 @@
 //! `PostgreSQL` function discovery and validation.
 
+use std::collections::BTreeMap;
 use std::fmt::Write as _;
 use std::iter::zip;
 
@@ -9,8 +10,10 @@ use martin_core::tiles::postgres::{PgPool, PgResult, PgSqlInfo};
 use postgres_protocol::escape::escape_identifier;
 use serde_json::Value;
 
-use crate::config::file::pg::FunctionInfo;
-use crate::pg::builder::SqlFuncInfoMapMap;
+use crate::config::file::postgres::FunctionInfo;
+
+/// Map of `PostgreSQL` functions organized by schema and function name.
+pub type SqlFuncInfoMapMap = BTreeMap<String, BTreeMap<String, (PgSqlInfo, FunctionInfo)>>;
 
 /// Queries the database for available tile-generating functions.
 ///
diff --git a/martin/src/config/file/postgres/resolver/query_tables.rs b/martin/src/config/file/postgres/resolver/query_tables.rs
@@ -1,6 +1,6 @@
 //! `PostgreSQL` table discovery and validation.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 
 use futures::pin_mut;
 use log::{debug, warn};
@@ -14,12 +14,14 @@ use tilejson::Bounds;
 use tokio::time::timeout;
 
 use crate::config::args::{BoundsCalcType, DEFAULT_BOUNDS_TIMEOUT};
-use crate::config::file::pg::{PgInfo, TableInfo};
-use crate::pg::builder::SqlTableInfoMapMapMap;
+use crate::config::file::postgres::{PgInfo, TableInfo};
 
-static DEFAULT_EXTENT: u32 = 4096;
-static DEFAULT_BUFFER: u32 = 64;
-static DEFAULT_CLIP_GEOM: bool = true;
+/// Map of `PostgreSQL` tables organized by schema, table, and geometry column.
+pub type SqlTableInfoMapMapMap = BTreeMap<String, BTreeMap<String, BTreeMap<String, TableInfo>>>;
+
+const DEFAULT_EXTENT: u32 = 4096;
+const DEFAULT_BUFFER: u32 = 64;
+const DEFAULT_CLIP_GEOM: bool = true;
 
 /// Queries the database for available tables with geometry columns.
 pub async fn query_available_tables(pool: &PgPool) -> PgResult<SqlTableInfoMapMapMap> {
diff --git a/martin/src/config/file/postgres/resolver/scripts/query_available_function.sql b/martin/src/config/file/postgres/resolver/scripts/query_available_function.sql
diff --git a/martin/src/config/file/postgres/resolver/scripts/query_available_tables.sql b/martin/src/config/file/postgres/resolver/scripts/query_available_tables.sql
diff --git a/martin/src/config/file/postgres/utils.rs b/martin/src/config/file/postgres/utils.rs
@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
 
 use itertools::Itertools;
 use log::{error, info, warn};
+use tilejson::TileJSON;
 
 #[must_use]
 pub fn normalize_key<T>(
@@ -87,3 +88,31 @@ pub fn find_kv_ignore_case<'a, T>(
         Err(multiple)
     }
 }
+
+#[must_use]
+pub fn patch_json(target: TileJSON, patch: Option<&serde_json::Value>) -> TileJSON {
+    let Some(tj) = patch else {
+        // Nothing to merge in, keep the original
+        return target;
+    };
+    // Not the most efficient, but this is only executed once per source:
+    // * Convert the TileJSON struct to a serde_json::Value
+    // * Merge the self.tilejson into the value
+    // * Convert the merged value back to a TileJSON struct
+    // * In case of errors, return the original tilejson
+    let mut tilejson2 = match serde_json::to_value(target.clone()) {
+        Ok(v) => v,
+        Err(e) => {
+            error!("Failed to serialize tilejson, unable to merge function comment: {e}");
+            return target;
+        }
+    };
+    json_patch::merge(&mut tilejson2, tj);
+    match serde_json::from_value(tilejson2.clone()) {
+        Ok(v) => v,
+        Err(e) => {
+            error!("Failed to deserialize merged function comment tilejson: {e}");
+            target
+        }
+    }
+}
diff --git a/martin/src/lib.rs b/martin/src/lib.rs
@@ -9,8 +9,6 @@ pub use source::TileSources;
 mod utils;
 pub use utils::{IdResolver, MartinError, MartinResult};
 
-#[cfg(feature = "postgres")]
-pub mod pg;
 pub mod srv;
 
 // Ensure README.md contains valid code
diff --git a/martin/src/pg/mod.rs b/martin/src/pg/mod.rs
diff --git a/martin/src/srv/tiles.rs b/martin/src/srv/tiles.rs
@@ -22,7 +22,7 @@ use crate::config::file::srv::SrvConfig;
 use crate::source::TileSources;
 use crate::srv::server::map_internal_error;
 
-static SUPPORTED_ENC: &[HeaderEnc] = &[
+const SUPPORTED_ENC: &[HeaderEnc] = &[
     HeaderEnc::gzip(),
     HeaderEnc::brotli(),
     HeaderEnc::identity(),
diff --git a/martin/tests/utils/pg_utils.rs b/martin/tests/utils/pg_utils.rs
@@ -1,6 +1,6 @@
 use indoc::formatdoc;
 #[cfg(feature = "postgres")]
-use martin::config::file::pg::TableInfo;
+use martin::config::file::postgres::TableInfo;
 use martin::config::file::{Config, ServerState};
 use martin_core::tiles::BoxedSource;
 
diff --git a/tests/test.sh b/tests/test.sh
@@ -403,8 +403,8 @@ test_pbf table_and_view_two_schemas2_0_0_0  table_and_view_two_schemas.1/0/0/0
 
 kill_process "$MARTIN_PROC_ID" Martin
 
-test_log_has_str "$LOG_FILE" 'WARN  martin::pg::query_tables] Table public.table_source has no spatial index on column geom'
-test_log_has_str "$LOG_FILE" 'WARN  martin::pg::query_tables] Table public.table_source_geog has no spatial index on column geog'
+test_log_has_str "$LOG_FILE" 'WARN  martin::config::file::postgres::resolver::query_tables] Table public.table_source has no spatial index on column geom'
+test_log_has_str "$LOG_FILE" 'WARN  martin::config::file::postgres::resolver::query_tables] Table public.table_source_geog has no spatial index on column geog'
 test_log_has_str "$LOG_FILE" 'WARN  martin_core::resources::fonts] Ignoring duplicate font Overpass Mono Regular from tests'
 test_log_has_str "$LOG_FILE" 'was renamed to `stamen_toner__raster_CC-BY-ODbL_z3`'
 test_log_has_str "$LOG_FILE" 'was renamed to `table_source_multiple_geom.1`'
@@ -515,8 +515,8 @@ test_jsn fnc_comment_cfg  function_Mixed_Name
 test_metrics "metrics_1"
 
 kill_process "$MARTIN_PROC_ID" Martin
-test_log_has_str "$LOG_FILE" 'WARN  martin::pg::query_tables] Table public.table_source has no spatial index on column geom'
-test_log_has_str "$LOG_FILE" 'WARN  martin::pg::query_tables] Table public.table_source_geog has no spatial index on column geog'
+test_log_has_str "$LOG_FILE" 'WARN  martin::config::file::postgres::resolver::query_tables] Table public.table_source has no spatial index on column geom'
+test_log_has_str "$LOG_FILE" 'WARN  martin::config::file::postgres::resolver::query_tables] Table public.table_source_geog has no spatial index on column geog'
 test_log_has_str "$LOG_FILE" 'WARN  martin_core::resources::fonts] Ignoring duplicate font Overpass Mono Regular from tests'
 test_log_has_str "$LOG_FILE" "WARN  martin::config::file::main] Ignoring unrecognized configuration key 'warning'. Please check your configuration file for typos."
 test_log_has_str "$LOG_FILE" "WARN  martin::config::file::main] Ignoring unrecognized configuration key 'observability.warning'. Please check your configuration file for typos."
