test(mbtiles): Generate mbtiles dynamically from SQL files to increase debuggability and transparency

.dockerignore

@@ -33,6 +33,7 @@ tests/mbtiles_temp_files/
 tests/output/
 tmp/
 tsconfig.tsbuildinfo
+*.mbtiles
 
 ####
 #### Above content must match .gitignore ####

.github/workflows/ci.yml

@@ -463,7 +463,10 @@ jobs:
       - name: Set up gdal-bin and sqlite3-tools
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt-get update && sudo apt-get install -y gdal-bin sqlite3-tools
-
+      - name: Install sqlite3 and sqldiff on Windows
+        if: matrix.os == 'windows-latest'
+        run: choco install sqlite --yes
+        shell: powershell
       - name: Install and run Postgis (Windows)
         if: matrix.os == 'windows-latest'
         uses: nyurik/action-setup-postgis@5183a5e26ade31809faed263cd4d638e144fe186 # v2.2

.gitignore

@@ -21,6 +21,7 @@ tests/mbtiles_temp_files/
 tests/output/
 tmp/
 tsconfig.tsbuildinfo
+*.mbtiles
 
 ####
 #### Above content must match .dockerignore ####

.pre-commit-config.yaml

@@ -47,13 +47,23 @@ repos:
   - repo: https://github.com/sqlfluff/sqlfluff
     rev: 3.4.2
     hooks:
+        # 1. Postgres dialect for general .sql files, excluding sqlite dirs
       - id: sqlfluff-fix
         # AL07: Avoid table aliases in from clauses and join conditions <- we sometimes do this for long names
         # LT05: Line is too long <- not useful. mostly triggers for comments
         # LT12: Files must end with a single trailing newline <- already done by trailing-whitespace
         # If you update this rule, please also update the rules in justfile, the fmt-sql recipe
         args: [ --dialect=postgres, "--exclude-rules=AL07,LT05,LT12" ]
         files: \.sql$
+        exclude: ^tests/fixtures/(mbtiles|files)/.*\.sql$
+
+        # 2. SQLite dialect for .sql files in specific directories
+      - id: sqlfluff-fix
+        # LT01: Layout spacing adds space between x and 'blob' breaking the blobs
+        args: [--dialect=sqlite, "--exclude-rules=LT01,LT05"]
+        files: ^tests/fixtures/(mbtiles|files)/.*\.sql$
+
+  # TODO: Add NPM hooks for formatting and linting
   # NPM hooks for JavaScript/TypeScript formatting and linting
   - repo: https://github.com/biomejs/pre-commit
     rev: "v2.2.4"  # Use the sha / tag you want to point at

docs/src/mbtiles-copy.md

@@ -13,7 +13,7 @@ This command can also be used to generate files of different [supported schema](
 
 ```bash
 mbtiles copy normalized.mbtiles dst.mbtiles \
-         --dst-mbttype flat-with-hash
+         --dst-type flat-with-hash
 ```
 
 ## `mbtiles copy --diff-with-file`

docs/src/mbtiles-meta.md

@@ -7,6 +7,11 @@ number of tiles per zoom level, the size of the smallest and largest tiles, and
 level. The command will also print the bounding box of the covered area per zoom level.
 
 ```bash
+# Create a sample .mbtiles file in some dir
+sqlite3 target/world_cities.mbtiles < tests/fixtures/mbtiles/world_cities.sql
+# Get mbtiles summary
+mbtiles summary target/world_cities.mbtiles
+
 MBTiles file summary for tests/fixtures/mbtiles/world_cities.mbtiles
 Schema: flat
 File size: 48.00KiB

docs/src/mbtiles-schema.md

@@ -7,78 +7,27 @@ The `mbtiles` tool builds on top of the original [MBTiles specification](https:/
 Flat schema is the closest to the original MBTiles specification. It stores all tiles in a single table. This schema is the most efficient when the tileset contains no duplicate tiles.
 
 ```sql, ignore
-CREATE TABLE tiles (
-    zoom_level  INTEGER,
-    tile_column INTEGER,
-    tile_row    INTEGER,
-    tile_data   BLOB);
-
-CREATE UNIQUE INDEX tile_index on tiles (
-    zoom_level, tile_column, tile_row);
+{{#include ../../mbtiles/sql/init-flat.sql}}
 ```
 
 ## flat-with-hash
 
 Similar to the `flat` schema, but also includes a `tile_hash` column that contains a hash value of the `tile_data` column. Use this schema when the tileset has no duplicate tiles, but you still want to be able to validate the content of each tile individually.
 
 ```sql, ignore
-CREATE TABLE tiles_with_hash (
-    zoom_level  INTEGER NOT NULL,
-    tile_column INTEGER NOT NULL,
-    tile_row    INTEGER NOT NULL,
-    tile_data   BLOB,
-    tile_hash   TEXT);
-
-CREATE UNIQUE INDEX tiles_with_hash_index on tiles_with_hash (
-    zoom_level, tile_column, tile_row);
-
-CREATE VIEW tiles AS
-    SELECT zoom_level, tile_column, tile_row, tile_data
-    FROM tiles_with_hash;
+{{#include ../../mbtiles/sql/init-flat-with-hash.sql}}
 ```
 
 ## normalized
 
 Normalized schema is the most efficient when the tileset contains duplicate tiles. It stores all tile blobs in the `images` table, and stores the tile Z,X,Y coordinates in a `map` table. The `map` table contains a `tile_id` column that is a foreign key to the `images` table. The `tile_id` column is a hash of the `tile_data` column, making it possible to both validate each individual tile like in the `flat-with-hash` schema, and also to optimize storage by storing each unique tile only once.
 
 ```sql, ignore
-CREATE TABLE map (
-    zoom_level  INTEGER,
-    tile_column INTEGER,
-    tile_row    INTEGER,
-    tile_id     TEXT);
-
-CREATE TABLE images (
-    tile_id   TEXT,
-    tile_data BLOB);
-
-CREATE UNIQUE INDEX map_index ON map (
-    zoom_level, tile_column, tile_row);
-CREATE UNIQUE INDEX images_id ON images (
-    tile_id);
-
-CREATE VIEW tiles AS
-    SELECT
-        map.zoom_level   AS zoom_level,
-        map.tile_column  AS tile_column,
-        map.tile_row     AS tile_row,
-        images.tile_data AS tile_data
-    FROM
-        map JOIN images
-        ON images.tile_id = map.tile_id;
+{{#include ../../mbtiles/sql/init-normalized.sql}}
 ```
 
 Optionally, `.mbtiles` files with `normalized` schema can include a `tiles_with_hash` view. All `normalized` files created by the `mbtiles` tool will contain this view.
 
 ```sql, ignore
-CREATE VIEW tiles_with_hash AS
-    SELECT
-        map.zoom_level   AS zoom_level,
-        map.tile_column  AS tile_column,
-        map.tile_row     AS tile_row,
-        images.tile_data AS tile_data,
-        images.tile_id   AS tile_hash
-    FROM
-        map JOIN images
-        ON map.tile_id = images.tile_id;
+{{#include ../../mbtiles/sql/init-normalized.sql:26:}}
 ```

docs/src/quick-start-linux.md

@@ -5,7 +5,13 @@ mkdir martin
 cd martin
 
 # Download some sample data
-curl -L -O https://github.com/maplibre/martin/raw/main/tests/fixtures/mbtiles/world_cities.mbtiles
+curl -L -O https://github.com/maplibre/martin/raw/main/tests/fixtures/mbtiles/world_cities.sql
+
+# Check if sqlite is installed
+sqlite3 --version
+
+# Initialize a database
+sqlite3 world_cities.mbtiles < world_cities.sql
 
 # Download the latest version of Martin binary, extract it, and make it executable
 curl -L -O https://github.com/maplibre/martin/releases/latest/download/martin-x86_64-unknown-linux-gnu.tar.gz

docs/src/quick-start-macos.md

@@ -1,6 +1,6 @@
 ## Quick start on macOS
 
-1. Download some [demo tiles](https://github.com/maplibre/martin/raw/main/tests/fixtures/mbtiles/world_cities.mbtiles).
+1. Download some [demo tiles](https://github.com/maplibre/martin/raw/main/tests/fixtures/mbtiles/world_cities.sql).
 
 2. Download the latest version of Martin from
    the [release page](https://github.com/maplibre/martin/releases/latest).
@@ -10,11 +10,14 @@
 
 3. Extract content of both files and place them in a same directory.
 
-4. Open the command prompt and navigate to the directory where `martin` and `world_cities.mbtiles` are located.
+4. Open the command prompt and navigate to the directory where `martin` and `world_cities.sql` are located.
 
 5. Run the following command to start Martin with the demo data:
 
 ```bash
+# Initialize a database
+sqlite3 world_cities.mbtiles < world_cities.sql
+
 # Show Martin help screen
 ./martin --help
 

docs/src/quick-start-windows.md

@@ -1,17 +1,23 @@
 ## Quick start on Windows
 
-1. Download some [demo tiles](https://github.com/maplibre/martin/raw/main/tests/fixtures/mbtiles/world_cities.mbtiles).
+1. Download some [demo tiles](https://github.com/maplibre/martin/raw/main/tests/fixtures/mbtiles/world_cities.sql).
 
 2. Download the latest Windows version of Martin from
    the [release page](https://github.com/maplibre/martin/releases):  [martin-x86_64-pc-windows-msvc.zip](https://github.com/maplibre/martin/releases/latest/download/martin-x86_64-pc-windows-msvc.zip)
 
 3. Extract content of both files and place them in a same directory.
 
-4. Open the command prompt and navigate to the directory where `martin` and `world_cities.mbtiles` are located.
+4. Open the command prompt and navigate to the directory where `martin` and `world_cities.sql` are located.
 
 5. Run the following command to start Martin with the demo data:
 
 ```bash
+# Check if sqlite is installed
+sqlite3 --version
+
+# Initialize a database
+sqlite3 world_cities.mbtiles < world_cities.sql
+
 # Show Martin help screen
 martin --help
 

justfile

@@ -204,7 +204,8 @@ fmt-md:
 
 # Reformat all SQL files using docker
 fmt-sql:
-    docker run -it --rm -v $PWD:/sql sqlfluff/sqlfluff:latest fix --dialect=postgres --exclude-rules=AL07,LT05,LT12
+    docker run -it --rm -v $PWD:/sql sqlfluff/sqlfluff:latest fix --dialect=postgres --exclude-rules=AL07,LT05,LT12 --exclude '^tests/fixtures/(mbtiles|files)/.*\.sql$'
+    docker run -it --rm -v $PWD:/sql sqlfluff/sqlfluff:latest fix --dialect=sqlite --exclude-rules=LT01,LT05 --files '^tests/fixtures/(mbtiles|files)/.*\.sql$'
 
 # Reformat all Cargo.toml files using cargo-sort
 fmt-toml *args: (cargo-install 'cargo-sort')
@@ -328,7 +329,7 @@ test-int: clean-test install-sqlx
             echo "** If this is expected, run 'just bless' to update expected output"
             echo ""
             echo "::group::Resulting diff (max 100 lines)"
-            diff --recursive --new-file --exclude='*.pbf' tests/output tests/expected | head -n 100
+            diff --recursive --new-file --exclude='*.pbf' tests/output tests/expected | head -n 100 | cat --show-nonprinting
             echo "::endgroup::"
             exit 1
         else

martin/src/config/args/root.rs

@@ -174,6 +174,23 @@ fn is_url(s: &str, extension: &[&str]) -> bool {
     }
 }
 
+/// Check if a string is a `file:` scheme URI with a specified extension.
+///
+/// This is used for `SQLite` connection strings like `file:name.mbtiles?mode=memory&cache=shared`
+#[cfg(any(feature = "cog", feature = "mbtiles", feature = "pmtiles"))]
+fn is_file_scheme_uri(s: &str, extensions: &[&str]) -> bool {
+    let Ok(url) = url::Url::parse(s) else {
+        return false;
+    };
+    if url.scheme() != "file" {
+        return false;
+    }
+    url.path()
+        .rsplit('.')
+        .next()
+        .is_some_and(|ext| extensions.contains(&ext))
+}
+
 #[cfg(any(feature = "cog", feature = "mbtiles", feature = "pmtiles"))]
 pub fn parse_file_args<T: crate::config::file::ConfigExtras>(
     cli_strings: &mut Arguments,
@@ -186,6 +203,9 @@ pub fn parse_file_args<T: crate::config::file::ConfigExtras>(
         let path = PathBuf::from(s);
         if allow_url && is_url(s, extensions) {
             Take(path)
+        } else if is_file_scheme_uri(s, extensions) {
+            // Handle file: scheme URIs (SQLite connection strings) as valid paths
+            Take(path)
         } else if path.is_dir() {
             Share(path)
         } else if path.is_file()
@@ -285,6 +305,33 @@ mod tests {
         assert_eq!(config4.unwrap().0.srv.preferred_encoding, None);
     }
 
+    #[cfg(any(feature = "cog", feature = "mbtiles", feature = "pmtiles"))]
+    #[test]
+    fn test_is_file_scheme_uri() {
+        // Valid file scheme URIs
+        assert!(is_file_scheme_uri("file:test.mbtiles", &["mbtiles"]));
+        assert!(is_file_scheme_uri(
+            "file:test.mbtiles?mode=memory&cache=shared",
+            &["mbtiles"]
+        ));
+        assert!(is_file_scheme_uri(
+            "file:/path/to/test.mbtiles",
+            &["mbtiles"]
+        ));
+        assert!(is_file_scheme_uri("file:data.pmtiles", &["pmtiles"]));
+        assert!(is_file_scheme_uri("file:image.tiff", &["tiff", "tif"]));
+
+        // Invalid cases
+        assert!(!is_file_scheme_uri(
+            "http://example.com/test.mbtiles",
+            &["mbtiles"]
+        ));
+        assert!(!is_file_scheme_uri("test.mbtiles", &["mbtiles"]));
+        assert!(!is_file_scheme_uri("file:test.txt", &["mbtiles"]));
+        assert!(!is_file_scheme_uri("file:", &["mbtiles"]));
+        assert!(!is_file_scheme_uri("", &["mbtiles"]));
+    }
+
     #[test]
     fn cli_bad_arguments() {
         for params in [
@@ -318,23 +365,26 @@ mod tests {
     }
 
     #[cfg(all(feature = "pmtiles", feature = "mbtiles", feature = "cog"))]
-    #[test]
-    fn cli_multiple_extensions() {
+    #[tokio::test]
+    async fn cli_multiple_extensions() {
+        use std::ffi::OsString;
+
+        let script = include_str!("../../../../tests/fixtures/mbtiles/json.sql");
+        let (_mbt, _conn, file) = mbtiles::temp_named_mbtiles("json.mbtiles", script).await;
         let args = Args::parse_from([
-            "martin",
-            "../tests/fixtures/pmtiles/png.pmtiles",
-            "../tests/fixtures/mbtiles/json.mbtiles",
-            "../tests/fixtures/cog/rgba_u8_nodata.tiff",
-            "../tests/fixtures/cog/rgba_u8.tif",
+            OsString::from("martin"),
+            OsString::from("../tests/fixtures/pmtiles/png.pmtiles"),
+            file.as_os_str().to_owned(),
+            OsString::from("../tests/fixtures/cog/rgba_u8_nodata.tiff"),
+            OsString::from("../tests/fixtures/cog/rgba_u8.tif"),
         ]);
 
         let env = FauxEnv::default();
         let mut config = Config::default();
-        let err = args.merge_into_config(&mut config, &env);
-        assert!(err.is_ok());
+        args.merge_into_config(&mut config, &env).unwrap();
         insta::assert_yaml_snapshot!(config, @r#"
         pmtiles: "../tests/fixtures/pmtiles/png.pmtiles"
-        mbtiles: "../tests/fixtures/mbtiles/json.mbtiles"
+        mbtiles: "file:json.mbtiles?mode=memory&cache=shared"
         cog:
           - "../tests/fixtures/cog/rgba_u8_nodata.tiff"
           - "../tests/fixtures/cog/rgba_u8.tif"

martin/src/config/file/file_config.rs

@@ -11,9 +11,8 @@ use martin_core::tiles::BoxedSource;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
-use crate::config::file::ConfigFileError::{
-    InvalidFilePath, InvalidSourceFilePath, InvalidSourceUrl, IoError,
-};
+use crate::config::file::ConfigFileError::{InvalidFilePath, InvalidSourceUrl, IoError};
+
 use crate::{MartinError, MartinResult};
 
 pub type ConfigFileResult<T> = Result<T, ConfigFileError>;
@@ -244,10 +243,24 @@ impl FileConfigSrc {
 
     pub fn abs_path(&self) -> ConfigFileResult<PathBuf> {
         let path = self.get_path();
+
+        if is_sqlite_memory_uri(path) {
+            // Skip canonicalization for in-memory DB URIs
+            return Ok(path.clone());
+        }
+
         path.canonicalize().map_err(|e| IoError(e, path.clone()))
     }
 }
 
+fn is_sqlite_memory_uri(path: &Path) -> bool {
+    if let Some(s) = path.to_str() {
+        s.starts_with("file:") && s.contains("mode=memory") && s.contains("cache=shared")
+    } else {
+        false
+    }
+}
+
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct FileConfigSource {
     pub path: PathBuf,
@@ -289,11 +302,7 @@ async fn resolve_int<T: SourceConfigExtras>(
             } else {
                 let can = source.abs_path()?;
                 if !can.is_file() {
-                    // todo: maybe warn instead?
-                    return Err(MartinError::ConfigFileError(InvalidSourceFilePath(
-                        id.to_string(),
-                        can,
-                    )));
+                    log::warn!("The file: {} does not exist", can.display());
                 }
 
                 let dup = !files.insert(can.clone());