Add support for parsing MLT-format vector tile sources (#3246)

Co-authored-by: Bart Louwers <[email protected]>
Co-authored-by: Bart Louwers <[email protected]>

Author: TimSylvester

.gitmodules

@@ -100,6 +100,10 @@
 [submodule "vendor/filesystem"]
 	path = vendor/filesystem
 	url = https://github.com/gulrak/filesystem.git
+[submodule "vendor/maplibre-tile-spec"]
+	path = vendor/maplibre-tile-spec
+	url = https://github.com/maplibre/maplibre-tile-spec.git
+	branch = cpp
 [submodule "vendor/kdbush.hpp"]
 	path = vendor/kdbush.hpp
 	url = https://github.com/mourner/kdbush.hpp.git

BUILD.bazel

@@ -144,6 +144,7 @@ cc_library(
         "//vendor:eternal",
         "//vendor:expected-lite",
         "//vendor:maplibre-native-base",
+        "//vendor/maplibre-tile-spec/cpp:mlt_cpp",
         "//vendor:parsedate",
         "//vendor:pmtiles",
         "//vendor:polylabel",

CMakeLists.txt

@@ -302,9 +302,9 @@ list(APPEND INCLUDE_FILES
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/light.hpp
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/position.hpp
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/property_value.hpp
-    ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/raster_dem_options.hpp
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/rotation.hpp
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/source.hpp
+    ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/source_options.hpp
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/tileset.hpp
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/conversion/transition_options.hpp
     ${PROJECT_SOURCE_DIR}/include/mbgl/style/expression/assertion.hpp
@@ -676,9 +676,9 @@ list(APPEND SRC_FILES
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/light.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/position.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/property_value.cpp
-    ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/raster_dem_options.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/rotation.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/source.cpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/source_options.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/stringify.hpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/tileset.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/style/conversion/transition_options.cpp
@@ -879,10 +879,15 @@ list(APPEND SRC_FILES
     ${PROJECT_SOURCE_DIR}/src/mbgl/tile/tile_operation.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_tile.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_tile.hpp
-    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_tile_data.cpp
-    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_tile_data.hpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mlt_tile.cpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mlt_tile.hpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mlt_tile_data.cpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mlt_tile_data.hpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mvt_tile.cpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mvt_tile.hpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mvt_tile_data.cpp
+    ${PROJECT_SOURCE_DIR}/src/mbgl/tile/vector_mvt_tile_data.hpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/util/action_journal.cpp
-    ${PROJECT_SOURCE_DIR}/src/mbgl/util/action_journal_impl.hpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/util/action_journal_impl.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/util/camera.cpp
     ${PROJECT_SOURCE_DIR}/src/mbgl/util/bounding_volumes.hpp
@@ -1553,6 +1558,7 @@ include(${PROJECT_SOURCE_DIR}/vendor/unordered_dense.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/vector-tile.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/wagyu.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/metal-cpp.cmake)
+include(${PROJECT_SOURCE_DIR}/vendor/maplibre-tile-spec.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/supercluster.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/expected-lite.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/kdbush.cmake)
@@ -1581,6 +1587,7 @@ target_link_libraries(
         mbgl-vendor-unique_resource
         mbgl-vendor-vector-tile
         mbgl-vendor-wagyu
+	    mlt-cpp
         $<$<BOOL:${MLN_WITH_METAL}>:mbgl-vendor-metal-cpp>
         $<IF:$<BOOL:${MLN_USE_RUST}>,mbgl-rustutils,mbgl-vendor-csscolorparser>
         $<$<BOOL:${MLN_TEXT_SHAPING_HARFBUZZ}>:mbgl-freetype>
@@ -1614,6 +1621,8 @@ set(EXPORT_TARGETS
     mbgl-vendor-boost
     mbgl-vendor-earcut.hpp
     mbgl-vendor-eternal
+    mbgl-vendor-filesystem
+    mbgl-vendor-metal-cpp
     mbgl-vendor-rapidjson
     mbgl-vendor-parsedate
     mbgl-vendor-pmtiles
@@ -1622,8 +1631,7 @@ set(EXPORT_TARGETS
     mbgl-vendor-unique_resource
     mbgl-vendor-vector-tile
     mbgl-vendor-wagyu
-    mbgl-vendor-metal-cpp
-    mbgl-vendor-filesystem
+    mlt-cpp
     unordered_dense
 )
 

bazel/core.bzl

@@ -336,9 +336,9 @@ MLN_CORE_SOURCE = [
     "src/mbgl/style/conversion/light.cpp",
     "src/mbgl/style/conversion/position.cpp",
     "src/mbgl/style/conversion/property_value.cpp",
-    "src/mbgl/style/conversion/raster_dem_options.cpp",
     "src/mbgl/style/conversion/rotation.cpp",
     "src/mbgl/style/conversion/source.cpp",
+    "src/mbgl/style/conversion/source_options.cpp",
     "src/mbgl/style/conversion/stringify.hpp",
     "src/mbgl/style/conversion/tileset.cpp",
     "src/mbgl/style/conversion/transition_options.cpp",
@@ -514,8 +514,14 @@ MLN_CORE_SOURCE = [
     "src/mbgl/tile/tile_operation.cpp",
     "src/mbgl/tile/vector_tile.cpp",
     "src/mbgl/tile/vector_tile.hpp",
-    "src/mbgl/tile/vector_tile_data.cpp",
-    "src/mbgl/tile/vector_tile_data.hpp",
+    "src/mbgl/tile/vector_mlt_tile.cpp",
+    "src/mbgl/tile/vector_mlt_tile.hpp",
+    "src/mbgl/tile/vector_mlt_tile_data.cpp",
+    "src/mbgl/tile/vector_mlt_tile_data.hpp",
+    "src/mbgl/tile/vector_mvt_tile.cpp",
+    "src/mbgl/tile/vector_mvt_tile.hpp",
+    "src/mbgl/tile/vector_mvt_tile_data.cpp",
+    "src/mbgl/tile/vector_mvt_tile_data.hpp",
     "src/mbgl/util/action_journal.cpp",
     "src/mbgl/util/action_journal_impl.hpp",
     "src/mbgl/util/action_journal_impl.cpp",
@@ -696,9 +702,9 @@ MLN_CORE_HEADERS = [
     "include/mbgl/style/conversion/light.hpp",
     "include/mbgl/style/conversion/position.hpp",
     "include/mbgl/style/conversion/property_value.hpp",
-    "include/mbgl/style/conversion/raster_dem_options.hpp",
     "include/mbgl/style/conversion/rotation.hpp",
     "include/mbgl/style/conversion/source.hpp",
+    "include/mbgl/style/conversion/source_options.hpp",
     "include/mbgl/style/conversion/tileset.hpp",
     "include/mbgl/style/conversion/transition_options.hpp",
     "include/mbgl/style/conversion_impl.hpp",

benchmark/parse/vector_tile.benchmark.cpp

@@ -1,6 +1,6 @@
 #include <benchmark/benchmark.h>
 
-#include <mbgl/tile/vector_tile_data.hpp>
+#include <mbgl/tile/vector_mvt_tile_data.hpp>
 #include <mbgl/util/io.hpp>
 
 using namespace mbgl;
@@ -11,7 +11,7 @@ static void Parse_VectorTile(benchmark::State& state) {
 
     while (state.KeepRunning()) {
         std::size_t length = 0;
-        VectorTileData tile(data);
+        VectorMVTTileData tile(data);
         for (const auto& name : tile.layerNames()) {
             if (auto layer = tile.getLayer(name)) {
                 const std::size_t count = layer->featureCount();

bin/cache.cpp

@@ -7,14 +7,13 @@
 #include <args.hxx>
 #include <mapbox/io/io.hpp>
 
-#include <fstream>
 #include <iostream>
 #include <memory>
 #include <string>
 #include <unordered_map>
 
 int main(int argc, char* argv[]) {
-    args::ArgumentParser p("Mapbox GL cache tool", "");
+    args::ArgumentParser p("MapLibre cache tool", "");
     args::HelpFlag helpFlag(p, "help", "Display this help menu", {'h'});
 
     args::ValueFlag<std::string> urlValue(p, "URL", "Resource URL (required)", {'u'}, args::Options::Required);
@@ -35,7 +34,7 @@ int main(int argc, char* argv[]) {
                                                                   {"tile", mbgl::Resource::Tile}};
 
     std::string typeHelp("One of the following (required):");
-    for (auto key : typeMap) {
+    for (const auto& key : typeMap) {
         typeHelp += " " + key.first;
     }
 
@@ -53,18 +52,18 @@ int main(int argc, char* argv[]) {
         std::cout << p;
         exit(0);
     } catch (const args::ParseError& e) {
-        std::cerr << e.what() << std::endl;
+        std::cerr << e.what() << '\n';
         std::cerr << p;
         exit(1);
     } catch (const args::ValidationError& e) {
-        std::cerr << e.what() << std::endl;
+        std::cerr << e.what() << '\n';
         std::cerr << p;
         exit(2);
     }
 
     auto file = mapbox::base::io::readFile(args::get(dataValue));
     if (!file) {
-        std::cerr << file.error() << std::endl;
+        std::cerr << file.error() << '\n';
         exit(3);
     }
 
@@ -78,15 +77,15 @@ int main(int argc, char* argv[]) {
 
     if (args::get(typeFlag) == mbgl::Resource::Tile) {
         if (!xValueFlag || !yValueFlag || !zValueFlag) {
-            std::cerr << "Error: -x, -y and -z must be provided for tiles" << std::endl;
+            std::cerr << "Error: -x, -y and -z must be provided for tiles" << '\n';
             exit(1);
         }
 
-        resource.tileData = {{args::get(urlValue),
-                              1,
-                              args::get(xValueFlag),
-                              args::get(yValueFlag),
-                              static_cast<int8_t>(args::get(zValueFlag))}};
+        resource.tileData = {{.urlTemplate = args::get(urlValue),
+                              .pixelRatio = 1,
+                              .x = args::get(xValueFlag),
+                              .y = args::get(yValueFlag),
+                              .z = static_cast<int8_t>(args::get(zValueFlag))}};
     }
 
     mbgl::util::RunLoop loop;

include/mbgl/style/conversion/raster_dem_options.hpp renamed to include/mbgl/style/conversion/source_options.hpp

@@ -8,8 +8,8 @@ namespace style {
 namespace conversion {
 
 template <>
-struct Converter<RasterDEMOptions> {
-    std::optional<RasterDEMOptions> operator()(const Convertible& value, Error& error) const;
+struct Converter<SourceOptions> {
+    std::optional<SourceOptions> operator()(const Convertible& value, Error& error) const;
 };
 
 } // namespace conversion

include/mbgl/style/sources/raster_dem_source.hpp

@@ -7,8 +7,9 @@
 namespace mbgl {
 namespace style {
 
-struct RasterDEMOptions {
-    std::optional<Tileset::DEMEncoding> encoding = std::nullopt;
+struct SourceOptions {
+    std::optional<Tileset::RasterEncoding> rasterEncoding = std::nullopt;
+    std::optional<Tileset::VectorEncoding> vectorEncoding = std::nullopt;
 };
 
 // NOTE: Any derived class must invalidate `weakFactory` in the destructor
@@ -17,15 +18,15 @@ class RasterDEMSource final : public RasterSource {
     RasterDEMSource(std::string id,
                     variant<std::string, Tileset> urlOrTileset,
                     uint16_t tileSize,
-                    std::optional<RasterDEMOptions> options = std::nullopt);
+                    std::optional<SourceOptions> options = std::nullopt);
     ~RasterDEMSource() override;
     bool supportsLayerType(const mbgl::style::LayerTypeInfo*) const override;
 
 protected:
     void setTilesetOverrides(Tileset& tileset) override;
 
 private:
-    std::optional<RasterDEMOptions> options;
+    std::optional<SourceOptions> options;
 };
 
 template <>

include/mbgl/style/sources/vector_source.hpp

@@ -16,7 +16,8 @@ class VectorSource final : public TileSource {
     VectorSource(std::string id,
                  variant<std::string, Tileset> urlOrTileset,
                  std::optional<float> maxZoom = std::nullopt,
-                 std::optional<float> minZoom = std::nullopt);
+                 std::optional<float> minZoom = std::nullopt,
+                 Tileset::VectorEncoding encoding = Tileset::VectorEncoding::Mapbox);
 
     void setTilesetOverrides(Tileset& tileset) override;
 
@@ -30,11 +31,14 @@ class VectorSource final : public TileSource {
 
     bool supportsLayerType(const mbgl::style::LayerTypeInfo*) const override;
 
+    Tileset::VectorEncoding getEncoding() const noexcept { return encoding; }
+
     mapbox::base::WeakPtr<Source> makeWeakPtr() override { return weakFactory.makeWeakPtr(); }
 
 private:
     std::optional<float> maxZoom;
     std::optional<float> minZoom;
+    Tileset::VectorEncoding encoding;
     mapbox::base::WeakPtrFactory<Source> weakFactory{this}; // Must remain last
 };
 

include/mbgl/util/tileset.hpp

@@ -18,38 +18,40 @@ class Tileset {
         XYZ,
         TMS
     };
-    enum class DEMEncoding : bool {
-        Mapbox,
-        Terrarium
+    enum class RasterEncoding : std::uint8_t {
+        Mapbox,    // Mapbox DEM
+        Terrarium, // Terrarium DEM
+    };
+    enum class VectorEncoding : std::uint8_t {
+        Mapbox, // Mapbox Vector Tiles (MVT)
+        MLT,    // MapLibre Tiles
     };
 
     std::vector<std::string> tiles;
     Range<uint8_t> zoomRange;
     std::string attribution;
     Scheme scheme;
-    // DEMEncoding is not supported by the TileJSON spec
-    DEMEncoding encoding;
+    std::optional<RasterEncoding> rasterEncoding;
+    std::optional<VectorEncoding> vectorEncoding;
     std::optional<LatLngBounds> bounds;
 
     Tileset(std::vector<std::string> tiles_ = std::vector<std::string>(),
             Range<uint8_t> zoomRange_ = {0, util::DEFAULT_MAX_ZOOM},
             std::string attribution_ = {},
             Scheme scheme_ = Scheme::XYZ,
-            DEMEncoding encoding_ = DEMEncoding::Mapbox)
+            std::optional<RasterEncoding> rasterEncoding_ = std::nullopt,
+            std::optional<VectorEncoding> encoding_ = std::nullopt)
         : tiles(std::move(tiles_)),
           zoomRange(zoomRange_),
           attribution(std::move(attribution_)),
           scheme(scheme_),
-          encoding(encoding_) {};
+          rasterEncoding(rasterEncoding_),
+          vectorEncoding(encoding_) {}
 
     // TileJSON also includes center and zoom but they are not used by mbgl.
 
-    friend bool operator==(const Tileset& lhs, const Tileset& rhs) noexcept {
-        return std::tie(lhs.tiles, lhs.zoomRange, lhs.attribution, lhs.scheme, lhs.bounds) ==
-               std::tie(rhs.tiles, rhs.zoomRange, rhs.attribution, rhs.scheme, rhs.bounds);
-    }
-
-    friend bool operator!=(const Tileset& lhs, const Tileset& rhs) noexcept { return !(lhs == rhs); }
+    bool operator==(const Tileset&) const = default;
+    bool operator!=(const Tileset&) const = default;
 };
 
 } // namespace mbgl