feat(cli-vector): Support NZTM mbtiles creation.BM-1300

packages/cli-vector/schema/place_labels.json

@@ -1,6 +1,6 @@
 {
   "name": "place_labels",
-  "metadata": { "attributes": ["water", "name", "natural", "place"] },
+  "metadata": { "attributes": ["kind", "water", "name", "natural", "place"] },
   "layers": [
     {
       "id": "51154",

packages/cli-vector/schema/pois.json

@@ -792,6 +792,20 @@
       "source": "s3://linz-lds-cache/103476/",
       "tags": { "man_made": "trig_point" },
       "style": { "minZoom": 12, "maxZoom": 15 }
+    },
+    {
+      "id": "50245",
+      "name": "50245-nz-building-points-topo-150k",
+      "source": "s3://linz-lds-cache/50245/",
+      "tags": { "building": "building" },
+      "style": { "minZoom": 14, "maxZoom": 15 }
+    },
+    {
+      "id": "50246",
+      "name": "50246-nz-building-polygons-topo-150k",
+      "source": "s3://linz-lds-cache/50246/",
+      "tags": { "building": "building" },
+      "style": { "minZoom": 14, "maxZoom": 15 }
     }
   ]
 }

packages/cli-vector/schema/street_labels.json

@@ -7,10 +7,8 @@
       "name": "50329-nz-road-centrelines-topo-150k",
       "source": "s3://linz-lds-cache/50329/",
       "tags": { "kind": "road" },
-      "style": { "minZoom": 8, "maxZoom": 15 },
+      "style": { "minZoom": 2, "maxZoom": 15 },
       "simplify": [
-        { "style": { "minZoom": 0, "maxZoom": 0 }, "tolerance": 0.1 },
-        { "style": { "minZoom": 1, "maxZoom": 1 }, "tolerance": 0.01 },
         { "style": { "minZoom": 2, "maxZoom": 2 }, "tolerance": 0.01 },
         { "style": { "minZoom": 3, "maxZoom": 3 }, "tolerance": 0.03 },
         { "style": { "minZoom": 4, "maxZoom": 4 }, "tolerance": 0.008 },

packages/cli-vector/schema/streets.json

@@ -27,10 +27,8 @@
       "name": "50329-nz-road-centrelines-topo-150k",
       "source": "s3://linz-lds-cache/50329/",
       "tags": { "kind": "road" },
-      "style": { "minZoom": 8, "maxZoom": 15 },
+      "style": { "minZoom": 2, "maxZoom": 15 },
       "simplify": [
-        { "style": { "minZoom": 0, "maxZoom": 0 }, "tolerance": 0.1 },
-        { "style": { "minZoom": 1, "maxZoom": 1 }, "tolerance": 0.01 },
         { "style": { "minZoom": 2, "maxZoom": 2 }, "tolerance": 0.01 },
         { "style": { "minZoom": 3, "maxZoom": 3 }, "tolerance": 0.03 },
         { "style": { "minZoom": 4, "maxZoom": 4 }, "tolerance": 0.008 },

packages/cli-vector/schema/water_polygons.json

@@ -34,56 +34,64 @@
       "name": "50293-nz-lake-polygons-topo-150k",
       "source": "s3://linz-lds-cache/50293/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 1, "maxZoom": 15 }
+      "style": { "minZoom": 1, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "50088",
       "name": "50088-nz-chatham-island-lake-polygons-topo-150k",
       "source": "s3://linz-lds-cache/50088/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 0, "maxZoom": 15 }
+      "style": { "minZoom": 0, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "50894",
       "name": "50894-nz-snares-island-tini-heke-lake-polygons-topo-125k",
       "source": "s3://linz-lds-cache/50894/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 0, "maxZoom": 15 }
+      "style": { "minZoom": 0, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "50909",
       "name": "50909-nz-kermadec-is-lake-polygons-topo-125k",
       "source": "s3://linz-lds-cache/50909/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 0, "maxZoom": 15 }
+      "style": { "minZoom": 0, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "50933",
       "name": "50933-nz-campbell-island-motu-ihupuku-lake-polygons-topo-150k",
       "source": "s3://linz-lds-cache/50933/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 0, "maxZoom": 15 }
+      "style": { "minZoom": 0, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "50960",
       "name": "50960-nz-auckland-island-lake-polygons-topo-150k",
       "source": "s3://linz-lds-cache/50960/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 0, "maxZoom": 15 }
+      "style": { "minZoom": 0, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "52253",
       "name": "52253-cook-islands-lake-polygons-topo-125k-zone4",
       "source": "s3://linz-lds-cache/52253/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 0, "maxZoom": 15 }
+      "style": { "minZoom": 0, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "52296",
       "name": "52296-cook-islands-lake-polygons-topo-125k-zone3",
       "source": "s3://linz-lds-cache/52296/",
       "tags": { "kind": "water", "water": "lake" },
-      "style": { "minZoom": 0, "maxZoom": 15 }
+      "style": { "minZoom": 0, "maxZoom": 15 },
+      "includeDerivedArea": true
     },
     {
       "id": "50298",

packages/cli-vector/src/cli/cli.create.ts

@@ -1,3 +1,4 @@
+import { TileMatrixSets } from '@basemaps/geo';
 import { fsa, LogType, Url, UrlArrayJsonFile } from '@basemaps/shared';
 import { CliInfo } from '@basemaps/shared/build/cli/info.js';
 import { getLogger, logArguments } from '@basemaps/shared/build/cli/log.js';
@@ -173,6 +174,8 @@ async function createMbtilesFile(
 
   const layer = options.layer;
   const shortbreadLayer = options.name;
+  const tileMatrix = TileMatrixSets.find(options.tileMatrix);
+  if (tileMatrix == null) throw new Error(`Tile matrix ${options.tileMatrix} is not supported`);
 
   logger.info({ shortbreadLayer, dataset: layer.name }, 'CreateMbtilesFile: Start');
 
@@ -181,7 +184,7 @@ async function createMbtilesFile(
    */
   logger.info({ source: tmpPaths.source.path, dataset: layer.name }, '[1/5] Convert source file to ndjson: Start');
   if (!(await fsa.exists(tmpPaths.ndjson))) {
-    await ogr2ogrNDJson(tmpPaths.source.path, tmpPaths.ndjson, logger);
+    await ogr2ogrNDJson(tmpPaths.source.path, tmpPaths.ndjson, layer, logger);
   }
   logger.info({ destination: tmpPaths.ndjson, dataset: layer.name }, '[1/5] Convert source file to ndjson: End');
 
@@ -191,7 +194,7 @@ async function createMbtilesFile(
   logger.info({ source: tmpPaths.ndjson, dataset: layer.name }, '[2/5] Generalise ndjson features: Start');
   let metrics: Metrics | null = null;
   if (!(await fsa.exists(tmpPaths.genNdjson))) {
-    metrics = await generalize(tmpPaths.ndjson, tmpPaths.genNdjson, options, logger);
+    metrics = await generalize(tmpPaths.ndjson, tmpPaths.genNdjson, tileMatrix, options, logger);
     if (metrics.output === 0) throw new Error(`Failed to generalize ndjson file ${tmpPaths.ndjson.href}`);
   }
   logger.info({ destination: tmpPaths.genNdjson, dataset: layer.name }, '[2/5] Generalise ndjson features: End');
@@ -204,7 +207,7 @@ async function createMbtilesFile(
     '[3/5] Transform generalised ndjson into mbtiles: Start',
   );
   if (!(await fsa.exists(tmpPaths.mbtiles))) {
-    await tippecanoe(tmpPaths.genNdjson, tmpPaths.mbtiles, layer, logger);
+    await tippecanoe(tmpPaths.genNdjson, tmpPaths.mbtiles, layer, tileMatrix, logger);
   }
   logger.info(
     { destination: tmpPaths.mbtiles, dataset: layer.name },

packages/cli-vector/src/cli/cli.extract.ts

@@ -51,7 +51,7 @@ export const ExtractCommand = command({
 
     // Find all lds layers that need to be process
     logger.info({ schema: args.schema }, 'Extract: Start');
-    const schemaLoader = new SchemaLoader(args.schema, logger, cache);
+    const schemaLoader = new SchemaLoader(args.schema, tileMatrix, logger, cache);
     const schemas = await schemaLoader.load();
     const smallLayers = [];
     const largeLayers = [];

packages/cli-vector/src/cli/cli.join.ts

@@ -1,5 +1,5 @@
 import { TileMatrixSets } from '@basemaps/geo';
-import { fsa, isArgo, LogType, Url, UrlArrayJsonFile } from '@basemaps/shared';
+import { fsa, isArgo, LogType, Url, UrlArrayJsonFile, urlToString } from '@basemaps/shared';
 import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
 import { getLogger, logArguments } from '@basemaps/shared/build/cli/log.js';
 import { command, option, optional, string } from 'cmd-ts';
@@ -145,7 +145,7 @@ export const JoinCommand = command({
     // Write output target for argo tasks to create pull request
     if (isArgo()) {
       const target = new URL(`topographic/${CliId}/${args.filename}.tar.co`, bucketPath);
-      await fsa.write(fsa.toUrl('/tmp/target'), JSON.stringify([target]));
+      await fsa.write(fsa.toUrl('/tmp/target'), urlToString(target));
       const mbTilesTarget = new URL(`topographic/${CliId}/${args.filename}.mbtiles`, bucketPath);
       await fsa.write(fsa.toUrl('/tmp/mbTilesTarget'), mbTilesTarget.toString());
       const analyseTarget = new URL(`topographic/${CliId}/`, bucketPath);

packages/cli-vector/src/generalization/generalization.ts

@@ -1,11 +1,13 @@
+import { TileMatrixSet } from '@basemaps/geo';
 import { LogType } from '@basemaps/shared';
 import { createWriteStream } from 'fs';
-import { Geometry, LineString, MultiPolygon, Polygon } from 'geojson';
+import { Feature, Geometry, LineString, MultiPolygon, Polygon } from 'geojson';
 import readline from 'readline';
 
 import { modifyFeature } from '../modify/modify.js';
 import { Metrics, Simplify } from '../schema-loader/schema.js';
 import { VectorCreationOptions } from '../stac.js';
+import { transformNdJson, transformZoom } from '../transform/nztm.js';
 import { VectorGeoFeature } from '../types/VectorGeoFeature.js';
 import { createReadStreamSafe } from '../util.js';
 import { Point, simplify } from './simplify.js';
@@ -18,6 +20,7 @@ import { Point, simplify } from './simplify.js';
 export async function generalize(
   input: URL,
   output: URL,
+  tileMatrix: TileMatrixSet,
   options: VectorCreationOptions,
   logger: LogType,
 ): Promise<Metrics> {
@@ -36,21 +39,23 @@ export async function generalize(
   let outputCount = 0;
   for await (const line of rl) {
     if (line === '') continue;
+    const feature = JSON.parse(line) as Feature;
+    if (tileMatrix.identifier === 'NZTM2000Quad') transformNdJson(feature);
     inputCount++;
     // For simplify, duplicate feature for each zoom level with different tolerance
     if (simplify != null) {
       for (const s of simplify) {
-        const feature = tag(options, line, s, logger);
-        if (feature == null) continue;
+        const vectorGeofeature = tag(tileMatrix, options, feature, s, logger);
+        if (vectorGeofeature == null) continue;
 
-        writeStream.write(JSON.stringify(feature) + '\n');
+        writeStream.write(JSON.stringify(vectorGeofeature) + '\n');
         outputCount++;
       }
     } else {
-      const feature = tag(options, line, null, logger);
-      if (feature == null) continue;
+      const vectorGeofeature = tag(tileMatrix, options, feature, null, logger);
+      if (vectorGeofeature == null) continue;
 
-      writeStream.write(JSON.stringify(feature) + '\n');
+      writeStream.write(JSON.stringify(vectorGeofeature) + '\n');
       outputCount++;
     }
   }
@@ -72,13 +77,14 @@ export async function generalize(
  * Tag feature for layer
  */
 function tag(
+  tileMatrix: TileMatrixSet,
   options: VectorCreationOptions,
-  line: string,
+  feature: Feature,
   simplify: Simplify | null,
   logger: LogType,
 ): VectorGeoFeature | null {
-  const feature = {
-    ...JSON.parse(line),
+  const vectorGeofeature = {
+    ...structuredClone(feature),
     tippecanoe: {
       layer: options.name,
       minzoom: options.layer.style.minZoom,
@@ -87,35 +93,41 @@ function tag(
   } as VectorGeoFeature;
 
   // copy the stac json's tags to the feature (i.e. 'kind')
-  Object.entries(options.layer.tags).forEach(([key, value]) => (feature.properties[key] = value));
-
-  // adjust the feature's metadata and properties
-  const modifiedFeature = modifyFeature(feature, options, logger);
-  if (modifiedFeature == null) {
-    return null;
-  }
+  Object.entries(options.layer.tags).forEach(([key, value]) => (vectorGeofeature.properties[key] = value));
 
   // Simplify geometry
   if (simplify != null) {
     // Update the simplified feature zoom level
-    modifiedFeature['tippecanoe'] = {
+    vectorGeofeature['tippecanoe'] = {
       layer: options.name,
       minzoom: simplify.style.minZoom,
       maxzoom: simplify.style.maxZoom,
     };
     if (simplify.tolerance != null) {
-      const geom = modifiedFeature.geometry;
+      const geom = vectorGeofeature.geometry;
       const type = geom.type;
-      const coordinates = simplifyFeature(type, geom, simplify.tolerance);
-      if (coordinates == null) {
+      const geometry = simplifyFeature(type, geom, simplify.tolerance);
+      if (geometry == null) {
         return null;
       }
-      modifiedFeature.geometry = coordinates;
+      vectorGeofeature.geometry = geometry;
     }
   }
 
+  // adjust the feature's metadata and properties
+  const modifiedFeature = modifyFeature(vectorGeofeature, options, logger);
+  if (modifiedFeature == null) {
+    return null;
+  }
+
+  // Skip features that maxzoom is less than minzoom, this could happened after simplification and special tags on zoom levels
+  if (modifiedFeature.tippecanoe.maxzoom < modifiedFeature.tippecanoe.minzoom) return null;
+
+  // Transform zoom level for NZTM2000Quad
+  modifiedFeature.tippecanoe.minzoom = transformZoom(modifiedFeature.tippecanoe.minzoom, tileMatrix);
+  modifiedFeature.tippecanoe.maxzoom = transformZoom(modifiedFeature.tippecanoe.maxzoom, tileMatrix);
+
   // Remove unused properties
-  // REVIEW: this function just removes the special tags. something isn't right here
   const cleanedFeature = removeAttributes(modifiedFeature, options);
   return cleanedFeature;
 }

packages/cli-vector/src/modify/layers/place_labels.ts

@@ -36,6 +36,7 @@ export function handleLayerPlaceLabels(feature: VectorGeoFeature, logger: LogTyp
   const zoomLevel = feature.properties['zoom_level'];
   if (typeof zoomLevel !== 'number') throw new Error('Zoom level is not a number');
 
+  //DATA PROBLEM: We need to store the first feature which have all the propertie values, the duplicate features will only have null values in the properties
   const storedFeature = PlaceLabelsFeatures.get(label);
   if (storedFeature == null) {
     const newFeature = createNewFeature(feature, label, zoomLevel, logger);
@@ -48,16 +49,9 @@ export function handleLayerPlaceLabels(feature: VectorGeoFeature, logger: LogTyp
   }
 
   // update the stored feature's 'minzoom' value
-  if (zoomLevel < storedFeature.tippecanoe.minzoom) {
-    storedFeature.tippecanoe.minzoom = zoomLevel;
-    PlaceLabelsFeatures.set(label, storedFeature);
-  }
-
+  storedFeature.tippecanoe.minzoom = zoomLevel;
   // update the stored feature's 'maxzoom' value
-  if (zoomLevel > storedFeature.tippecanoe.maxzoom) {
-    storedFeature.tippecanoe.maxzoom = zoomLevel;
-    PlaceLabelsFeatures.set(label, storedFeature);
-  }
+  storedFeature.tippecanoe.maxzoom = zoomLevel;
 
   logger.trace({}, 'HandlePlaceLabels:End');
   return storedFeature;

packages/cli-vector/src/modify/layers/pois.ts

@@ -1,6 +1,8 @@
 import { LogType } from '@basemaps/shared';
+import { Point } from 'geojson';
 
 import { VectorGeoFeature } from '../../types/VectorGeoFeature.js';
+import { getCoordinates, polylabel } from '../shared.js';
 
 /**
  * Processes a 'pois' layer feature.
@@ -14,9 +16,6 @@ export function handleLayerPois(feature: VectorGeoFeature, logger: LogType): Vec
   logger.trace({}, 'HandlePois:Start');
   feature = structuredClone(feature);
 
-  // REVIEW: We don't have any use for this criteria as we don't include the following layers:
-  // 1. 50245-nz-building-points-topo-150k
-  // 2. 50246-nz-building-polygons-topo-150k
   if (feature.properties['building'] === 'building') {
     const bldgUse = feature.properties['bldg_use'];
 
@@ -27,6 +26,15 @@ export function handleLayerPois(feature: VectorGeoFeature, logger: LogType): Vec
     }
 
     feature.properties['building'] = bldgUse;
+
+    // Covert the building polygon to a point for 50246-nz-building-polygons-topo-150k
+    if (feature.geometry.type === 'Polygon' || feature.geometry.type === 'MultiPolygon') {
+      const coordinates = getCoordinates(feature.geometry, logger);
+      const inaccessibilityPole = polylabel(coordinates);
+
+      const point: Point = { type: 'Point', coordinates: inaccessibilityPole };
+      feature.geometry = point;
+    }
   }
 
   logger.trace({}, 'HandlePois:End');