Skip to content

Commit f601a73

Browse files
authored
feat(cli-vector): Support NZTM mbtiles creation.BM-1300 (#3452)
### Motivation Upgrade ETL process to support NZTM mbtiles creation ### Modifications - Using the function `locationTransform` from landing package to convert the geometry into NZTM tilematrix during the generalisation step. Then apply the zoom level offset for it. - Fixes for the labels are not correctly set the zoom levels - Fixes the simplify that reset the modified the zoom levels, move the modify feature after simplification. ### Verification [Argo workflow](https://argo.linzaccess.com/workflows/argo/test-basemaps-vector-etl-shortbread-fwld2?tab=workflow&uid=ab21bf49-b76a-41b8-ad0c-db6e9ec7a5e1) Previews: [WebMercatorQuad](https://basemaps.linz.govt.nz/@-41.3705336,174.1157611,z5.34?style=topographic-v2&i=topographic&config=TmVmbYRQjL9T2JWgyaSie193b4D1qZnBD8hjaSqPFYSsEvEYYhaGYKrco3AC47SThKZHbuAsjSgeFU9MRxbdHWjafYuRnUL7rHRfY6SN4x8sBYyToSqLTi9cvBYwgf3&debug=true) and [NZTM2000Quad](https://basemaps.linz.govt.nz/@-41.5327847,171.3510225,z3.21?style=topographic-v2&i=topographic&tileMatrix=NZTM2000Quad&config=TmVmbYRQjL9T2JWgyaSie193b4D1qZnBD8hjaSqPFYSsEvEYYhaGYKrco3AC47SThKZHbuAsjSgeFU9MRxbdHWjafYuRnUL7rHRfY6SN4x8sBYyToSqLTi9cvBYwgf3&debug=true)
1 parent a57059e commit f601a73

30 files changed

+327
-195
lines changed

packages/cli-vector/schema/place_labels.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "place_labels",
3-
"metadata": { "attributes": ["water", "name", "natural", "place"] },
3+
"metadata": { "attributes": ["kind", "water", "name", "natural", "place"] },
44
"layers": [
55
{
66
"id": "51154",

packages/cli-vector/schema/pois.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -792,6 +792,20 @@
792792
"source": "s3://linz-lds-cache/103476/",
793793
"tags": { "man_made": "trig_point" },
794794
"style": { "minZoom": 12, "maxZoom": 15 }
795+
},
796+
{
797+
"id": "50245",
798+
"name": "50245-nz-building-points-topo-150k",
799+
"source": "s3://linz-lds-cache/50245/",
800+
"tags": { "building": "building" },
801+
"style": { "minZoom": 14, "maxZoom": 15 }
802+
},
803+
{
804+
"id": "50246",
805+
"name": "50246-nz-building-polygons-topo-150k",
806+
"source": "s3://linz-lds-cache/50246/",
807+
"tags": { "building": "building" },
808+
"style": { "minZoom": 14, "maxZoom": 15 }
795809
}
796810
]
797811
}

packages/cli-vector/schema/street_labels.json

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,8 @@
77
"name": "50329-nz-road-centrelines-topo-150k",
88
"source": "s3://linz-lds-cache/50329/",
99
"tags": { "kind": "road" },
10-
"style": { "minZoom": 8, "maxZoom": 15 },
10+
"style": { "minZoom": 2, "maxZoom": 15 },
1111
"simplify": [
12-
{ "style": { "minZoom": 0, "maxZoom": 0 }, "tolerance": 0.1 },
13-
{ "style": { "minZoom": 1, "maxZoom": 1 }, "tolerance": 0.01 },
1412
{ "style": { "minZoom": 2, "maxZoom": 2 }, "tolerance": 0.01 },
1513
{ "style": { "minZoom": 3, "maxZoom": 3 }, "tolerance": 0.03 },
1614
{ "style": { "minZoom": 4, "maxZoom": 4 }, "tolerance": 0.008 },

packages/cli-vector/schema/streets.json

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,8 @@
2727
"name": "50329-nz-road-centrelines-topo-150k",
2828
"source": "s3://linz-lds-cache/50329/",
2929
"tags": { "kind": "road" },
30-
"style": { "minZoom": 8, "maxZoom": 15 },
30+
"style": { "minZoom": 2, "maxZoom": 15 },
3131
"simplify": [
32-
{ "style": { "minZoom": 0, "maxZoom": 0 }, "tolerance": 0.1 },
33-
{ "style": { "minZoom": 1, "maxZoom": 1 }, "tolerance": 0.01 },
3432
{ "style": { "minZoom": 2, "maxZoom": 2 }, "tolerance": 0.01 },
3533
{ "style": { "minZoom": 3, "maxZoom": 3 }, "tolerance": 0.03 },
3634
{ "style": { "minZoom": 4, "maxZoom": 4 }, "tolerance": 0.008 },

packages/cli-vector/schema/water_polygons.json

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,56 +34,64 @@
3434
"name": "50293-nz-lake-polygons-topo-150k",
3535
"source": "s3://linz-lds-cache/50293/",
3636
"tags": { "kind": "water", "water": "lake" },
37-
"style": { "minZoom": 1, "maxZoom": 15 }
37+
"style": { "minZoom": 1, "maxZoom": 15 },
38+
"includeDerivedArea": true
3839
},
3940
{
4041
"id": "50088",
4142
"name": "50088-nz-chatham-island-lake-polygons-topo-150k",
4243
"source": "s3://linz-lds-cache/50088/",
4344
"tags": { "kind": "water", "water": "lake" },
44-
"style": { "minZoom": 0, "maxZoom": 15 }
45+
"style": { "minZoom": 0, "maxZoom": 15 },
46+
"includeDerivedArea": true
4547
},
4648
{
4749
"id": "50894",
4850
"name": "50894-nz-snares-island-tini-heke-lake-polygons-topo-125k",
4951
"source": "s3://linz-lds-cache/50894/",
5052
"tags": { "kind": "water", "water": "lake" },
51-
"style": { "minZoom": 0, "maxZoom": 15 }
53+
"style": { "minZoom": 0, "maxZoom": 15 },
54+
"includeDerivedArea": true
5255
},
5356
{
5457
"id": "50909",
5558
"name": "50909-nz-kermadec-is-lake-polygons-topo-125k",
5659
"source": "s3://linz-lds-cache/50909/",
5760
"tags": { "kind": "water", "water": "lake" },
58-
"style": { "minZoom": 0, "maxZoom": 15 }
61+
"style": { "minZoom": 0, "maxZoom": 15 },
62+
"includeDerivedArea": true
5963
},
6064
{
6165
"id": "50933",
6266
"name": "50933-nz-campbell-island-motu-ihupuku-lake-polygons-topo-150k",
6367
"source": "s3://linz-lds-cache/50933/",
6468
"tags": { "kind": "water", "water": "lake" },
65-
"style": { "minZoom": 0, "maxZoom": 15 }
69+
"style": { "minZoom": 0, "maxZoom": 15 },
70+
"includeDerivedArea": true
6671
},
6772
{
6873
"id": "50960",
6974
"name": "50960-nz-auckland-island-lake-polygons-topo-150k",
7075
"source": "s3://linz-lds-cache/50960/",
7176
"tags": { "kind": "water", "water": "lake" },
72-
"style": { "minZoom": 0, "maxZoom": 15 }
77+
"style": { "minZoom": 0, "maxZoom": 15 },
78+
"includeDerivedArea": true
7379
},
7480
{
7581
"id": "52253",
7682
"name": "52253-cook-islands-lake-polygons-topo-125k-zone4",
7783
"source": "s3://linz-lds-cache/52253/",
7884
"tags": { "kind": "water", "water": "lake" },
79-
"style": { "minZoom": 0, "maxZoom": 15 }
85+
"style": { "minZoom": 0, "maxZoom": 15 },
86+
"includeDerivedArea": true
8087
},
8188
{
8289
"id": "52296",
8390
"name": "52296-cook-islands-lake-polygons-topo-125k-zone3",
8491
"source": "s3://linz-lds-cache/52296/",
8592
"tags": { "kind": "water", "water": "lake" },
86-
"style": { "minZoom": 0, "maxZoom": 15 }
93+
"style": { "minZoom": 0, "maxZoom": 15 },
94+
"includeDerivedArea": true
8795
},
8896
{
8997
"id": "50298",

packages/cli-vector/src/cli/cli.create.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { TileMatrixSets } from '@basemaps/geo';
12
import { fsa, LogType, Url, UrlArrayJsonFile } from '@basemaps/shared';
23
import { CliInfo } from '@basemaps/shared/build/cli/info.js';
34
import { getLogger, logArguments } from '@basemaps/shared/build/cli/log.js';
@@ -173,6 +174,8 @@ async function createMbtilesFile(
173174

174175
const layer = options.layer;
175176
const shortbreadLayer = options.name;
177+
const tileMatrix = TileMatrixSets.find(options.tileMatrix);
178+
if (tileMatrix == null) throw new Error(`Tile matrix ${options.tileMatrix} is not supported`);
176179

177180
logger.info({ shortbreadLayer, dataset: layer.name }, 'CreateMbtilesFile: Start');
178181

@@ -181,7 +184,7 @@ async function createMbtilesFile(
181184
*/
182185
logger.info({ source: tmpPaths.source.path, dataset: layer.name }, '[1/5] Convert source file to ndjson: Start');
183186
if (!(await fsa.exists(tmpPaths.ndjson))) {
184-
await ogr2ogrNDJson(tmpPaths.source.path, tmpPaths.ndjson, logger);
187+
await ogr2ogrNDJson(tmpPaths.source.path, tmpPaths.ndjson, layer, logger);
185188
}
186189
logger.info({ destination: tmpPaths.ndjson, dataset: layer.name }, '[1/5] Convert source file to ndjson: End');
187190

@@ -191,7 +194,7 @@ async function createMbtilesFile(
191194
logger.info({ source: tmpPaths.ndjson, dataset: layer.name }, '[2/5] Generalise ndjson features: Start');
192195
let metrics: Metrics | null = null;
193196
if (!(await fsa.exists(tmpPaths.genNdjson))) {
194-
metrics = await generalize(tmpPaths.ndjson, tmpPaths.genNdjson, options, logger);
197+
metrics = await generalize(tmpPaths.ndjson, tmpPaths.genNdjson, tileMatrix, options, logger);
195198
if (metrics.output === 0) throw new Error(`Failed to generalize ndjson file ${tmpPaths.ndjson.href}`);
196199
}
197200
logger.info({ destination: tmpPaths.genNdjson, dataset: layer.name }, '[2/5] Generalise ndjson features: End');
@@ -204,7 +207,7 @@ async function createMbtilesFile(
204207
'[3/5] Transform generalised ndjson into mbtiles: Start',
205208
);
206209
if (!(await fsa.exists(tmpPaths.mbtiles))) {
207-
await tippecanoe(tmpPaths.genNdjson, tmpPaths.mbtiles, layer, logger);
210+
await tippecanoe(tmpPaths.genNdjson, tmpPaths.mbtiles, layer, tileMatrix, logger);
208211
}
209212
logger.info(
210213
{ destination: tmpPaths.mbtiles, dataset: layer.name },

packages/cli-vector/src/cli/cli.extract.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ export const ExtractCommand = command({
5151

5252
// Find all lds layers that need to be process
5353
logger.info({ schema: args.schema }, 'Extract: Start');
54-
const schemaLoader = new SchemaLoader(args.schema, logger, cache);
54+
const schemaLoader = new SchemaLoader(args.schema, tileMatrix, logger, cache);
5555
const schemas = await schemaLoader.load();
5656
const smallLayers = [];
5757
const largeLayers = [];

packages/cli-vector/src/cli/cli.join.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { TileMatrixSets } from '@basemaps/geo';
2-
import { fsa, isArgo, LogType, Url, UrlArrayJsonFile } from '@basemaps/shared';
2+
import { fsa, isArgo, LogType, Url, UrlArrayJsonFile, urlToString } from '@basemaps/shared';
33
import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
44
import { getLogger, logArguments } from '@basemaps/shared/build/cli/log.js';
55
import { command, option, optional, string } from 'cmd-ts';
@@ -145,7 +145,7 @@ export const JoinCommand = command({
145145
// Write output target for argo tasks to create pull request
146146
if (isArgo()) {
147147
const target = new URL(`topographic/${CliId}/${args.filename}.tar.co`, bucketPath);
148-
await fsa.write(fsa.toUrl('/tmp/target'), JSON.stringify([target]));
148+
await fsa.write(fsa.toUrl('/tmp/target'), urlToString(target));
149149
const mbTilesTarget = new URL(`topographic/${CliId}/${args.filename}.mbtiles`, bucketPath);
150150
await fsa.write(fsa.toUrl('/tmp/mbTilesTarget'), mbTilesTarget.toString());
151151
const analyseTarget = new URL(`topographic/${CliId}/`, bucketPath);

packages/cli-vector/src/generalization/generalization.ts

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1+
import { TileMatrixSet } from '@basemaps/geo';
12
import { LogType } from '@basemaps/shared';
23
import { createWriteStream } from 'fs';
3-
import { Geometry, LineString, MultiPolygon, Polygon } from 'geojson';
4+
import { Feature, Geometry, LineString, MultiPolygon, Polygon } from 'geojson';
45
import readline from 'readline';
56

67
import { modifyFeature } from '../modify/modify.js';
78
import { Metrics, Simplify } from '../schema-loader/schema.js';
89
import { VectorCreationOptions } from '../stac.js';
10+
import { transformNdJson, transformZoom } from '../transform/nztm.js';
911
import { VectorGeoFeature } from '../types/VectorGeoFeature.js';
1012
import { createReadStreamSafe } from '../util.js';
1113
import { Point, simplify } from './simplify.js';
@@ -18,6 +20,7 @@ import { Point, simplify } from './simplify.js';
1820
export async function generalize(
1921
input: URL,
2022
output: URL,
23+
tileMatrix: TileMatrixSet,
2124
options: VectorCreationOptions,
2225
logger: LogType,
2326
): Promise<Metrics> {
@@ -36,21 +39,23 @@ export async function generalize(
3639
let outputCount = 0;
3740
for await (const line of rl) {
3841
if (line === '') continue;
42+
const feature = JSON.parse(line) as Feature;
43+
if (tileMatrix.identifier === 'NZTM2000Quad') transformNdJson(feature);
3944
inputCount++;
4045
// For simplify, duplicate feature for each zoom level with different tolerance
4146
if (simplify != null) {
4247
for (const s of simplify) {
43-
const feature = tag(options, line, s, logger);
44-
if (feature == null) continue;
48+
const vectorGeofeature = tag(tileMatrix, options, feature, s, logger);
49+
if (vectorGeofeature == null) continue;
4550

46-
writeStream.write(JSON.stringify(feature) + '\n');
51+
writeStream.write(JSON.stringify(vectorGeofeature) + '\n');
4752
outputCount++;
4853
}
4954
} else {
50-
const feature = tag(options, line, null, logger);
51-
if (feature == null) continue;
55+
const vectorGeofeature = tag(tileMatrix, options, feature, null, logger);
56+
if (vectorGeofeature == null) continue;
5257

53-
writeStream.write(JSON.stringify(feature) + '\n');
58+
writeStream.write(JSON.stringify(vectorGeofeature) + '\n');
5459
outputCount++;
5560
}
5661
}
@@ -72,13 +77,14 @@ export async function generalize(
7277
* Tag feature for layer
7378
*/
7479
function tag(
80+
tileMatrix: TileMatrixSet,
7581
options: VectorCreationOptions,
76-
line: string,
82+
feature: Feature,
7783
simplify: Simplify | null,
7884
logger: LogType,
7985
): VectorGeoFeature | null {
80-
const feature = {
81-
...JSON.parse(line),
86+
const vectorGeofeature = {
87+
...structuredClone(feature),
8288
tippecanoe: {
8389
layer: options.name,
8490
minzoom: options.layer.style.minZoom,
@@ -87,35 +93,41 @@ function tag(
8793
} as VectorGeoFeature;
8894

8995
// copy the stac json's tags to the feature (i.e. 'kind')
90-
Object.entries(options.layer.tags).forEach(([key, value]) => (feature.properties[key] = value));
91-
92-
// adjust the feature's metadata and properties
93-
const modifiedFeature = modifyFeature(feature, options, logger);
94-
if (modifiedFeature == null) {
95-
return null;
96-
}
96+
Object.entries(options.layer.tags).forEach(([key, value]) => (vectorGeofeature.properties[key] = value));
9797

9898
// Simplify geometry
9999
if (simplify != null) {
100100
// Update the simplified feature zoom level
101-
modifiedFeature['tippecanoe'] = {
101+
vectorGeofeature['tippecanoe'] = {
102102
layer: options.name,
103103
minzoom: simplify.style.minZoom,
104104
maxzoom: simplify.style.maxZoom,
105105
};
106106
if (simplify.tolerance != null) {
107-
const geom = modifiedFeature.geometry;
107+
const geom = vectorGeofeature.geometry;
108108
const type = geom.type;
109-
const coordinates = simplifyFeature(type, geom, simplify.tolerance);
110-
if (coordinates == null) {
109+
const geometry = simplifyFeature(type, geom, simplify.tolerance);
110+
if (geometry == null) {
111111
return null;
112112
}
113-
modifiedFeature.geometry = coordinates;
113+
vectorGeofeature.geometry = geometry;
114114
}
115115
}
116116

117+
// adjust the feature's metadata and properties
118+
const modifiedFeature = modifyFeature(vectorGeofeature, options, logger);
119+
if (modifiedFeature == null) {
120+
return null;
121+
}
122+
123+
// Skip features that maxzoom is less than minzoom, this could happened after simplification and special tags on zoom levels
124+
if (modifiedFeature.tippecanoe.maxzoom < modifiedFeature.tippecanoe.minzoom) return null;
125+
126+
// Transform zoom level for NZTM2000Quad
127+
modifiedFeature.tippecanoe.minzoom = transformZoom(modifiedFeature.tippecanoe.minzoom, tileMatrix);
128+
modifiedFeature.tippecanoe.maxzoom = transformZoom(modifiedFeature.tippecanoe.maxzoom, tileMatrix);
129+
117130
// Remove unused properties
118-
// REVIEW: this function just removes the special tags. something isn't right here
119131
const cleanedFeature = removeAttributes(modifiedFeature, options);
120132
return cleanedFeature;
121133
}

packages/cli-vector/src/modify/layers/place_labels.ts

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export function handleLayerPlaceLabels(feature: VectorGeoFeature, logger: LogTyp
3636
const zoomLevel = feature.properties['zoom_level'];
3737
if (typeof zoomLevel !== 'number') throw new Error('Zoom level is not a number');
3838

39+
//DATA PROBLEM: We need to store the first feature which have all the propertie values, the duplicate features will only have null values in the properties
3940
const storedFeature = PlaceLabelsFeatures.get(label);
4041
if (storedFeature == null) {
4142
const newFeature = createNewFeature(feature, label, zoomLevel, logger);
@@ -48,16 +49,9 @@ export function handleLayerPlaceLabels(feature: VectorGeoFeature, logger: LogTyp
4849
}
4950

5051
// update the stored feature's 'minzoom' value
51-
if (zoomLevel < storedFeature.tippecanoe.minzoom) {
52-
storedFeature.tippecanoe.minzoom = zoomLevel;
53-
PlaceLabelsFeatures.set(label, storedFeature);
54-
}
55-
52+
storedFeature.tippecanoe.minzoom = zoomLevel;
5653
// update the stored feature's 'maxzoom' value
57-
if (zoomLevel > storedFeature.tippecanoe.maxzoom) {
58-
storedFeature.tippecanoe.maxzoom = zoomLevel;
59-
PlaceLabelsFeatures.set(label, storedFeature);
60-
}
54+
storedFeature.tippecanoe.maxzoom = zoomLevel;
6155

6256
logger.trace({}, 'HandlePlaceLabels:End');
6357
return storedFeature;

0 commit comments

Comments
 (0)