feat(cogify): Update cogify to support the topo raster processes. BM-1116

packages/cogify/src/cogify/__test__/covering.test.ts

@@ -4,7 +4,7 @@ import { describe, it } from 'node:test';
 import { GoogleTms, QuadKey } from '@basemaps/geo';
 
 import { gsdToMeter } from '../cli/cli.cover.js';
-import { addChildren, addSurrounding } from '../covering.js';
+import { addChildren, addSurrounding } from '../covering/covering.js';
 
 describe('getChildren', () => {
   it('should get children', () => {

packages/cogify/src/cogify/__test__/extract.test.ts

@@ -0,0 +1,37 @@
+import { strictEqual, throws } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { extractMapCodeAndVersion } from '../topo/extract.js';
+
+describe('extractMapCodeAndVersion', () => {
+  const FakeDomain = 's3://topographic/fake-domain';
+  const validFiles = [
+    { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    {
+      input: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AX32ptsAX31AY31AY32', version: 'v1-00' },
+    },
+    {
+      input: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AZ36ptsAZ35BA35BA36', version: 'v1-00' },
+    },
+  ];
+  const invalidFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`];
+
+  it('should parse the correct MapSheet Names', () => {
+    for (const file of validFiles) {
+      const output = extractMapCodeAndVersion(new URL(file.input));
+      strictEqual(output.mapCode, file.expected.mapCode, 'Map code does not match');
+      strictEqual(output.version, file.expected.version, 'Version does not match');
+    }
+  });
+
+  it('should not able to parse a version from file', () => {
+    for (const file of invalidFiles) {
+      throws(() => extractMapCodeAndVersion(new URL(file)), new Error(`Version not found in the file name: "${file}"`));
+    }
+  });
+});

packages/cogify/src/cogify/cli.ts

@@ -3,6 +3,7 @@ import { subcommands } from 'cmd-ts';
 import { BasemapsCogifyCreateCommand } from './cli/cli.cog.js';
 import { BasemapsCogifyConfigCommand } from './cli/cli.config.js';
 import { BasemapsCogifyCoverCommand } from './cli/cli.cover.js';
+import { TopoStacCreationCommand } from './cli/cli.topo.js';
 import { BasemapsCogifyValidateCommand } from './cli/cli.validate.js';
 
 export const CogifyCli = subcommands({
@@ -12,5 +13,6 @@ export const CogifyCli = subcommands({
     create: BasemapsCogifyCreateCommand,
     config: BasemapsCogifyConfigCommand,
     validate: BasemapsCogifyValidateCommand,
+    topo: TopoStacCreationCommand,
   },
 });

packages/cogify/src/cogify/cli/__test__/cli.topo.test.ts

@@ -0,0 +1,69 @@
+import assert from 'node:assert';
+import { beforeEach, describe, it } from 'node:test';
+
+import { fsa, FsMemory, LogConfig } from '@basemaps/shared';
+import { TestTiff } from '@basemaps/test';
+import { StacCollection } from 'stac-ts';
+
+import { TopoStacCreationCommand } from '../cli.topo.js';
+
+describe('cli.topo', () => {
+  const fsMemory = new FsMemory();
+
+  beforeEach(async () => {
+    LogConfig.get().level = 'silent';
+    fsa.register('memory://', fsMemory);
+    fsMemory.files.clear();
+
+    await fsa.write(new URL('memory://source/CJ10_GRIDLESS_GeoTifv1-00.tif'), fsa.readStream(TestTiff.Nztm2000));
+    await fsa.write(new URL('memory://source/CJ10_GRIDLESS_GeoTifv1-01.tif'), fsa.readStream(TestTiff.Nztm2000));
+  });
+
+  const baseArgs = {
+    paths: [new URL('memory://source/')],
+    target: new URL('memory://target/'),
+    mapSeries: 'topo50',
+    latestOnly: false,
+    title: undefined,
+    output: undefined,
+
+    // extra logging arguments
+    verbose: false,
+    extraVerbose: false,
+  };
+
+  it('should generate a covering', async () => {
+    const ret = await TopoStacCreationCommand.handler({ ...baseArgs }).catch((e) => String(e));
+    assert.equal(ret, undefined); // no errors returned
+
+    const files = [...fsMemory.files.keys()];
+    files.sort();
+
+    assert.deepEqual(files, [
+      'memory://source/CJ10_GRIDLESS_GeoTifv1-00.tif',
+      'memory://source/CJ10_GRIDLESS_GeoTifv1-01.tif',
+      'memory://target/topo50/gridless_600dpi/2193/CJ10_v1-00.json',
+      'memory://target/topo50/gridless_600dpi/2193/CJ10_v1-01.json',
+      'memory://target/topo50/gridless_600dpi/2193/collection.json',
+      'memory://target/topo50_latest/gridless_600dpi/2193/CJ10.json',
+      'memory://target/topo50_latest/gridless_600dpi/2193/collection.json',
+    ]);
+
+    const collectionJson = await fsa.readJson<StacCollection>(
+      new URL('memory://target/topo50/gridless_600dpi/2193/collection.json'),
+    );
+    assert.equal(collectionJson['description'], 'Topographic maps of New Zealand');
+    assert.equal(collectionJson['linz:slug'], 'topo50-new-zealand-mainland');
+    assert.equal(collectionJson['linz:region'], 'new-zealand');
+
+    const latestItemUrl = new URL('memory://target/topo50_latest/gridless_600dpi/2193/CJ10.json');
+    const latestVersion = await fsa.readJson<StacCollection>(latestItemUrl);
+
+    // Latest file should be derived_from the source file
+    const derived = latestVersion.links.filter((f) => f.rel === 'derived_from');
+    assert.equal(derived.length, 1);
+
+    const derivedFile = new URL(derived[0].href, latestItemUrl);
+    assert.equal(derivedFile.href, 'memory://target/topo50/gridless_600dpi/2193/CJ10_v1-01.json');
+  });
+});

packages/cogify/src/cogify/cli/cli.cog.ts

@@ -11,18 +11,20 @@ import path from 'path';
 import { StacAsset, StacCollection } from 'stac-ts';
 import { pathToFileURL } from 'url';
 
-import { CutlineOptimizer } from '../../cutline.js';
 import { SourceDownloader } from '../../download.js';
 import { HashTransform } from '../../hash.stream.js';
 import { getLogger, logArguments } from '../../log.js';
-import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp, gdalCreate } from '../gdal.command.js';
-import { GdalRunner } from '../gdal.runner.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
+import {
+  gdalBuildCog,
+  gdalBuildTopoRasterCommands,
+  gdalBuildVrt,
+  gdalBuildVrtWarp,
+  gdalCreate,
+} from '../gdal/gdal.command.js';
+import { GdalRunner } from '../gdal/gdal.runner.js';
 import { Url, UrlArrayJsonFile } from '../parsers.js';
-import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources } from '../stac.js';
-
-function extractSourceFiles(item: CogifyStacItem, baseUrl: URL): URL[] {
-  return item.links.filter((link) => link.rel === 'linz_basemaps:source').map((link) => new URL(link.href, baseUrl));
-}
+import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources, isTopoStacItem } from '../stac.js';
 
 const Collections = new Map<string, Promise<StacCollection>>();
 
@@ -153,16 +155,17 @@ export const BasemapsCogifyCreateCommand = command({
 
       const promises = filtered.map(async (f) => {
         const { item, url } = f;
+
         const cutlineLink = getCutline(item.links);
         const options = item.properties['linz_basemaps:options'];
-        const tileId = TileId.fromTile(options.tile);
+        const tileId = isTopoStacItem(item) ? item.id : TileId.fromTile(options.tile);
 
         // Location to where the tiff should be stored
         const tiffPath = new URL(tileId + '.tiff', url);
         const itemStacPath = new URL(tileId + '.json', url);
         const tileMatrix = TileMatrixSets.find(options.tileMatrix);
         if (tileMatrix == null) throw new Error('Failed to find tileMatrix: ' + options.tileMatrix);
-        const sourceFiles = extractSourceFiles(item, url);
+        const sourceFiles = getSources(item.links);
 
         // Skip creating the COG if the item STAC contains no source tiffs
         if (sourceFiles.length === 0) {
@@ -174,10 +177,31 @@ export const BasemapsCogifyCreateCommand = command({
         const outputTiffPath = await Q(async () => {
           metrics.start(tileId); // Only start the timer when the cog is actually being processed
 
+          // Download all tiff files needed for the processing
+          const sourceLocations = await Promise.all(
+            sourceFiles.map((link) => sources.get(new URL(link.href, url), logger)),
+          );
+
           const cutline = await CutlineOptimizer.loadFromLink(cutlineLink, tileMatrix);
-          const sourceLocations = await Promise.all(sourceFiles.map((f) => sources.get(f, logger)));
+          if (isTopoStacItem(item)) {
+            if (sourceFiles.length !== 1) {
+              throw new Error('Topo MapSheet procesing is limited to one input file, found: ' + sourceLocations.length);
+            }
+            const width = sourceFiles[0]['linz_basemaps:source_width'];
+            const height = sourceFiles[0]['linz_basemaps:source_height'];
+            return createTopoCog({
+              tileId,
+              options,
+              tempFolder: tmpFolder,
+              sourceFiles: sourceLocations,
+              cutline,
+              size: { width, height },
+              logger,
+            });
+          }
 
           return createCog({
+            tileId,
             options,
             tempFolder: tmpFolder,
             sourceFiles: sourceLocations,
@@ -190,8 +214,9 @@ export const BasemapsCogifyCreateCommand = command({
         logger.debug({ files: sourceFiles.length }, 'Cog:Cleanup');
         const deleted = await Promise.all(
           sourceFiles.map(async (f) => {
-            const asset = sources.items.get(f.href);
-            await sources.done(f, item.id, logger);
+            const sourceLocation = new URL(f.href, url);
+            const asset = sources.items.get(sourceLocation.href);
+            await sources.done(sourceLocation, item.id, logger);
             // Update the STAC Document with the checksum and file size of the files used to create this asset
             if (asset == null || asset.size == null || asset.hash == null) return;
             const link = item.links.find((link) => new URL(link.href, url).href === asset.url.href);
@@ -268,14 +293,18 @@ export const BasemapsCogifyCreateCommand = command({
       {
         count: toCreate.length,
         created: filtered.length,
-        files: filtered.map((f) => TileId.fromTile(f.item.properties['linz_basemaps:options'].tile)),
+        files: filtered.map((f) => {
+          return isTopoStacItem(f.item) ? f.item.id : TileId.fromTile(f.item.properties['linz_basemaps:options'].tile);
+        }),
       },
       'Cog:Done',
     );
   },
 });
 
 export interface CogCreationContext {
+  /** TileId for the file name */
+  tileId: string;
   /** COG Creation options */
   options: CogifyCreationOptions;
   /** Location to store all the temporary files */
@@ -284,15 +313,17 @@ export interface CogCreationContext {
   sourceFiles: URL[];
   /** Optional cutline to cut the imagery too */
   cutline: CutlineOptimizer;
+  /** Optional Source imagery size for topo raster trim pixel */
+  size?: { width: number; height: number };
   /** Optional logger */
   logger?: LogType;
 }
 
-/** Create a cog from the creation options */
+/** Create a generic COG from the creation options */
 async function createCog(ctx: CogCreationContext): Promise<URL> {
   const options = ctx.options;
   await ProjectionLoader.load(options.sourceEpsg);
-  const tileId = TileId.fromTile(options.tile);
+  const tileId = ctx.tileId;
 
   const logger = ctx.logger?.child({ tileId });
 
@@ -309,7 +340,7 @@ async function createCog(ctx: CogCreationContext): Promise<URL> {
   logger?.debug({ tileId }, 'Cog:Create:VrtWarp');
 
   const cutlineProperties: { url: URL | null; blend: number } = { url: null, blend: ctx.cutline.blend };
-  if (ctx.cutline.path) {
+  if (ctx.cutline) {
     logger?.debug('Cog:Cutline');
     const optimizedCutline = ctx.cutline.optimize(options.tile);
     if (optimizedCutline) {
@@ -355,6 +386,32 @@ async function createCog(ctx: CogCreationContext): Promise<URL> {
   return cogCreateCommand.output;
 }
 
+/** Create a COG specific to LINZ's Topographic 50k and 250k map series from the creation options */
+async function createTopoCog(ctx: CogCreationContext): Promise<URL> {
+  const options = ctx.options;
+  await ProjectionLoader.load(options.sourceEpsg);
+  const tileId = ctx.tileId;
+
+  const logger = ctx.logger?.child({ tileId });
+
+  logger?.debug({ tileId }, 'TopoCog:Create:VrtSource');
+  // Create the vrt of all the source files
+  const vrtSourceCommand = gdalBuildVrt(new URL(`${tileId}-source.vrt`, ctx.tempFolder), ctx.sourceFiles, true);
+  await new GdalRunner(vrtSourceCommand).run(logger);
+
+  // Create the COG from the vrt file
+  if (ctx.size == null) throw new Error('TopoCog: Source image size is required for pixel trim');
+  const cogCreateCommand = gdalBuildTopoRasterCommands(
+    new URL(`${tileId}.tiff`, ctx.tempFolder),
+    vrtSourceCommand.output,
+    options,
+    ctx.size?.width,
+    ctx.size?.height,
+  );
+  await new GdalRunner(cogCreateCommand).run(logger);
+  return cogCreateCommand.output;
+}
+
 /**
  * Very basic checking for the output tiff to ensure it was uploaded ok
  * Just open it as a COG and ensure the metadata looks about  right

packages/cogify/src/cogify/cli/cli.cover.ts

@@ -7,10 +7,10 @@ import { Metrics } from '@linzjs/metrics';
 import { command, flag, number, oneOf, option, optional, restPositionals, string } from 'cmd-ts';
 
 import { isArgo } from '../../argo.js';
-import { CutlineOptimizer } from '../../cutline.js';
 import { getLogger, logArguments } from '../../log.js';
 import { Presets } from '../../preset.js';
-import { createTileCover, TileCoverContext } from '../../tile.cover.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
+import { createTileCover, TileCoverContext } from '../covering/tile.cover.js';
 import { RgbaType, Url, UrlFolder } from '../parsers.js';
 import { createFileStats } from '../stac.js';
 
@@ -84,6 +84,9 @@ export const BasemapsCogifyCoverCommand = command({
       throw new Error(`No collection.json found with imagery: ${im.url.href}`);
     }
 
+    const slug = im.collection?.['linz:slug'];
+    if (slug != null) im.name = slug as string;
+
     const tms = SupportedTileMatrix.find((f) => f.identifier.toLowerCase() === args.tileMatrix.toLowerCase());
     if (tms == null) throw new Error('--tile-matrix: ' + args.tileMatrix + ' not found');
 
@@ -144,11 +147,13 @@ export const BasemapsCogifyCoverCommand = command({
     const items = [];
     const tilesByZoom: number[] = [];
     for (const item of res.items) {
-      const tileId = TileId.fromTile(item.properties['linz_basemaps:options'].tile);
+      const tile = item.properties['linz_basemaps:options'].tile;
+      if (tile == null) throw new Error('Tile not found in item');
+      const tileId = TileId.fromTile(tile);
       const itemPath = new URL(`${tileId}.json`, targetPath);
       items.push({ path: itemPath });
       await fsa.write(itemPath, JSON.stringify(item, null, 2));
-      const z = item.properties['linz_basemaps:options'].tile.z;
+      const z = tile.z;
       tilesByZoom[z] = (tilesByZoom[z] ?? 0) + 1;
       ctx.logger?.trace({ path: itemPath }, 'Imagery:Stac:Item:Write');
     }