Revert the mbtile file path to string instead of url

Author: Wentao-Kuang

packages/cli-vector/src/cli/cli.join.ts

@@ -1,5 +1,5 @@
 import { TileMatrixSets } from '@basemaps/geo';
-import { fsa, LogType, Url, UrlArrayJsonFile } from '@basemaps/shared';
+import { fsa, isArgo, LogType, Url, UrlArrayJsonFile } from '@basemaps/shared';
 import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
 import { getLogger, logArguments } from '@basemaps/shared/build/cli/log.js';
 import { command, option, string } from 'cmd-ts';
@@ -109,7 +109,8 @@ export const JoinCommand = command({
     const bucketPath = new URL(`vector/${tileMatrix.projection.code}/`, args.target);
     const filePaths = await download(args.fromFile, outputPath, logger);
 
-    const outputMbtiles = fsa.toUrl(`${outputPath}/${args.filename}.mbtiles`);
+    // Mbtiles output path to be string type path to avoid issues with tippecanoe and better-sqlite3
+    const outputMbtiles = `${outputPath}/${args.filename}.mbtiles`;
     logger.info({ files: filePaths.length, outputMbtiles }, 'JoinMbtiles: Start');
     await tileJoin(filePaths, outputMbtiles, logger);
     logger.info({ files: filePaths.length, outputMbtiles }, 'JoinMbtiles: End');
@@ -130,13 +131,19 @@ export const JoinCommand = command({
 
     // Upload output to s3
     logger.info({ target: bucketPath, tileMatrix: tileMatrix.identifier }, 'Upload: Start');
-    await upload(outputMbtiles, bucketPath, logger);
+    await upload(fsa.toUrl(outputMbtiles), bucketPath, logger);
     await upload(outputCotar, bucketPath, logger);
     await upload(outputIndex, bucketPath, logger);
     // Upload stac Files
     for (const file of stacFiles) {
       await upload(file, bucketPath, logger);
     }
     logger.info({ target: bucketPath, tileMatrix: tileMatrix.identifier }, 'Upload: End');
+
+    // Write output target for argo tasks to create pull request
+    if (isArgo()) {
+      const target = new URL(`topographic/${CliId}/${args.filename}.tar.co`, bucketPath);
+      await fsa.write(fsa.toUrl('/tmp/target'), JSON.stringify([target]));
+    }
   },
 });

packages/cli-vector/src/transform/mbtiles.to.ttiles.ts

@@ -1,4 +1,4 @@
-import { LogType, urlToString } from '@basemaps/shared';
+import { LogType } from '@basemaps/shared';
 import bs3 from 'better-sqlite3';
 import { createWriteStream } from 'fs';
 import * as tar from 'tar-stream';
@@ -17,7 +17,9 @@ export function xyzToPath(x: number | string, y: number | string, z: number | st
 export async function* readMbTiles(
   fileName: string,
   limit = -1,
+  logger: LogType,
 ): AsyncGenerator<{ tile: TileTable; index: number; total: number }, null> {
+  logger.debug({ file: fileName }, 'ReadMbTiles:Start');
   const db = bs3(fileName);
   let limitQuery = '';
   if (limit > 0) limitQuery = 'LIMIT ' + limit;
@@ -27,19 +29,20 @@ export async function* readMbTiles(
 
   let index = 0;
   for (const tile of query.iterate()) yield { tile: tile as TileTable, index: index++, total: total as number };
+  logger.debug({ file: fileName }, 'ReadMbTiles:End');
   return null;
 }
 
-export async function toTarTiles(input: URL, output: URL, logger: LogType, limit = -1): Promise<void> {
+export async function toTarTiles(input: string, output: URL, logger: LogType, limit = -1): Promise<void> {
   const packer = tar.pack();
   const startTime = Date.now();
   let writeCount = 0;
   const writeProm = new Promise((resolve) => packer.on('end', resolve));
 
-  packer.pipe(createWriteStream(urlToString(output)));
+  packer.pipe(createWriteStream(output));
 
   let startTileTime = Date.now();
-  for await (const { tile, index, total } of readMbTiles(input.pathname, limit)) {
+  for await (const { tile, index, total } of readMbTiles(input, limit, logger)) {
     if (index === 0) logger.info({ path: output.href, count: total }, 'Covt.Tar:Start');
 
     const z = tile.zoom_level;

packages/cli-vector/src/transform/tippecanoe.ts

@@ -57,13 +57,13 @@ export async function tippecanoe(input: URL, output: URL, layer: Layer, logger:
  *
  * @returns { cmd: string; args: string[] } cmd and arguments for tippecanoe tile-join docker command
  */
-export async function tileJoin(inputs: URL[], output: URL, logger: LogType): Promise<void> {
+export async function tileJoin(inputs: URL[], output: string, logger: LogType): Promise<void> {
   const cmd = Command.create('tile-join');
 
-  cmd.mount(dirname(output.pathname));
+  cmd.mount(dirname(output));
 
   cmd.args.push('-pk');
-  cmd.args.push('-o', output.pathname);
+  cmd.args.push('-o', output);
   for (const input of inputs) {
     if (input.href.endsWith('mbtiles')) {
       cmd.mount(dirname(input.pathname));