feat: allow config to be loaded from a environment variable

.github/workflows/push.yml

@@ -22,6 +22,10 @@ jobs:
     runs-on: ubuntu-latest
     concurrency: deploy-${{ github.ref }}
 
+    outputs:
+      # Current configuration location
+      config_location: ${{ steps.config.outputs.config_location }}
+
     permissions:
       id-token: write
       contents: write
@@ -73,6 +77,13 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NODE_AUTH_TOKEN: ${{secrets.NPM_AUTH_TOKEN_LINZJS}}
 
+      - name: Get current configuration location
+        id: config
+        run: |
+          CONFIG_HASH=$(curl https://basemaps.linz.govt.nz/v1/version | jq .config.hash)
+
+          echo "config_location=s3://linz-basemaps/config/config-${CONFIG_HASH}.json.gz" >> $GITHUB_OUTPUT
+
       - name: Benchmark
         uses: blacha/hyperfine-action@v1
 
@@ -124,6 +135,7 @@ jobs:
           npx lerna run bundle --stream
           npx lerna run deploy:deploy --stream
         env:
+          BASEMAPS_CONFIG_LOCATION: "s3://linz-basemaps/config/"
           BASEMAPS_API_KEY_BLOCKS: ${{ secrets.BASEMAPS_API_KEY_BLOCKS }}
           GOOGLE_ANALYTICS: ${{ secrets.GOOGLE_ANALYTICS }}
           NODE_ENV: 'dev'
@@ -180,6 +192,7 @@ jobs:
           npx lerna run bundle --stream
           npx lerna run deploy:deploy --stream
         env:
+          BASEMAPS_CONFIG_LOCATION: "s3://linz-basemaps/config/"
           BASEMAPS_API_KEY_BLOCKS: ${{ secrets.BASEMAPS_API_KEY_BLOCKS }}
           GOOGLE_ANALYTICS: ${{ secrets.GOOGLE_ANALYTICS }}
           NODE_ENV: 'production'

packages/_infra/README.md

@@ -29,6 +29,10 @@ CDK_DEFAULT_ACCOUNT;
 
 // Due to the convoluted way that TLS certificates are made inside LINZ a hard coded TLS ARN is needed for the Cloudfront
 CLOUDFRONT_CERTIFICATE_ARN;
+
+// Configuration is stored as JSON files, a configuration file is required generally stored in s3
+// eg: s3://linz-basemaps/config/config-:someConfigHash.json.gz
+BASEMAPS_CONFIG_LOCATION;
 ```
 
 For first usage you will need to bootstrap the account, this will create a s3 bucket to store CDK assets in

packages/_infra/src/serve/lambda.tiler.ts

@@ -33,9 +33,11 @@ export class LambdaTiler extends Construct {
     const environment: Record<string, string> = {
       [Env.PublicUrlBase]: config.PublicUrlBase,
       [Env.AwsRoleConfigPath]: `s3://${config.AwsRoleConfigBucket}/config.json`,
+      [Env.ConfigLocation]: Env.get(Env.ConfigLocation) as string,
       AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
     };
 
+    if (environment[Env.ConfigLocation] == null) throw new Error(`$${Env.ConfigLocation} is required`);
     if (props.staticBucketName) {
       environment[Env.StaticAssetLocation] = `s3://${props.staticBucketName}/`;
     }

packages/cli-config/src/cli/action.import.ts

@@ -10,18 +10,9 @@ import {
   TileSetType,
 } from '@basemaps/config';
 import { GoogleTms, Nztm2000QuadTms, TileMatrixSet } from '@basemaps/geo';
-import {
-  Env,
-  fsa,
-  getDefaultConfig,
-  getLogger,
-  getPreviewUrl,
-  logArguments,
-  LogType,
-  setDefaultConfig,
-} from '@basemaps/shared';
+import { Env, fsa, getLogger, getPreviewUrl, logArguments, LogType } from '@basemaps/shared';
 import { CliInfo } from '@basemaps/shared/build/cli/info.js';
-import { command, flag, option, optional, string } from 'cmd-ts';
+import { command, flag, option, string } from 'cmd-ts';
 import fetch from 'node-fetch';
 
 import { invalidateCache } from '../util.js';
@@ -49,7 +40,7 @@ export const ImportCommand = command({
       description: 'Output a markdown file with the config changes',
     }),
     target: option({
-      type: optional(string),
+      type: string,
       long: 'target',
       description: 'Target config file to compare',
     }),
@@ -144,17 +135,13 @@ export const ImportCommand = command({
   },
 });
 
-async function getConfig(logger: LogType, target?: string): Promise<BasemapsConfigProvider> {
-  if (target) {
-    logger.info({ config: target }, 'Import:Target:Load');
-    const configJson = await fsa.readJson<ConfigBundled>(fsa.toUrl(target));
-    const mem = ConfigProviderMemory.fromJson(configJson);
-    mem.createVirtualTileSets();
+async function getConfig(logger: LogType, target: string): Promise<BasemapsConfigProvider> {
+  logger.info({ config: target }, 'Import:Target:Load');
+  const configJson = await fsa.readJson<ConfigBundled>(fsa.toUrl(target));
+  const mem = ConfigProviderMemory.fromJson(configJson);
+  mem.createVirtualTileSets();
 
-    setDefaultConfig(mem);
-    return mem;
-  }
-  return getDefaultConfig();
+  return mem;
 }
 
 const promises: Promise<boolean>[] = [];

packages/config/src/memory/memory.config.ts

@@ -69,8 +69,14 @@ function removeUndefined(obj: unknown): void {
 export class ConfigProviderMemory extends BasemapsConfigProvider {
   override type = 'memory' as const;
 
+  static is(cfg: BasemapsConfigProvider): cfg is ConfigProviderMemory {
+    return cfg.type === 'memory';
+  }
+
   /** Optional id of the configuration */
   id?: string;
+  /** Optional hash of the config if the config was loaded from JSON */
+  hash?: string;
 
   Imagery = new MemoryConfigObject<ConfigImagery>(this, ConfigPrefix.Imagery);
   Style = new MemoryConfigObject<ConfigVectorStyle>(this, ConfigPrefix.Style);
@@ -245,6 +251,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
 
     mem.assets = cfg.assets;
     mem.id = cfg.id;
+    mem.hash = cfg.hash;
 
     return mem;
   }

packages/lambda-tiler/src/__tests__/index.test.ts

@@ -1,7 +1,10 @@
 import assert from 'node:assert';
 import { afterEach, describe, it } from 'node:test';
 
+import { ConfigProviderMemory } from '@basemaps/config';
+
 import { handler } from '../index.js';
+import { ConfigLoader } from '../util/config.loader.js';
 import { mockRequest } from './xyz.util.js';
 
 describe('LambdaXyz index', () => {
@@ -17,7 +20,10 @@ describe('LambdaXyz index', () => {
       delete process.env['BUILD_ID'];
     });
 
-    it('should return version', async () => {
+    it('should return version', async (t) => {
+      const config = new ConfigProviderMemory();
+      t.mock.method(ConfigLoader, 'getDefaultConfig', () => config);
+
       process.env['GIT_VERSION'] = '1.2.3';
       process.env['GIT_HASH'] = 'abc456';
 
@@ -32,7 +38,10 @@ describe('LambdaXyz index', () => {
       });
     });
 
-    it('should include buildId if exists', async () => {
+    it('should include buildId if exists', async (t) => {
+      const config = new ConfigProviderMemory();
+      t.mock.method(ConfigLoader, 'getDefaultConfig', () => config);
+
       process.env['GIT_VERSION'] = '1.2.3';
       process.env['BUILD_ID'] = '1658821493-3';
 
@@ -46,6 +55,25 @@ describe('LambdaXyz index', () => {
         buildId: '1658821493-3',
       });
     });
+
+    it('should return config information if present', async (t) => {
+      const config = new ConfigProviderMemory();
+      t.mock.method(ConfigLoader, 'getDefaultConfig', () => config);
+      config.id = 'config-id';
+
+      const response = await handler.router.handle(mockRequest('/v1/version'));
+      assert.deepEqual(JSON.parse(response.body), {
+        version: 'dev',
+        config: { id: 'config-id' },
+      });
+
+      config.hash = 'config-hash';
+      const responseHash = await handler.router.handle(mockRequest('/v1/version'));
+      assert.deepEqual(JSON.parse(responseHash.body), {
+        version: 'dev',
+        config: { id: 'config-id', hash: 'config-hash' },
+      });
+    });
   });
 
   it('should respond to /ping', async () => {

packages/lambda-tiler/src/cli/render.preview.ts

@@ -1,11 +1,12 @@
 import { ConfigProviderMemory } from '@basemaps/config';
 import { initConfigFromUrls } from '@basemaps/config-loader';
 import { TileMatrixSet, TileMatrixSets } from '@basemaps/geo';
-import { fsa, LogConfig, setDefaultConfig } from '@basemaps/shared';
+import { fsa, LogConfig } from '@basemaps/shared';
 import { LambdaHttpRequest, LambdaUrlRequest, UrlEvent } from '@linzjs/lambda';
 import { Context } from 'aws-lambda';
 
 import { renderPreview } from '../routes/preview.js';
+import { ConfigLoader } from '../util/config.loader.js';
 
 const target = fsa.toUrl(`/home/blacha/tmp/basemaps/bm-724/test-north-island_20230220_10m/`);
 const location = { lat: -39.0852555, lon: 177.3998405 };
@@ -17,7 +18,7 @@ let tileMatrix: TileMatrixSet | null = null;
 async function main(): Promise<void> {
   const log = LogConfig.get();
   const provider = new ConfigProviderMemory();
-  setDefaultConfig(provider);
+  ConfigLoader.setDefaultConfig(provider);
   const { tileSet, imagery } = await initConfigFromUrls(provider, [target]);
 
   if (tileSet.layers.length === 0) throw new Error('No imagery found in path: ' + target.href);

packages/lambda-tiler/src/cli/render.tile.ts

@@ -1,12 +1,13 @@
 import { ConfigProviderMemory } from '@basemaps/config';
 import { initConfigFromUrls } from '@basemaps/config-loader';
 import { Tile, TileMatrixSet, TileMatrixSets } from '@basemaps/geo';
-import { fsa, LogConfig, setDefaultConfig } from '@basemaps/shared';
+import { fsa, LogConfig } from '@basemaps/shared';
 import { LambdaHttpRequest, LambdaUrlRequest, UrlEvent } from '@linzjs/lambda';
 import { Context } from 'aws-lambda';
 import { extname } from 'path';
 
 import { TileXyzRaster } from '../routes/tile.xyz.raster.js';
+import { ConfigLoader } from '../util/config.loader.js';
 
 // Render configuration
 const source = fsa.toUrl(`/home/blacha/data/elevation/christchurch_2020-2021/`);
@@ -27,7 +28,7 @@ async function main(): Promise<void> {
   const log = LogConfig.get();
   log.level = 'trace';
   const provider = new ConfigProviderMemory();
-  setDefaultConfig(provider);
+  ConfigLoader.setDefaultConfig(provider);
   const { imagery, tileSets } = await initConfigFromUrls(provider, [source]);
 
   const tileSet = tileSets.find((f) => f.layers.length > 0);

packages/lambda-tiler/src/routes/__tests__/fonts.test.ts

@@ -1,8 +1,7 @@
 import assert from 'node:assert';
 import { afterEach, beforeEach, describe, it } from 'node:test';
 
-import { base58, BaseConfig, ConfigProviderMemory } from '@basemaps/config';
-import { setDefaultConfig } from '@basemaps/shared';
+import { base58, ConfigProviderMemory } from '@basemaps/config';
 import { fsa, FsMemory } from '@chunkd/fs';
 
 import { Api, mockRequest, mockUrlRequest } from '../../__tests__/xyz.util.js';
@@ -20,13 +19,14 @@ describe('/v1/fonts', () => {
     config.objects.clear();
     fsa.register('memory://', memory);
     config.assets = 'memory://assets/';
-    setDefaultConfig(config);
+    ConfigLoader.setDefaultConfig(config);
   });
 
   afterEach(() => {
     CachedConfig.cache.clear();
     CoSources.cache.clear();
     memory.files.clear();
+    ConfigLoader._defaultConfig = undefined;
   });
 
   it('should return 404 if no font found', async (t) => {
@@ -84,28 +84,6 @@ describe('/v1/fonts', () => {
     assert.equal(res.status, 404);
   });
 
-  it('should get the correct utf8 font with default assets', async () => {
-    config.assets = undefined;
-    const configJson = config.toJson();
-    await fsa.write(new URL(`memory://config-${configJson.hash}.json`), JSON.stringify(configJson));
-
-    config.objects.set('cb_latest', {
-      id: 'cb_latest',
-      name: 'latest',
-      path: `memory://config-${configJson.hash}.json`,
-      hash: configJson.hash,
-      assets: 'memory://new-location/',
-    } as BaseConfig);
-
-    await fsa.write(new URL('memory://new-location/fonts/Roboto Thin/0-255.pbf'), Buffer.from(''));
-    const res255 = await handler.router.handle(mockRequest('/v1/fonts/Roboto Thin/0-255.pbf'));
-    assert.equal(res255.status, 200, res255.statusDescription);
-    assert.equal(res255.header('content-type'), 'application/x-protobuf');
-    assert.equal(res255.header('content-encoding'), undefined);
-    assert.notEqual(res255.header('etag'), undefined);
-    assert.equal(res255.header('cache-control'), 'public, max-age=604800, stale-while-revalidate=86400');
-  });
-
   it('should get the correct utf8 font with config assets', async (t) => {
     t.mock.method(ConfigLoader, 'getDefaultConfig', () => config);
 

packages/lambda-tiler/src/routes/version.ts

@@ -1,6 +1,16 @@
-import { HttpHeader, LambdaHttpResponse } from '@linzjs/lambda';
+import { BasemapsConfigProvider, ConfigProviderMemory } from '@basemaps/config';
+import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
 
-export function versionGet(): Promise<LambdaHttpResponse> {
+import { ConfigLoader } from '../util/config.loader.js';
+
+function getConfigHash(cfg: BasemapsConfigProvider): { id: string; hash?: string } | undefined {
+  if (!ConfigProviderMemory.is(cfg)) return undefined;
+  if (cfg.id == null) return undefined;
+  return { id: cfg.id, hash: cfg.hash };
+}
+
+export async function versionGet(req: LambdaHttpRequest): Promise<LambdaHttpResponse> {
+  const config = await ConfigLoader.load(req);
   const response = new LambdaHttpResponse(200, 'ok');
   response.header(HttpHeader.CacheControl, 'no-store');
   response.json({
@@ -9,16 +19,26 @@ export function versionGet(): Promise<LambdaHttpResponse> {
      * @example "v6.42.1"
      */
     version: process.env['GIT_VERSION'] ?? 'dev',
+
     /**
      * Full git commit hash
      * @example "e4231b1ee62c276c8657c56677ced02681dfe5d6"
      */
     hash: process.env['GIT_HASH'],
+
     /**
+     *
      * The exact build that this release was run from
      * @example "1658821493-3"
      */
     buildId: process.env['BUILD_ID'],
+
+    /**
+     * Configuration id that was used to power this config
+     * @example { "id": "cb_01JTQ7ZK49F8EY4N5DRJ3XFT73", hash: "HcByZ8WS2zpaTxFJp6wSKg2eUpwahLqAGEQdcDxKxqp6" }
+     */
+    config: getConfigHash(config),
   });
+
   return Promise.resolve(response);
 }

packages/lambda-tiler/src/util/config.loader.ts

@@ -1,7 +1,6 @@
 import { base58, BasemapsConfigProvider, isBase58 } from '@basemaps/config';
-import { fsa, getDefaultConfig } from '@basemaps/shared';
-import { LambdaHttpResponse } from '@linzjs/lambda';
-import { LambdaHttpRequest } from '@linzjs/lambda';
+import { Env, fsa } from '@basemaps/shared';
+import { LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
 
 import { CachedConfig } from './config.cache.js';
 
@@ -17,20 +16,22 @@ const SafeBuckets = new Set([
 const SafeProtocols = new Set([new URL('s3://foo').protocol, new URL('memory://foo.json').protocol]);
 
 export class ConfigLoader {
+  static _defaultConfig: BasemapsConfigProvider | undefined;
+  static setDefaultConfig(config: BasemapsConfigProvider): void {
+    this._defaultConfig = config;
+  }
   /** Exposed for testing */
   static async getDefaultConfig(req?: LambdaHttpRequest): Promise<BasemapsConfigProvider> {
-    const config = getDefaultConfig();
+    if (ConfigLoader._defaultConfig) return ConfigLoader._defaultConfig;
 
-    // Look up the latest config bundle out of dynamodb, then load the config from the provided path
-    const cb = await config.ConfigBundle.get('cb_latest');
-    if (cb == null) return config;
+    const configLocation = Env.get(Env.ConfigLocation);
+    if (configLocation == null) throw new Error(`Missing configuration: $${Env.ConfigLocation}`);
 
     req?.timer.start('config:load');
 
-    return CachedConfig.get(fsa.toUrl(cb.path)).then((cfg) => {
+    return CachedConfig.get(fsa.toUrl(configLocation)).then((cfg) => {
       req?.timer.end('config:load');
       if (cfg == null) throw new LambdaHttpResponse(500, 'Unable to find latest configuration');
-      if (cfg.assets == null) cfg.assets = cb.assets;
       return cfg;
     });
   }