Merge pull request #7 from thatapicompany/codex/add-bigquery-enrichment-for-places

feat: add optional BigQuery-backed enrichment for Places

Author: AdenForshaw

.env.example

@@ -1,4 +1,10 @@
 GOOGLE_APPLICATION_CREDENTIALS=gcp-credentials.json
 BIGQUERY_PROJECT_ID=your-GCP-project-id
 GCS_BUCKET_NAME=your-GCS-bucket-name
-AUTH_API_ACCESS_KEY=create-one-from-TheAuthAPI.com
+AUTH_API_ACCESS_KEY=create-one-from-TheAuthAPI.com
+ENABLE_ENRICHMENT=false
+ENRICHMENT_ADAPTER_PACKAGE=
+ENRICHMENT_FIELDS_ALLOWLIST=
+ENRICHMENT_BQ_PROJECT=
+ENRICHMENT_BQ_DATASET=
+ENRICHMENT_BQ_TABLE=

README.md

@@ -22,6 +22,17 @@ The hosted version of the Overture Maps API has additional datasets including th
 
 - [Deploy to Google Cloud Platform using Cloudrun and BigQuery](./docs/google-cloud-platform.md)
 
+### Enrichment
+
+Enrichment is disabled by default. Set `ENABLE_ENRICHMENT=true` in hosted deployments and provide the following environment variables to enable the built-in BigQuery-backed enrichment:
+
+- `ENRICHMENT_BQ_PROJECT`
+- `ENRICHMENT_BQ_DATASET`
+- `ENRICHMENT_BQ_TABLE`
+- `ENRICHMENT_FIELDS_ALLOWLIST` (optional comma-separated field names)
+
+Alternative adapters can be supplied via `ENRICHMENT_ADAPTER_PACKAGE`. This allows future data sources such as Postgres without changing the public API.
+
 ## API Endpoints
 
 - [OpenAPI Spec Doc](https://api.overturemapsapi.com/api-docs.json)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "overture-maps-api",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "",
   "author": "",
   "private": true,

src/bigquery/bigquery.service.ts

@@ -27,10 +27,17 @@ export class BigQueryService {
   logger = new Logger('BigQueryService');
 
   constructor() {
-    this.bigQueryClient = new BigQuery({
-      projectId: process.env.BIGQUERY_PROJECT_ID,
-      keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS,
-    });
+
+    const config: any = {
+      projectId: process.env.BIGQUERY_PROJECT_ID
+    };
+
+    // Only add keyFilename if GOOGLE_APPLICATION_CREDENTIALS is set
+    if (process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+      config.keyFilename = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+    }
+
+    this.bigQueryClient = new BigQuery(config);
   }
 
 

src/data-adapters/EnrichmentAdapter.ts

@@ -0,0 +1,15 @@
+export interface EnrichmentAdapter {
+  /**
+   * Fetch enrichment data for a set of place IDs.
+   * A future PgEnrichmentAdapter can be injected via ENRICHMENT_ADAPTER_PACKAGE without API changes.
+   */
+  fetchEnrichmentByIds(
+    ids: string[],
+    options?: { fields?: string[] }
+  ): Promise<Record<string, Record<string, unknown>>>;
+
+  /**
+   * List supported enrichment fields.
+   */
+  supportedFields(): Promise<string[]>;
+}

src/data-adapters/OpenSourceEnrichmentAdapter.ts

@@ -0,0 +1,14 @@
+import { EnrichmentAdapter } from './EnrichmentAdapter';
+
+export class OpenSourceEnrichmentAdapter implements EnrichmentAdapter {
+  async fetchEnrichmentByIds(
+    ids: string[],
+    options?: { fields?: string[] }
+  ): Promise<Record<string, Record<string, unknown>>> {
+    return {};
+  }
+
+  async supportedFields(): Promise<string[]> {
+    return [];
+  }
+}

src/data-adapters/bq-enrichment/BqEnrichmentAdapter.ts

@@ -0,0 +1,90 @@
+import { BigQuery } from '@google-cloud/bigquery';
+import { EnrichmentAdapter } from '../EnrichmentAdapter';
+import { Logger } from '@nestjs/common';
+
+function chunk<T>(arr: T[], size: number): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < arr.length; i += size) {
+    chunks.push(arr.slice(i, i + size));
+  }
+  return chunks;
+}
+
+export class BqEnrichmentAdapter implements EnrichmentAdapter {
+  private client: BigQuery;
+  private project: string;
+  private dataset: string;
+  private table: string;
+  private allowlist: string[];
+  private cachedFields?: string[];
+
+  private logger: Logger;
+  constructor() {
+
+      this.logger = new Logger('BqEnrichmentAdapter');
+
+    this.project = process.env.ENRICHMENT_BQ_PROJECT || '';
+    this.dataset = process.env.ENRICHMENT_BQ_DATASET || '';
+    this.table = process.env.ENRICHMENT_BQ_TABLE || '';
+    this.allowlist = (process.env.ENRICHMENT_FIELDS_ALLOWLIST || '')
+      .split(',')
+      .map((s) => s.trim())
+      .filter((s) => s.length > 0);
+    this.logger.log(
+      `Enrichment fields allowlist: ${this.allowlist.join(', ')}`
+    );
+    const config: any = {
+      projectId: this.project,
+    };
+
+    // Only add keyFilename if GOOGLE_APPLICATION_CREDENTIALS is set
+    if (process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+      config.keyFilename = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+    }
+
+    this.client = new BigQuery(config);
+  }
+
+  async supportedFields(): Promise<string[]> {
+    if (this.cachedFields) return this.cachedFields;
+    const [metadata] = await this.client
+      .dataset(this.dataset)
+      .table(this.table)
+      .getMetadata();
+    let fields = (metadata.schema?.fields || [])
+      .map((f: any) => f.name)
+      .filter((name: string) => name !== 'id');
+    if (this.allowlist.length) {
+      fields = fields.filter((f) => this.allowlist.includes(f));
+    }
+    this.cachedFields = fields;
+    return fields;
+  }
+
+  async fetchEnrichmentByIds(
+    ids: string[],
+    options?: { fields?: string[] },
+  ): Promise<Record<string, Record<string, unknown>>> {
+    if (!ids.length) return {};
+    const allowed = await this.supportedFields();
+    if (!allowed.length) {
+      return {};
+    }
+    const requested = options?.fields?.length
+      ? options.fields.filter((f) => allowed.includes(f))
+      : allowed;
+
+    const projection = requested.length ? requested.join(', ') : '';
+    const results: Record<string, Record<string, unknown>> = {};
+    for (const batch of chunk(ids, 5000)) {
+      const query = `SELECT id${projection ? ', ' + projection : ''} FROM \`${this.project}.${this.dataset}.${this.table}\` WHERE id IN UNNEST(@ids)`;
+      const options = { query, params: { ids: batch } } as any;
+      const [rows] = await this.client.query(options);
+      for (const row of rows as any[]) {
+        const { id, ...rest } = row;
+        results[id] = rest;
+      }
+    }
+    return results;
+  }
+}

src/data-adapters/loadEnrichmentAdapter.spec.ts

@@ -0,0 +1,10 @@
+import { loadEnrichmentAdapter } from './loadEnrichmentAdapter';
+import { OpenSourceEnrichmentAdapter } from './OpenSourceEnrichmentAdapter';
+
+describe('loadEnrichmentAdapter', () => {
+  it('returns OpenSourceEnrichmentAdapter when disabled', () => {
+    process.env.ENABLE_ENRICHMENT = 'false';
+    const adapter = loadEnrichmentAdapter();
+    expect(adapter).toBeInstanceOf(OpenSourceEnrichmentAdapter);
+  });
+});

src/data-adapters/loadEnrichmentAdapter.ts

@@ -0,0 +1,28 @@
+import { Logger } from '@nestjs/common';
+import { EnrichmentAdapter } from './EnrichmentAdapter';
+import { OpenSourceEnrichmentAdapter } from './OpenSourceEnrichmentAdapter';
+
+// Loader that selects the enrichment adapter at runtime. A PgEnrichmentAdapter can
+// be plugged in via ENRICHMENT_ADAPTER_PACKAGE in the future.
+export function loadEnrichmentAdapter(): EnrichmentAdapter {
+  const logger = new Logger('EnrichmentAdapterLoader');
+
+  try {
+    if (process.env.ENABLE_ENRICHMENT !== 'true') {
+      return new OpenSourceEnrichmentAdapter();
+    }
+
+    if (process.env.ENRICHMENT_ADAPTER_PACKAGE) {
+      const pkg = require(process.env.ENRICHMENT_ADAPTER_PACKAGE);
+      const Adapter = pkg.default || pkg;
+      return new Adapter();
+    }
+
+    // Default to built-in BigQuery adapter
+    const { BqEnrichmentAdapter } = require('./bq-enrichment/BqEnrichmentAdapter');
+    return new BqEnrichmentAdapter();
+  } catch (err: any) {
+    logger.warn(`Failed to load enrichment adapter: ${err.message}`);
+    return new OpenSourceEnrichmentAdapter();
+  }
+}

src/gcs/gcs.service.ts

@@ -17,10 +17,17 @@ export class GcsService {
       this.logger.error('GCS environment variables not set');
       return;
     }
-    this.storage = new Storage({
-      projectId: process.env.BIGQUERY_PROJECT_ID,
-      keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS,
-    });
+
+    const config: any = {
+      projectId: process.env.BIGQUERY_PROJECT_ID
+    };
+
+    // Only add keyFilename if GOOGLE_APPLICATION_CREDENTIALS is set
+    if (process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+      config.keyFilename = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+    }
+
+    this.storage = new Storage(config);
 
     this.bucket = this.storage.bucket(process.env.GCS_BUCKET_NAME);