feat(backend): add Google Cloud Translation API with EU data residency support (#406)

nicobao · web-flow · commit a91c7245669b · 2025-10-29T19:50:45.000+01:00
Implement GDPR-compliant Google Cloud Translation API integration with
configurable regional endpoints to ensure data residency requirements.

Changes:
- Add GOOGLE_CLOUD_TRANSLATION_ENDPOINT config parameter to support
  regional API endpoints (e.g., translate-eu.googleapis.com)
- Set EU defaults: europe-west1 location and translate-eu.googleapis.com
  endpoint for GDPR compliance by default
- Update googleCloudAuth.ts to pass apiEndpoint option to
  TranslationServiceClient constructor
- Configure docker-compose-production.yml to mount Google Cloud service
  account credentials via volume (./&lt;name of service account&gt;.json)
- Update both api and math-updater services to initialize translation
  client with endpoint configuration
- Add comprehensive documentation in env.example for new configuration
  options

The translation service is optional and only enabled when credentials
are provided. Both endpoint and location can be overridden via
environment variables if different regions are needed.

Technical details:
- Uses Google Cloud Translation Advanced API (v3)
- Supports both AWS Secrets Manager (production) and local file
  (development) authentication methods
- Credentials mounted read-only for security
- Enhanced logging to display active endpoint configuration
diff --git a/script/docker-compose-production.yml b/script/docker-compose-production.yml
@@ -30,6 +30,9 @@ services:
       TWILIO_AUTH_TOKEN: "CHANGEME"
       TWILIO_SERVICE_SID: "CHANGEME"
       SERVER_DID_PROD: "did:web:agoracitizen.app"
+      GOOGLE_APPLICATION_CREDENTIALS: "/secrets/service-account.json"
+    volumes:
+      - ./<name of service account>.json:/secrets/service-account.json:ro
     expose:
       - "8080"
     restart: unless-stopped
@@ -41,6 +44,9 @@ services:
       NODE_ENV: "production"
       POLIS_BASE_URL: "http://polis-server:8004"
       TOTAL_VCPUS: "2" # t3.medium = 2 vCPUs (adjust for your instance type)
+      GOOGLE_APPLICATION_CREDENTIALS: "/secrets/service-account.json"
+    volumes:
+      - ./<name of service account>.json:/secrets/service-account.json:ro
     restart: unless-stopped
     logging:
       driver: awslogs
diff --git a/services/api/src/index.ts b/services/api/src/index.ts
@@ -306,6 +306,8 @@ if (
                 config.GOOGLE_APPLICATION_CREDENTIALS,
             googleCloudTranslationLocation:
                 config.GOOGLE_CLOUD_TRANSLATION_LOCATION,
+            googleCloudTranslationEndpoint:
+                config.GOOGLE_CLOUD_TRANSLATION_ENDPOINT,
             log,
         });
         log.info("[API] Google Cloud Translation initialized successfully");
diff --git a/services/api/src/shared-backend/config.ts b/services/api/src/shared-backend/config.ts
@@ -19,8 +19,12 @@ export const sharedConfigSchema = z.object({
     AWS_SECRET_REGION_READ: z.string().optional(),
     DB_HOST_READ: z.string().optional(),
     DB_PORT_READ: z.coerce.number().int().nonnegative().default(5433),
-    // Google Cloud Translation
-    GOOGLE_CLOUD_TRANSLATION_LOCATION: z.string().default("global"),
+    // Google Cloud Translation (defaults to EU for GDPR compliance)
+    GOOGLE_CLOUD_TRANSLATION_LOCATION: z.string().default("europe-west1"),
+    // API endpoint for Google Cloud Translation (defaults to EU endpoint for data residency)
+    GOOGLE_CLOUD_TRANSLATION_ENDPOINT: z
+        .string()
+        .default("translate-eu.googleapis.com"),
     // AWS Secret Manager key for Google Cloud service account JSON (production)
     // If set, this takes precedence over GOOGLE_APPLICATION_CREDENTIALS
     GOOGLE_CLOUD_SERVICE_ACCOUNT_AWS_SECRET_KEY: z.string().optional(),
diff --git a/services/api/src/shared-backend/googleCloudAuth.ts b/services/api/src/shared-backend/googleCloudAuth.ts
@@ -96,7 +96,8 @@ function parseServiceAccountJson(
  * @param googleCloudServiceAccountAwsSecretKey - AWS secret key containing service account JSON (optional)
  * @param awsSecretRegion - AWS region for Secrets Manager (required if using AWS Secrets Manager)
  * @param googleApplicationCredentialsPath - Path to service account JSON file (for local development)
- * @param googleCloudTranslationLocation - Translation API location (e.g., "global", "us-central1")
+ * @param googleCloudTranslationLocation - Translation API location (e.g., "global", "europe-west1")
+ * @param googleCloudTranslationEndpoint - API endpoint (e.g., "translate-eu.googleapis.com" for EU data residency)
  * @param log - Pino or Fastify logger instance
  * @returns GoogleCloudCredentials with initialized client and config
  * @throws Error if authentication fails or required parameters are missing
@@ -106,12 +107,14 @@ export async function initializeGoogleCloudCredentials({
     awsSecretRegion,
     googleApplicationCredentialsPath,
     googleCloudTranslationLocation,
+    googleCloudTranslationEndpoint,
     log,
 }: {
     googleCloudServiceAccountAwsSecretKey?: string;
     awsSecretRegion?: string;
     googleApplicationCredentialsPath?: string;
     googleCloudTranslationLocation: string;
+    googleCloudTranslationEndpoint?: string;
     log: pino.Logger | FastifyBaseLogger;
 }): Promise<GoogleCloudCredentials> {
     let serviceAccount: ServiceAccountCredentials;
@@ -165,10 +168,13 @@ export async function initializeGoogleCloudCredentials({
             client_email: serviceAccount.client_email,
             private_key: serviceAccount.private_key,
         },
+        ...(googleCloudTranslationEndpoint && {
+            apiEndpoint: googleCloudTranslationEndpoint,
+        }),
     });
 
     log.info(
-        `[Google Cloud Auth] Initialized Translation client for project: ${serviceAccount.project_id}`,
+        `[Google Cloud Auth] Initialized Translation client for project: ${serviceAccount.project_id}${googleCloudTranslationEndpoint ? ` with endpoint: ${googleCloudTranslationEndpoint}` : ""}`,
     );
 
     return {
diff --git a/services/math-updater/env.example b/services/math-updater/env.example
@@ -30,8 +30,11 @@ AWS_AI_LABEL_SUMMARY_MAX_TOKENS=8192
 # AWS_AI_LABEL_SUMMARY_PROMPT="Your custom prompt here..."
 
 # Google Cloud Translation API Configuration
-# Optional: Translation API location (default: "global")
-# GOOGLE_CLOUD_TRANSLATION_LOCATION=global
+# Optional: Translation API location (default: "europe-west1" for EU data residency)
+# GOOGLE_CLOUD_TRANSLATION_LOCATION=europe-west1
+
+# Optional: API endpoint (default: "translate-eu.googleapis.com" for EU data residency)
+# GOOGLE_CLOUD_TRANSLATION_ENDPOINT=translate-eu.googleapis.com
 
 # Authentication Options (choose one):
 
diff --git a/services/math-updater/src/index.ts b/services/math-updater/src/index.ts
@@ -91,6 +91,8 @@ async function main() {
                     config.GOOGLE_APPLICATION_CREDENTIALS,
                 googleCloudTranslationLocation:
                     config.GOOGLE_CLOUD_TRANSLATION_LOCATION,
+                googleCloudTranslationEndpoint:
+                    config.GOOGLE_CLOUD_TRANSLATION_ENDPOINT,
                 log,
             });
             log.info(
diff --git a/services/math-updater/src/shared-backend/config.ts b/services/math-updater/src/shared-backend/config.ts
@@ -19,8 +19,12 @@ export const sharedConfigSchema = z.object({
     AWS_SECRET_REGION_READ: z.string().optional(),
     DB_HOST_READ: z.string().optional(),
     DB_PORT_READ: z.coerce.number().int().nonnegative().default(5433),
-    // Google Cloud Translation
-    GOOGLE_CLOUD_TRANSLATION_LOCATION: z.string().default("global"),
+    // Google Cloud Translation (defaults to EU for GDPR compliance)
+    GOOGLE_CLOUD_TRANSLATION_LOCATION: z.string().default("europe-west1"),
+    // API endpoint for Google Cloud Translation (defaults to EU endpoint for data residency)
+    GOOGLE_CLOUD_TRANSLATION_ENDPOINT: z
+        .string()
+        .default("translate-eu.googleapis.com"),
     // AWS Secret Manager key for Google Cloud service account JSON (production)
     // If set, this takes precedence over GOOGLE_APPLICATION_CREDENTIALS
     GOOGLE_CLOUD_SERVICE_ACCOUNT_AWS_SECRET_KEY: z.string().optional(),
diff --git a/services/math-updater/src/shared-backend/googleCloudAuth.ts b/services/math-updater/src/shared-backend/googleCloudAuth.ts
@@ -96,7 +96,8 @@ function parseServiceAccountJson(
  * @param googleCloudServiceAccountAwsSecretKey - AWS secret key containing service account JSON (optional)
  * @param awsSecretRegion - AWS region for Secrets Manager (required if using AWS Secrets Manager)
  * @param googleApplicationCredentialsPath - Path to service account JSON file (for local development)
- * @param googleCloudTranslationLocation - Translation API location (e.g., "global", "us-central1")
+ * @param googleCloudTranslationLocation - Translation API location (e.g., "global", "europe-west1")
+ * @param googleCloudTranslationEndpoint - API endpoint (e.g., "translate-eu.googleapis.com" for EU data residency)
  * @param log - Pino or Fastify logger instance
  * @returns GoogleCloudCredentials with initialized client and config
  * @throws Error if authentication fails or required parameters are missing
@@ -106,12 +107,14 @@ export async function initializeGoogleCloudCredentials({
     awsSecretRegion,
     googleApplicationCredentialsPath,
     googleCloudTranslationLocation,
+    googleCloudTranslationEndpoint,
     log,
 }: {
     googleCloudServiceAccountAwsSecretKey?: string;
     awsSecretRegion?: string;
     googleApplicationCredentialsPath?: string;
     googleCloudTranslationLocation: string;
+    googleCloudTranslationEndpoint?: string;
     log: pino.Logger | FastifyBaseLogger;
 }): Promise<GoogleCloudCredentials> {
     let serviceAccount: ServiceAccountCredentials;
@@ -165,10 +168,13 @@ export async function initializeGoogleCloudCredentials({
             client_email: serviceAccount.client_email,
             private_key: serviceAccount.private_key,
         },
+        ...(googleCloudTranslationEndpoint && {
+            apiEndpoint: googleCloudTranslationEndpoint,
+        }),
     });
 
     log.info(
-        `[Google Cloud Auth] Initialized Translation client for project: ${serviceAccount.project_id}`,
+        `[Google Cloud Auth] Initialized Translation client for project: ${serviceAccount.project_id}${googleCloudTranslationEndpoint ? ` with endpoint: ${googleCloudTranslationEndpoint}` : ""}`,
     );
 
     return {
diff --git a/services/shared-backend/src/config.ts b/services/shared-backend/src/config.ts
@@ -18,8 +18,12 @@ export const sharedConfigSchema = z.object({
     AWS_SECRET_REGION_READ: z.string().optional(),
     DB_HOST_READ: z.string().optional(),
     DB_PORT_READ: z.coerce.number().int().nonnegative().default(5433),
-    // Google Cloud Translation
-    GOOGLE_CLOUD_TRANSLATION_LOCATION: z.string().default("global"),
+    // Google Cloud Translation (defaults to EU for GDPR compliance)
+    GOOGLE_CLOUD_TRANSLATION_LOCATION: z.string().default("europe-west1"),
+    // API endpoint for Google Cloud Translation (defaults to EU endpoint for data residency)
+    GOOGLE_CLOUD_TRANSLATION_ENDPOINT: z
+        .string()
+        .default("translate-eu.googleapis.com"),
     // AWS Secret Manager key for Google Cloud service account JSON (production)
     // If set, this takes precedence over GOOGLE_APPLICATION_CREDENTIALS
     GOOGLE_CLOUD_SERVICE_ACCOUNT_AWS_SECRET_KEY: z.string().optional(),
diff --git a/services/shared-backend/src/googleCloudAuth.ts b/services/shared-backend/src/googleCloudAuth.ts
@@ -95,7 +95,8 @@ function parseServiceAccountJson(
  * @param googleCloudServiceAccountAwsSecretKey - AWS secret key containing service account JSON (optional)
  * @param awsSecretRegion - AWS region for Secrets Manager (required if using AWS Secrets Manager)
  * @param googleApplicationCredentialsPath - Path to service account JSON file (for local development)
- * @param googleCloudTranslationLocation - Translation API location (e.g., "global", "us-central1")
+ * @param googleCloudTranslationLocation - Translation API location (e.g., "global", "europe-west1")
+ * @param googleCloudTranslationEndpoint - API endpoint (e.g., "translate-eu.googleapis.com" for EU data residency)
  * @param log - Pino or Fastify logger instance
  * @returns GoogleCloudCredentials with initialized client and config
  * @throws Error if authentication fails or required parameters are missing
@@ -105,12 +106,14 @@ export async function initializeGoogleCloudCredentials({
     awsSecretRegion,
     googleApplicationCredentialsPath,
     googleCloudTranslationLocation,
+    googleCloudTranslationEndpoint,
     log,
 }: {
     googleCloudServiceAccountAwsSecretKey?: string;
     awsSecretRegion?: string;
     googleApplicationCredentialsPath?: string;
     googleCloudTranslationLocation: string;
+    googleCloudTranslationEndpoint?: string;
     log: pino.Logger | FastifyBaseLogger;
 }): Promise<GoogleCloudCredentials> {
     let serviceAccount: ServiceAccountCredentials;
@@ -164,10 +167,13 @@ export async function initializeGoogleCloudCredentials({
             client_email: serviceAccount.client_email,
             private_key: serviceAccount.private_key,
         },
+        ...(googleCloudTranslationEndpoint && {
+            apiEndpoint: googleCloudTranslationEndpoint,
+        }),
     });
 
     log.info(
-        `[Google Cloud Auth] Initialized Translation client for project: ${serviceAccount.project_id}`,
+        `[Google Cloud Auth] Initialized Translation client for project: ${serviceAccount.project_id}${googleCloudTranslationEndpoint ? ` with endpoint: ${googleCloudTranslationEndpoint}` : ""}`,
     );
 
     return {
