feat(export): set up multi-service export capabilities

At this time the behavior should be the same, but
there is an additional layer acting like a state machine
to schedule the work, determine if it's finished, and
notify the user if it is.

Author: kschelonka

infrastructure/account-data-deleter/src/config/index.ts

@@ -43,14 +43,14 @@ export const config = {
     listImportBatchQueue: `${prefix}-List-Import-Batches`,
     databaseTz: 'US/Central',
     eventBusName: `PocketEventBridge-${environment}-Shared-Event-Bus`,
-    shareListSecretName: `ShareableListsApi-Prod/shareablelistsapi-prod20230207160218618400000003`,
   },
   isDev,
   isProd,
   lambda: {
     snsTopicName: {
       userEvents: `PocketEventBridge-${environment}-UserEvents`,
       listEvents: `PocketEventBridge-${environment}-ListEvents`,
+      exportUpdateEvents: `PocketEventBridge-${environment}-ListExportReadyEvents`,
     },
     batchDeleteLambda: {
       name: 'BatchDeleteLambda',

infrastructure/account-data-deleter/src/dataDeleterApp.ts

@@ -122,6 +122,10 @@ export class DataDeleterApp extends Construct {
               name: 'EXPORT_REQUEST_QUEUE_URL',
               value: `https://sqs.${region.name}.amazonaws.com/${caller.accountId}/${config.envVars.exportRequestQueueName}`,
             },
+            {
+              name: 'EXPORT_REQUEST_STATE_TABLE',
+              value: this.config.exportStateDb.name,
+            },
             {
               name: 'SQS_LIST_EXPORT_QUEUE_URL',
               value: `https://sqs.${region.name}.amazonaws.com/${caller.accountId}/${config.envVars.listExportQueueName}`,

infrastructure/account-data-deleter/src/main.ts

@@ -150,6 +150,22 @@ class AccountDataDeleter extends TerraformStack {
       },
     );
 
+    // Forward status updates on export components (shareable list, list, annotations)
+    new ApplicationSqsSnsTopicSubscription(
+      this,
+      'export-status-events-sns-subscription',
+      {
+        name: `${config.envVars.exportRequestQueueName}-Status-SNS`,
+        snsTopicArn: `arn:aws:sns:${pocketVpc.region}:${pocketVpc.accountId}:${config.lambda.snsTopicName.exportUpdateEvents}`,
+        sqsQueue: exportRequestQueue.sqsQueue,
+        filterPolicyScope: 'MessageBody',
+        filterPolicy: JSON.stringify({
+          'detail-type': ['export-part-complete'],
+        }),
+        tags: config.tags,
+      },
+    );
+
     // Bucket for exports plus auto-expiry rules
     const exportBucket = new s3Bucket.S3Bucket(this, 'list-export-bucket', {
       bucket: `com.getpocket-${config.environment.toLowerCase()}.list-exports`,

infrastructure/pocket-event-bridge/src/main.ts

@@ -272,8 +272,11 @@ class PocketEventBus extends TerraformStack {
       name: 'ListExportReadyEvents',
       tags: config.tags,
       eventPattern: {
-        'detail-type': [PocketEventType.EXPORT_READY],
-        source: ['account-data-deleter'],
+        'detail-type': [
+          PocketEventType.EXPORT_READY,
+          PocketEventType.EXPORT_PART_COMPLETE,
+        ],
+        source: ['account-data-deleter', 'shareable-list-events'],
       },
     });
   }

infrastructure/transactional-emails/src/config/index.ts

@@ -44,7 +44,13 @@ export const config = {
       { name: 'PremiumPurchaseEvents' },
       { name: 'UserRegistrationEvents' },
       { name: 'ForgotPasswordEvents' },
-      { name: 'ListExportReadyEvents' },
+      {
+        name: 'ListExportReadyEvents',
+        filterPolicyScope: 'MessageBody',
+        filterPolicy: JSON.stringify({
+          'detail-type': ['list-export-ready'],
+        }),
+      },
       {
         name: 'ListEvents',
         filterPolicyScope: 'MessageBody',

packages/event-bridge/src/events/events.ts

@@ -11,6 +11,7 @@ export enum PocketEventType {
   ACCOUNT_PASSWORD_CHANGED = 'account-password-changed', //source: user-event
   FORGOT_PASSWORD = 'Forgot Password Request', //source: web-repo
   EXPORT_READY = 'list-export-ready', // source: account-data-deleter
+  EXPORT_PART_COMPLETE = 'export-part-complete', // source: account-data-deleter or shareable-lists-api
   EXPORT_REQUESTED = 'list-export-requested', // source: list-api
   // List Events
   ADD_ITEM = 'ADD_ITEM',

packages/event-bridge/src/events/generated/schema.json

@@ -2237,6 +2237,55 @@
       ],
       "type": "object"
     },
+    "ExportPartComplete": {
+      "properties": {
+        "detail": {
+          "properties": {
+            "encodedId": {
+              "type": "string"
+            },
+            "prefix": {
+              "type": "string"
+            },
+            "requestId": {
+              "type": "string"
+            },
+            "service": {
+              "enum": [
+                "list",
+                "shareable-lists",
+                "annotations"
+              ],
+              "type": "string"
+            },
+            "timestamp": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "encodedId",
+            "requestId",
+            "service",
+            "timestamp",
+            "prefix"
+          ],
+          "type": "object"
+        },
+        "detail-type": {
+          "const": "export-part-complete",
+          "type": "string"
+        },
+        "source": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "detail",
+        "detail-type",
+        "source"
+      ],
+      "type": "object"
+    },
     "ExportReady": {
       "properties": {
         "detail": {

packages/event-bridge/src/events/index.ts

@@ -49,6 +49,7 @@ import {
   CollectionUpdated,
   ProspectDismissed,
   IncomingBaseEvent,
+  ExportPartComplete,
 } from './types/index.ts';
 import { ProspectEvent } from './types/prospect.ts';
 export * from './types/index.ts';
@@ -59,6 +60,7 @@ export type PocketEvent =
   | AccountEvent
   | ExportReady
   | ExportRequested
+  | ExportPartComplete
   | PremiumPurchaseEvent
   | ListEvent
   | ShareableListEvent
@@ -80,6 +82,7 @@ export type PocketEventTypeMap = {
   [PocketEventType.PREMIUM_PURCHASE]: PremiumPurchaseEvent;
   [PocketEventType.EXPORT_READY]: ExportReady;
   [PocketEventType.EXPORT_REQUESTED]: ExportRequested;
+  [PocketEventType.EXPORT_PART_COMPLETE]: ExportPartComplete;
   [PocketEventType.ADD_ITEM]: AddItem;
   [PocketEventType.DELETE_ITEM]: DeleteItem;
   [PocketEventType.FAVORITE_ITEM]: FavoriteItem;

packages/event-bridge/src/events/types/account.ts

@@ -52,6 +52,18 @@ export interface ExportReady extends BaseEvent {
   };
 }
 
+export interface ExportPartComplete extends BaseEvent {
+  // 'source': 'account-data-deleter' | 'shareable-lists-api';
+  'detail-type': PocketEventType.EXPORT_PART_COMPLETE;
+  detail: {
+    encodedId: string;
+    requestId: string;
+    service: 'list' | 'shareable-lists' | 'annotations';
+    timestamp: string; // ISO-formatted string
+    prefix: string; // 
+  };
+}
+
 export interface AccountPayload {
   email: string;
   isPremium: boolean;