Bulk upload mailchimp users (#450)

Author: annarhughes

README.md

@@ -42,6 +42,7 @@ For a more detailed explanation of this project's key concepts and architecture,
 - [Slack](https://api.slack.com/messaging/webhooks) - Slack webhooks to send messages to the team
 - [Rollbar](https://rollbar.com/) - Error reporting
 - [Crisp](https://crisp.chat/en/) - User messaging
+- [Mailchimp](https://mailchimp.com/developer/marketing/) - Transactional email
 - [Docker](https://www.docker.com/) - Containers for api and db
 - [Heroku](https://heroku.com) - Build, deploy and operate staging and production apps
 - [GitHub Actions](https://github.com/features/actions) - CI pipeline
@@ -58,14 +59,15 @@ For a more detailed explanation of this project's key concepts and architecture,
 
 **Recommended for Visual Studio & Visual Studio Code users.**
 
-This method will automatically install all dependencies and IDE settings in a Dev Container (Docker container) within Visual Studio Code. 
+This method will automatically install all dependencies and IDE settings in a Dev Container (Docker container) within Visual Studio Code.
 
 Directions for running a dev container:
+
 1. Meet the [system requirements](https://code.visualstudio.com/docs/devcontainers/containers#_system-requirements)
 2. Follow the [installation instructions](https://code.visualstudio.com/docs/devcontainers/containers#_installation)
 3. [Check the installation](https://code.visualstudio.com/docs/devcontainers/tutorial#_check-installation)
 4. After you've verified that the extension is installed and working, click on the "Remote Status" bar icon and select
-"Reopen in Container". From here, the option to "Re-open in Container" should pop up in notifications whenever opening this project in VS.
+   "Reopen in Container". From here, the option to "Re-open in Container" should pop up in notifications whenever opening this project in VS.
 5. [Configure your environment variables](#configure-environment-variables) and develop as you normally would.
 
 The dev Container is configured in the `.devcontainer` directory:
@@ -84,8 +86,8 @@ yarn
 ### Configure Environment Variables
 
 Create a new `.env` file and populate it with the variables below. Note that only the Firebase and Simplybook tokens are required.
-To configure the Firebase variables, first [create a Firebase project in the Firebase console](https://firebase.google.com/) (Google account required). 
-Next, follow [these directions](https://firebase.google.com/docs/cloud-messaging/auth-server#provide-credentials-manually) to generate a private key file in JSON format. 
+To configure the Firebase variables, first [create a Firebase project in the Firebase console](https://firebase.google.com/) (Google account required).
+Next, follow [these directions](https://firebase.google.com/docs/cloud-messaging/auth-server#provide-credentials-manually) to generate a private key file in JSON format.
 These will generate all the required Firebase variables.
 
 The Simplybook variables can be mocked data, meaning **you do not need to use real Simplybook variables, simply copy paste the values given below.**

src/api/crisp/crisp-api.interfaces.ts

@@ -10,8 +10,8 @@ export interface CrispProfileCustomFields {
   therapy_sessions_redeemed?: number;
   course_hst?: string;
   course_hst_sessions?: string;
-  course_pst?: string;
-  course_pst_sessions?: string;
+  course_spst?: string;
+  course_spst_sessions?: string;
   course_dbr?: string;
   course_dbr_sessions?: string;
   course_iaro?: string;

src/api/mailchimp/mailchimp-api.ts

@@ -1,6 +1,8 @@
 import mailchimp from '@mailchimp/mailchimp_marketing';
 import { createHash } from 'crypto';
+import { UserEntity } from 'src/entities/user.entity';
 import { mailchimpApiKey, mailchimpAudienceId, mailchimpServerPrefix } from 'src/utils/constants';
+import { createCompleteMailchimpUserProfile } from 'src/utils/serviceUserProfiles';
 import {
   ListMember,
   ListMemberPartial,
@@ -32,6 +34,36 @@ export const createMailchimpProfile = async (
   }
 };
 
+export const batchCreateMailchimpProfiles = async (users: UserEntity[]) => {
+  try {
+    const operations = [];
+
+    users.forEach((user) => {
+      const profileData = createCompleteMailchimpUserProfile(user);
+      operations.push({
+        method: 'POST',
+        path: `/lists/${mailchimpAudienceId}/members`,
+        operation_id: user.id,
+        body: JSON.stringify(profileData),
+      });
+    });
+
+    const batchRequest = await mailchimp.batches.start({
+      operations: operations,
+    });
+    console.log('Mailchimp batch request:', batchRequest);
+    console.log('Wait 2 minutes before calling response...');
+
+    setTimeout(async () => {
+      const batchResponse = await mailchimp.batches.status(batchRequest.id);
+      console.log('Mailchimp batch response:', batchResponse);
+    }, 120000);
+  } catch (error) {
+    console.log(error);
+    throw new Error(`Batch create mailchimp profiles API call failed: ${error}`);
+  }
+};
+
 // Note getMailchimpProfile is not currently used
 export const getMailchimpProfile = async (email: string): Promise<ListMember> => {
   try {

src/user/user.controller.ts

@@ -109,4 +109,12 @@ export class UserController {
       : { include: [], fields: [], limit: undefined };
     return await this.userService.getUsers(userQuery, include, fields, limit);
   }
+
+  // Use only if users have not been added to mailchimp due to e.g. an ongoing bug
+  @ApiBearerAuth()
+  @Post('/bulk-mailchimp-upload')
+  @UseGuards(FirebaseAuthGuard)
+  async bulkUploadMailchimpProfiles() {
+    return await this.userService.bulkUploadMailchimpProfiles();
+  }
 }

src/user/user.service.ts

@@ -1,5 +1,6 @@
 import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
+import { batchCreateMailchimpProfiles } from 'src/api/mailchimp/mailchimp-api';
 import { PartnerAccessEntity } from 'src/entities/partner-access.entity';
 import { PartnerEntity } from 'src/entities/partner.entity';
 import { UserEntity } from 'src/entities/user.entity';
@@ -12,7 +13,7 @@ import {
   createServiceUserProfiles,
   updateServiceUserProfilesUser,
 } from 'src/utils/serviceUserProfiles';
-import { ILike, Repository } from 'typeorm';
+import { And, ILike, Raw, Repository } from 'typeorm';
 import { deleteCypressCrispProfiles } from '../api/crisp/crisp-api';
 import { AuthService } from '../auth/auth.service';
 import { PartnerAccessService, basePartnerAccess } from '../partner-access/partner-access.service';
@@ -298,4 +299,35 @@ export class UserService {
     const usersDto = users.map((user) => formatGetUsersObject(user));
     return usersDto;
   }
+
+  // Static bulk upload function to be used in specific cases
+  // UPDATE THE FILTERS to the current requirements
+  public async bulkUploadMailchimpProfiles() {
+    try {
+      const filterStartDate = '2023-01-01'; // UPDATE
+      const filterEndDate = '2024-01-01'; // UPDATE
+      const users = await this.userRepository.find({
+        where: {
+          // UPDATE TO ANY FILTERS
+          createdAt: And(
+            Raw((alias) => `${alias} >= :filterStartDate`, { filterStartDate: filterStartDate }),
+            Raw((alias) => `${alias} < :filterEndDate`, { filterEndDate: filterEndDate }),
+          ),
+        },
+        relations: {
+          partnerAccess: { partner: true, therapySession: true },
+          courseUser: { course: true, sessionUser: { session: true } },
+        },
+      });
+      const usersWithCourseUsers = users.filter((user) => user.courseUser.length > 0);
+
+      console.log(usersWithCourseUsers);
+      await batchCreateMailchimpProfiles(usersWithCourseUsers);
+      this.logger.log(
+        `Created batch mailchimp profiles for ${usersWithCourseUsers.length} users, created before ${filterStartDate}`,
+      );
+    } catch (error) {
+      throw new Error(`Bulk upload mailchimp profiles API call failed: ${error}`);
+    }
+  }
 }

src/utils/serviceUserProfiles.spec.ts

@@ -300,12 +300,7 @@ describe('Service user profiles', () => {
         },
       ];
 
-      await updateServiceUserProfilesTherapy(
-        partnerAccesses,
-        SIMPLYBOOK_ACTION_ENUM.NEW_BOOKING,
-        therapySession.startDateTime,
-        mockUserEntity.email,
-      );
+      await updateServiceUserProfilesTherapy(partnerAccesses, mockUserEntity.email);
 
       const firstTherapySessionAt = therapySession.startDateTime.toISOString();
       const nextTherapySessionAt = therapySession.startDateTime.toISOString();
@@ -339,12 +334,7 @@ describe('Service user profiles', () => {
     it('should update crisp and mailchimp profile combined therapy data for new booking', async () => {
       const partnerAccesses = [mockPartnerAccessEntity, mockAltPartnerAccessEntity];
 
-      await updateServiceUserProfilesTherapy(
-        partnerAccesses,
-        SIMPLYBOOK_ACTION_ENUM.NEW_BOOKING,
-        mockAltPartnerAccessEntity.therapySession[1].startDateTime,
-        mockUserEntity.email,
-      );
+      await updateServiceUserProfilesTherapy(partnerAccesses, mockUserEntity.email);
 
       const firstTherapySessionAt =
         mockPartnerAccessEntity.therapySession[0].startDateTime.toISOString();
@@ -381,12 +371,7 @@ describe('Service user profiles', () => {
     it('should update crisp and mailchimp profile combined therapy data for updated booking', async () => {
       const partnerAccesses = [mockPartnerAccessEntity, mockAltPartnerAccessEntity];
 
-      await updateServiceUserProfilesTherapy(
-        partnerAccesses,
-        SIMPLYBOOK_ACTION_ENUM.UPDATED_BOOKING,
-        mockAltPartnerAccessEntity.therapySession[1].startDateTime,
-        mockUserEntity.email,
-      );
+      await updateServiceUserProfilesTherapy(partnerAccesses, mockUserEntity.email);
 
       const firstTherapySessionAt =
         mockPartnerAccessEntity.therapySession[0].startDateTime.toISOString();
@@ -425,12 +410,7 @@ describe('Service user profiles', () => {
         SIMPLYBOOK_ACTION_ENUM.CANCELLED_BOOKING;
       const partnerAccesses = [mockPartnerAccessEntity, mockAltPartnerAccessEntity];
 
-      await updateServiceUserProfilesTherapy(
-        partnerAccesses,
-        SIMPLYBOOK_ACTION_ENUM.CANCELLED_BOOKING,
-        mockAltPartnerAccessEntity.therapySession[1].startDateTime,
-        mockUserEntity.email,
-      );
+      await updateServiceUserProfilesTherapy(partnerAccesses, mockUserEntity.email);
 
       const firstTherapySessionAt =
         mockPartnerAccessEntity.therapySession[0].startDateTime.toISOString();

src/utils/serviceUserProfiles.ts

@@ -126,16 +126,10 @@ export const updateServiceUserProfilesPartnerAccess = async (
 
 export const updateServiceUserProfilesTherapy = async (
   partnerAccesses: PartnerAccessEntity[],
-  therapySessionAction: SIMPLYBOOK_ACTION_ENUM,
-  therapySessionDate: Date,
   email,
 ) => {
   try {
-    const therapyData = serializeTherapyData(
-      partnerAccesses,
-      therapySessionAction,
-      therapySessionDate,
-    );
+    const therapyData = serializeTherapyData(partnerAccesses);
     await updateCrispProfile(therapyData.crispSchema, email);
     await updateMailchimpProfile(therapyData.mailchimpSchema, email);
   } catch (error) {
@@ -181,6 +175,36 @@ export const createMailchimpCourseMergeField = async (courseName: string) => {
   }
 };
 
+// Currently only used in bulk upload function, as mailchimp profiles are typically built
+// incrementally on sign up and subsequent user actions
+export const createCompleteMailchimpUserProfile = (user: UserEntity): ListMemberPartial => {
+  const userData = serializeUserData(user);
+  const partnerData = serializePartnerAccessData(user.partnerAccess);
+  const therapyData = serializeTherapyData(user.partnerAccess);
+
+  const courseData = {};
+  user.courseUser.forEach((courseUser) => {
+    const courseUserData = serializeCourseData(courseUser);
+    Object.keys(courseUserData.mailchimpSchema.merge_fields).forEach((key) => {
+      courseData[key] = courseUserData.mailchimpSchema.merge_fields[key];
+    });
+  });
+
+  const profileData = {
+    email_address: user.email,
+    ...userData.mailchimpSchema,
+
+    merge_fields: {
+      SIGNUPD: user.createdAt?.toISOString(),
+      ...userData.mailchimpSchema.merge_fields,
+      ...partnerData.mailchimpSchema.merge_fields,
+      ...therapyData.mailchimpSchema.merge_fields,
+      ...courseData,
+    },
+  };
+  return profileData;
+};
+
 export const serializePartnersString = (partnerAccesses: PartnerAccessEntity[]) => {
   return partnerAccesses?.map((pa) => pa.partner.name.toLowerCase()).join('; ') || '';
 };
@@ -203,7 +227,7 @@ const serializeUserData = (user: UserEntity) => {
         enabled: contactPermission,
       },
     ],
-    language: signUpLanguage,
+    language: signUpLanguage || 'en',
     merge_fields: { NAME: name },
   } as ListMemberPartial;
 
@@ -254,20 +278,14 @@ const serializePartnerAccessData = (partnerAccesses: PartnerAccessEntity[]) => {
   return { crispSchema, mailchimpSchema };
 };
 
-const serializeTherapyData = (
-  partnerAccesses: PartnerAccessEntity[],
-  therapySessionAction: SIMPLYBOOK_ACTION_ENUM,
-  therapySessionDate: Date,
-) => {
+const serializeTherapyData = (partnerAccesses: PartnerAccessEntity[]) => {
   const therapySessions = partnerAccesses
     .flatMap((partnerAccess) => partnerAccess.therapySession)
     .filter((therapySession) => therapySession.action !== SIMPLYBOOK_ACTION_ENUM.CANCELLED_BOOKING)
     .sort((a, b) => a.startDateTime.getTime() - b.startDateTime.getTime());
 
   const pastTherapySessions = therapySessions.filter(
-    (therapySession) =>
-      therapySession.startDateTime !== therapySessionDate &&
-      therapySession.startDateTime.getTime() < new Date().getTime(),
+    (therapySession) => therapySession.startDateTime.getTime() < new Date().getTime(),
   );
   const futureTherapySessions = therapySessions.filter(
     (therapySession) => therapySession.startDateTime.getTime() > new Date().getTime(),

src/webhooks/webhooks.service.ts

@@ -128,12 +128,7 @@ export class WebhooksService {
         },
       });
 
-      updateServiceUserProfilesTherapy(
-        [...partnerAccesses],
-        action,
-        therapySession.startDateTime,
-        user.email,
-      );
+      updateServiceUserProfilesTherapy([...partnerAccesses], user.email);
 
       this.logger.log(
         `Update therapy session webhook function COMPLETED for ${action} - ${user.email} - ${booking_code} - userId ${user_id}`,
@@ -249,12 +244,7 @@ export class WebhooksService {
       await this.partnerAccessRepository.save(partnerAccess);
       const therapySession = await this.therapySessionRepository.save(serializedTherapySession);
 
-      updateServiceUserProfilesTherapy(
-        [...partnerAccesses, partnerAccess],
-        SIMPLYBOOK_ACTION_ENUM.NEW_BOOKING,
-        therapySession.startDateTime,
-        user.email,
-      );
+      updateServiceUserProfilesTherapy([...partnerAccesses, partnerAccess], user.email);
 
       return therapySession;
     } catch (err) {