fix: improve member deduplication by parsing GitHub noreply emails (CM-958) (#3929)

Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>

Author: skwowet

services/apps/data_sink_worker/src/service/activity.service.ts

@@ -10,6 +10,7 @@ import {
   escapeNullByte,
   generateUUIDv1,
   isValidEmail,
+  parseGitHubNoreplyEmail,
   single,
   singleOrDefault,
   trimUtf8ToMaxByteLength,
@@ -805,6 +806,92 @@ export default class ActivityService extends LoggerBase {
         )
       }
 
+      // Look up members by parsing noreply emails to extract platform usernames
+      // e.g. "123+john@users.noreply.github.com" -> GitHub username "john"
+      const noreplyEmailFilterMap = new Map<
+        string,
+        { platform: PlatformType; username: string; segmentId: string }
+      >()
+
+      for (const payload of payloadsNotInDb.filter((p) => !p.dbMember)) {
+        for (const identity of payload.activity.member.identities.filter(
+          (i) => i.verified && i.type === MemberIdentityType.EMAIL,
+        )) {
+          const ghUsername = parseGitHubNoreplyEmail(identity.value)
+          if (ghUsername) {
+            const key = `${PlatformType.GITHUB}:${ghUsername}:${payload.segmentId}`
+            if (!noreplyEmailFilterMap.has(key)) {
+              noreplyEmailFilterMap.set(key, {
+                platform: PlatformType.GITHUB,
+                username: ghUsername,
+                segmentId: payload.segmentId,
+              })
+            }
+          }
+        }
+      }
+
+      for (const payload of payloadsNotInDb.filter(
+        (p) => !p.dbObjectMember && p.activity.objectMember,
+      )) {
+        for (const identity of payload.activity.objectMember.identities.filter(
+          (i) => i.verified && i.type === MemberIdentityType.EMAIL,
+        )) {
+          const ghUsername = parseGitHubNoreplyEmail(identity.value)
+          if (ghUsername) {
+            const key = `${PlatformType.GITHUB}:${ghUsername}:${payload.segmentId}`
+            if (!noreplyEmailFilterMap.has(key)) {
+              noreplyEmailFilterMap.set(key, {
+                platform: PlatformType.GITHUB,
+                username: ghUsername,
+                segmentId: payload.segmentId,
+              })
+            }
+          }
+        }
+      }
+
+      if (noreplyEmailFilterMap.size > 0) {
+        const dbMembersByNoreplyEmail = await logExecutionTimeV2(
+          async () =>
+            findMembersByVerifiedUsernames(this.pgQx, Array.from(noreplyEmailFilterMap.values())),
+          this.log,
+          'processActivities -> memberRepo.findMembersByVerifiedUsernames (noreply-email)',
+        )
+
+        mapResultsToPayloads(
+          dbMembersByNoreplyEmail,
+          (p, value) =>
+            !p.dbMember &&
+            p.activity.member.identities.some(
+              (i) =>
+                i.verified &&
+                i.type === MemberIdentityType.EMAIL &&
+                parseGitHubNoreplyEmail(i.value) === value.toLowerCase(),
+            ),
+          (p, member) => {
+            p.dbMember = member
+            p.dbMemberSource = 'email'
+          },
+        )
+
+        mapResultsToPayloads(
+          dbMembersByNoreplyEmail,
+          (p, value) =>
+            !p.dbObjectMember &&
+            p.activity.objectMember?.identities.some(
+              (i) =>
+                i.verified &&
+                i.type === MemberIdentityType.EMAIL &&
+                parseGitHubNoreplyEmail(i.value) === value.toLowerCase(),
+            ),
+          (p, member) => {
+            p.dbObjectMember = member
+            p.dbObjectMemberSource = 'email'
+          },
+        )
+      }
+
       // Look up members using cross-identity matching (different platforms)
       // we will check only on platforms that store email identities as usernames
 

services/apps/merge_suggestions_worker/src/activities/memberMergeSuggestions.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import uniqBy from 'lodash.uniqby'
 
+import { parseGitHubNoreplyEmail } from '@crowd/common'
 import { addMemberNoMerge } from '@crowd/data-access-layer/src/member_merge'
 import { MemberField, queryMembers } from '@crowd/data-access-layer/src/members'
 import MemberMergeSuggestionsRepository from '@crowd/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo'
@@ -14,6 +15,7 @@ import {
   MemberIdentityType,
   MemberMergeSuggestionTable,
   OpenSearchIndex,
+  PlatformType,
 } from '@crowd/types'
 
 import { EMAIL_AS_USERNAME_PLATFORMS } from '../enums'
@@ -77,6 +79,10 @@ export async function getMemberMergeSuggestions(
   const unverifiedEmailUsernameMatches = []
   const unverifiedUsernameEmailMatches = []
 
+  // Noreply email -> platform username matches
+  // e.g. "123+john@users.noreply.github.com" -> GitHub username "john"
+  const noreplyEmailUsernameMatches = []
+
   // Process up to 75 identities
   // This is a safety limit to prevent OpenSearch max clause errors
   for (const { verified, value, platform, type } of identities.slice(0, 75)) {
@@ -107,6 +113,14 @@ export async function getMemberMergeSuggestions(
       targetLists.usernameEmail.push({ value })
     }
 
+    // Noreply email -> platform username extraction
+    if (isEmail && verified) {
+      const ghUsername = parseGitHubNoreplyEmail(value)
+      if (ghUsername) {
+        noreplyEmailUsernameMatches.push({ value: ghUsername, platform: PlatformType.GITHUB })
+      }
+    }
+
     // Fuzzy matches (only for verified & non-numeric)
     if (verified && !isNumeric(value)) {
       targetLists.fuzzy.push({ value: stripProtocol(value) })
@@ -199,7 +213,20 @@ export async function getMemberMergeSuggestions(
       }),
     },
     {
-      // Query 4: Verified -> Verified fuzzy matches
+      // Query 8: Noreply/private email -> username (verified or unverified)
+      matches: uniqBy(noreplyEmailUsernameMatches, 'value'),
+      builder: ({ value, platform }) => ({
+        bool: {
+          must: [
+            { term: { [`nested_identities.keyword_value`]: value } },
+            { match: { [`nested_identities.string_platform`]: platform } },
+            { term: { [`nested_identities.keyword_type`]: MemberIdentityType.USERNAME } },
+          ],
+        },
+      }),
+    },
+    {
+      // Query 9: Verified -> Verified fuzzy matches
       matches: uniqBy(verifiedFuzzyMatches, 'value'),
       builder: ({ value }) => ({
         match: {

services/apps/merge_suggestions_worker/src/memberSimilarityCalculator.ts

@@ -1,5 +1,6 @@
 import { get as getLevenshteinDistance } from 'fast-levenshtein'
 
+import { parseGitHubNoreplyEmail } from '@crowd/common'
 import {
   IMemberIdentity,
   IMemberOpensearch,
@@ -105,6 +106,11 @@ class MemberSimilarityCalculator {
       }
     }
 
+    // Check if a noreply email on one member resolves to a username on the other
+    if (this.hasMatchingUsernameFromNoreplyEmail(primaryMember, similarMember)) {
+      return 0.95
+    }
+
     for (const primaryIdentity of primaryMember.identities.filter((i) => i.verified)) {
       // similar member has an unverified identity as one of primary members's verified identity, return score 95
       if (
@@ -187,7 +193,9 @@ class MemberSimilarityCalculator {
     similarMember: IMemberOpensearch,
   ): boolean {
     if (member.identities && member.identities.length > 0) {
-      for (const identity of member.identities) {
+      for (const identity of member.identities.filter(
+        (i) => i.type === MemberIdentityType.USERNAME,
+      )) {
         if (
           similarMember.nested_identities.some(
             (i) =>
@@ -204,6 +212,59 @@ class MemberSimilarityCalculator {
     return false
   }
 
+  /**
+   * Checks if a noreply email in one member matches a username in the other (e.g. GitHub noreply email -> GitHub username)
+   * Works bidirectionally: primary email -> similar username, and similar email -> primary username.
+   */
+  static hasMatchingUsernameFromNoreplyEmail(
+    primaryMember: IMemberWithAggregatesForMergeSuggestions,
+    similarMember: IMemberOpensearch,
+  ): boolean {
+    // Primary member's noreply emails -> similar member's platform usernames
+    const similarUsernamesByPlatform = {
+      [PlatformType.GITHUB]: new Set(
+        similarMember.nested_identities
+          .filter(
+            (i) =>
+              i.string_platform === PlatformType.GITHUB &&
+              i.keyword_type === MemberIdentityType.USERNAME,
+          )
+          .map((i) => i.string_value?.toLowerCase()),
+      ),
+    }
+
+    for (const identity of primaryMember.identities) {
+      if (!identity.verified || identity.type !== MemberIdentityType.EMAIL) continue
+
+      const ghUsername = parseGitHubNoreplyEmail(identity.value)
+      if (ghUsername && similarUsernamesByPlatform[PlatformType.GITHUB].has(ghUsername)) {
+        return true
+      }
+    }
+
+    // Similar member's noreply emails -> primary member's platform usernames
+    const primaryUsernamesByPlatform = {
+      [PlatformType.GITHUB]: new Set(
+        primaryMember.identities
+          .filter(
+            (i) => i.platform === PlatformType.GITHUB && i.type === MemberIdentityType.USERNAME,
+          )
+          .map((i) => i.value?.toLowerCase()),
+      ),
+    }
+
+    for (const identity of similarMember.nested_identities) {
+      if (!identity.bool_verified || identity.keyword_type !== MemberIdentityType.EMAIL) continue
+
+      const ghUsername = parseGitHubNoreplyEmail(identity.string_value)
+      if (ghUsername && primaryUsernamesByPlatform[PlatformType.GITHUB].has(ghUsername)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
   static hasEmailAsUsernameIdentityMatch(
     primaryIdentity: IMemberIdentity,
     similarMember: IMemberOpensearch,

services/libs/common/src/email.ts

@@ -3,3 +3,24 @@ import validator from 'validator'
 export const isValidEmail = (value: string): boolean => {
   return validator.isEmail(value)
 }
+
+const GITHUB_NOREPLY_EMAIL_SUFFIX = '@users.noreply.github.com'
+
+/**
+ * Extracts username from a GitHub noreply email.
+ * @see https://docs.github.com/en/account-and-profile/reference/email-addresses-reference#your-noreply-email-address
+ */
+export const parseGitHubNoreplyEmail = (email?: string | null): string | null => {
+  if (!email) return null
+
+  const lower = email.toLowerCase()
+  if (!lower.endsWith(GITHUB_NOREPLY_EMAIL_SUFFIX)) return null
+
+  const local = lower.slice(0, -GITHUB_NOREPLY_EMAIL_SUFFIX.length)
+  if (!local) return null
+
+  const plusIndex = local.indexOf('+')
+  const username = plusIndex >= 0 ? local.slice(plusIndex + 1) : local
+
+  return username || null
+}