From 85af31e8e7cbe58d9bc83d59f727b20ad9d5e478 Mon Sep 17 00:00:00 2001 From: bilhokista <59991975+bilhokista@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:44:12 +0700 Subject: [PATCH 1/5] feat(notifications): dead-letter undeliverable emails with counters --- .../entities/dead-lettered-email.entity.ts | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 backend/src/notifications/entities/dead-lettered-email.entity.ts diff --git a/backend/src/notifications/entities/dead-lettered-email.entity.ts b/backend/src/notifications/entities/dead-lettered-email.entity.ts new file mode 100644 index 00000000..14c55f91 --- /dev/null +++ b/backend/src/notifications/entities/dead-lettered-email.entity.ts @@ -0,0 +1,81 @@ +import { + Column, + CreateDateColumn, + Entity, + Index, + PrimaryGeneratedColumn, +} from 'typeorm'; + +/** + * Why a message stopped being retried. + * + * The distinction matters operationally: PERMANENT means the provider rejected + * the message and resending it unchanged will fail again (a bad address, a + * revoked key), while RETRIES_EXHAUSTED means the send never got a verdict and + * is worth replaying once the provider is healthy. + */ +export enum DeadLetterReason { + PERMANENT = 'permanent', + RETRIES_EXHAUSTED = 'retries_exhausted', +} + +/** + * A notification email that will not be delivered. + * + * Before this table, `EmailService.processQueue` caught the delivery error, + * logged one line, and dropped the message — it had already been shifted off + * the in-memory queue, so nothing was left to inspect or replay. + * + * The rendered body is stored alongside the metadata so a message can be + * replayed exactly as it was composed, without re-running template rendering + * that may since have changed. + */ +@Entity('dead_lettered_emails') +@Index(['created_at']) +@Index(['reason', 'created_at']) +export class DeadLetteredEmail { + @PrimaryGeneratedColumn('uuid') + id: string; + + /** The queue message id, so log lines can be tied back to this row. */ + @Index() + @Column({ type: 'varchar', length: 255 }) + message_id: string; + + @Column({ type: 'varchar', length: 320 }) + recipient: string; + + @Column({ type: 'varchar', length: 500 }) + subject: string; + + @Column({ type: 'text' }) + body_html: string; + + @Column({ type: 'text' }) + body_text: string; + + /** Wallet address of the recipient when the message was addressed to a user. */ + @Column({ type: 'varchar', length: 255, nullable: true }) + user_address: string | null; + + @Column({ + type: 'enum', + enum: DeadLetterReason, + }) + reason: DeadLetterReason; + + /** Message of the last error, kept short enough to scan in a list view. */ + @Column({ type: 'varchar', length: 1000 }) + failure_message: string; + + /** How many delivery attempts were made before giving up. */ + @Column({ type: 'integer' }) + attempts: number; + + /** When the message first entered the send queue. */ + @Column({ type: 'timestamptz' }) + queued_at: Date; + + @CreateDateColumn({ type: 'timestamptz' }) + created_at: Date; +} From 7dd4777a13fb9f008f7214a20fddc275f0f9b36e Mon Sep 17 00:00:00 2001 From: bilhokista <59991975+bilhokista@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:44:14 +0700 Subject: [PATCH 2/5] feat(notifications): dead-letter undeliverable emails with counters --- .../1788100000000-CreateDeadLetteredEmails.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 backend/src/migrations/1788100000000-CreateDeadLetteredEmails.ts diff --git a/backend/src/migrations/1788100000000-CreateDeadLetteredEmails.ts b/backend/src/migrations/1788100000000-CreateDeadLetteredEmails.ts new file mode 100644 index 00000000..8b817dde --- /dev/null +++ b/backend/src/migrations/1788100000000-CreateDeadLetteredEmails.ts @@ -0,0 +1,52 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class CreateDeadLetteredEmails1788100000000 + implements MigrationInterface +{ + name = 'CreateDeadLetteredEmails1788100000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TYPE "public"."dead_letter_reason" AS ENUM('permanent', 'retries_exhausted') + `); + + await queryRunner.query(` + CREATE TABLE "dead_lettered_emails" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "message_id" varchar(255) NOT NULL, + "recipient" varchar(320) NOT NULL, + "subject" varchar(500) NOT NULL, + "body_html" text NOT NULL, + "body_text" text NOT NULL, + "user_address" varchar(255), + "reason" "public"."dead_letter_reason" NOT NULL, + "failure_message" varchar(1000) NOT NULL, + "attempts" integer NOT NULL, + "queued_at" timestamptz NOT NULL, + "created_at" timestamptz NOT NULL DEFAULT now(), + CONSTRAINT "PK_dead_lettered_emails" PRIMARY KEY ("id") + ) + `); + + await queryRunner.query(` + CREATE INDEX "IDX_dead_lettered_emails_message_id" ON "dead_lettered_emails" ("message_id") + `); + + await queryRunner.query(` + CREATE INDEX "IDX_dead_lettered_emails_created_at" ON "dead_lettered_emails" ("created_at") + `); + + // Supports the common triage query: "everything worth replaying, newest first". + await queryRunner.query(` + CREATE INDEX "IDX_dead_lettered_emails_reason_created_at" ON "dead_lettered_emails" ("reason", "created_at") + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_dead_lettered_emails_reason_created_at"`); + await queryRunner.query(`DROP INDEX "public"."IDX_dead_lettered_emails_created_at"`); + await queryRunner.query(`DROP INDEX "public"."IDX_dead_lettered_emails_message_id"`); + await queryRunner.query(`DROP TABLE "dead_lettered_emails"`); + await queryRunner.query(`DROP TYPE "public"."dead_letter_reason"`); + } +} From deec43d608a36e97d3232ed9b607b79d3679bb59 Mon Sep 17 00:00:00 2001 From: bilhokista <59991975+bilhokista@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:44:17 +0700 Subject: [PATCH 3/5] feat(notifications): dead-letter undeliverable emails with counters --- backend/src/notifications/email.service.ts | 120 +++++++++++++++++++-- 1 file changed, 114 insertions(+), 6 deletions(-) diff --git a/backend/src/notifications/email.service.ts b/backend/src/notifications/email.service.ts index decc9a5b..f655133f 100644 --- a/backend/src/notifications/email.service.ts +++ b/backend/src/notifications/email.service.ts @@ -13,6 +13,10 @@ import { NotificationCategoryPreference, NotificationCategory, } from './entities/notification-category-preference.entity'; +import { + DeadLetteredEmail, + DeadLetterReason, +} from './entities/dead-lettered-email.entity'; import { EmailTemplateContext, EmailTemplateType, @@ -30,6 +34,34 @@ export interface QueuedEmail { queuedAt: number; } +/** + * Raised by {@link EmailService.deliverEmailWithRetry} once it gives up, so + * the caller can dead-letter the message with the two facts it needs — how + * many attempts were spent, and whether the provider gave a verdict — instead + * of re-deriving them from the underlying error. + */ +export class EmailDeliveryFailure extends Error { + constructor( + readonly reason: DeadLetterReason, + readonly attempts: number, + readonly cause: unknown, + ) { + super(cause instanceof Error ? cause.message : String(cause)); + this.name = 'EmailDeliveryFailure'; + } +} + +/** Cumulative delivery counters since process start. */ +export interface EmailDeliveryCounters { + sent: number; + /** Individual retry attempts, not messages — one message can add several. */ + retried: number; + deadLettered: number; +} + +/** Column width of `failure_message`; longer messages are truncated to fit. */ +const MAX_FAILURE_MESSAGE_LENGTH = 1000; + const DEFAULT_RATE_LIMIT = 30; const QUEUE_PROCESS_INTERVAL_MS = 2000; @@ -157,6 +189,11 @@ export class EmailService implements OnModuleInit, OnModuleDestroy { private readonly sentTimestamps: number[] = []; private processTimer: ReturnType | null = null; private isProcessing = false; + private readonly counters: EmailDeliveryCounters = { + sent: 0, + retried: 0, + deadLettered: 0, + }; constructor( private readonly configService: ConfigService, @@ -166,8 +203,18 @@ export class EmailService implements OnModuleInit, OnModuleDestroy { private readonly preferencesRepository: Repository, @InjectRepository(NotificationCategoryPreference) private readonly categoryPreferencesRepository: Repository, + @InjectRepository(DeadLetteredEmail) + private readonly deadLetterRepository: Repository, ) {} + /** + * Snapshot of the delivery counters, for a metrics endpoint or a health + * probe. Returns a copy so a caller cannot mutate the running totals. + */ + getDeliveryCounters(): EmailDeliveryCounters { + return { ...this.counters }; + } + onModuleInit(): void { this.processTimer = setInterval(() => { void this.processQueue(); @@ -313,18 +360,26 @@ export class EmailService implements OnModuleInit, OnModuleDestroy { this.isProcessing = true; + let email: QueuedEmail | undefined; try { - const email = this.queue.shift(); + email = this.queue.shift(); if (!email) { return; } await this.deliverEmailWithRetry(email); this.sentTimestamps.push(Date.now()); + this.counters.sent += 1; } catch (error) { - this.logger.error( - `Failed to process email queue: ${error instanceof Error ? error.message : 'Unknown error'}`, - ); + // The message was already shifted off the queue, so unless it is + // persisted here it is gone for good. + if (email) { + await this.deadLetter(email, error); + } else { + this.logger.error( + `Failed to process email queue: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } } finally { this.isProcessing = false; } @@ -375,7 +430,11 @@ export class EmailService implements OnModuleInit, OnModuleDestroy { `Permanent email failure for message ${email.id} (attempt ${attempt + 1}/${maxAttempts}): ` + `${error instanceof Error ? error.message : String(error)}`, ); - throw error; + throw new EmailDeliveryFailure( + DeadLetterReason.PERMANENT, + attempt + 1, + error, + ); } const attemptsRemaining = maxAttempts - attempt - 1; @@ -384,6 +443,7 @@ export class EmailService implements OnModuleInit, OnModuleDestroy { break; // exhausted — log final error below } + this.counters.retried += 1; const delayMs = computeBackoffDelay(baseDelayMs, attempt); this.logger.warn( `Transient email failure for message ${email.id} — attempt ${attempt + 1}/${maxAttempts}, ` + @@ -399,7 +459,55 @@ export class EmailService implements OnModuleInit, OnModuleDestroy { `Email delivery failed after ${maxAttempts} attempt(s) for message ${email.id} — ` + `final error: ${lastError instanceof Error ? lastError.message : String(lastError)}`, ); - throw lastError; + throw new EmailDeliveryFailure( + DeadLetterReason.RETRIES_EXHAUSTED, + maxAttempts, + lastError, + ); + } + + /** + * Persist a message that will not be delivered, with the reason it failed. + * + * Never rethrows: a dead-letter write that fails must not take down queue + * processing for every message behind it. It is logged at `error` so the + * loss is still visible. + */ + private async deadLetter(email: QueuedEmail, error: unknown): Promise { + const failure = + error instanceof EmailDeliveryFailure + ? error + : new EmailDeliveryFailure(DeadLetterReason.PERMANENT, 1, error); + + this.counters.deadLettered += 1; + + try { + await this.deadLetterRepository.save( + this.deadLetterRepository.create({ + message_id: email.id, + recipient: email.to, + subject: email.subject, + body_html: email.html, + body_text: email.text, + user_address: email.userAddress ?? null, + reason: failure.reason, + failure_message: failure.message.slice(0, MAX_FAILURE_MESSAGE_LENGTH), + attempts: failure.attempts, + queued_at: new Date(email.queuedAt), + }), + ); + this.logger.error( + `Dead-lettered email ${email.id} to ${email.to} after ` + + `${failure.attempts} attempt(s) (${failure.reason}): ${failure.message} — ` + + `counters sent=${this.counters.sent} retried=${this.counters.retried} ` + + `deadLettered=${this.counters.deadLettered}`, + ); + } catch (persistError) { + this.logger.error( + `Failed to dead-letter email ${email.id}; the message is lost: ` + + `${persistError instanceof Error ? persistError.message : String(persistError)}`, + ); + } } private async deliverEmail(email: QueuedEmail): Promise { From f90dcc24c475c40a4db36b24c7ae62c52e92c94a Mon Sep 17 00:00:00 2001 From: bilhokista <59991975+bilhokista@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:44:20 +0700 Subject: [PATCH 4/5] feat(notifications): dead-letter undeliverable emails with counters --- .../src/notifications/email.service.spec.ts | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/backend/src/notifications/email.service.spec.ts b/backend/src/notifications/email.service.spec.ts index 999f9c99..3290936a 100644 --- a/backend/src/notifications/email.service.spec.ts +++ b/backend/src/notifications/email.service.spec.ts @@ -11,6 +11,10 @@ import { import { User } from '../users/entities/user.entity'; import { UserPreferences } from '../users/entities/user-preferences.entity'; import { NotificationCategoryPreference } from './entities/notification-category-preference.entity'; +import { + DeadLetteredEmail, + DeadLetterReason, +} from './entities/dead-lettered-email.entity'; // --------------------------------------------------------------------------- // Helpers @@ -156,6 +160,12 @@ describe('EmailService — deliverEmailWithRetry', () => { let preferencesRepository: jest.Mocked< Pick, 'findOne'> >; + /** `create` passes the object straight through, so `save` receives the row + * the service built and tests can assert on it directly. */ + let deadLetterRepository: { + create: jest.Mock; + save: jest.Mock; + }; let configGetMock: jest.Mock; beforeEach(async () => { @@ -163,6 +173,10 @@ describe('EmailService — deliverEmailWithRetry', () => { userRepository = { findOne: jest.fn() }; preferencesRepository = { findOne: jest.fn() }; + deadLetterRepository = { + create: jest.fn((row: unknown) => row), + save: jest.fn((row: unknown) => Promise.resolve(row)), + }; // Default config: 3 attempts, 1000 ms base delay configGetMock = jest.fn((key: string) => { @@ -189,6 +203,10 @@ describe('EmailService — deliverEmailWithRetry', () => { provide: getRepositoryToken(NotificationCategoryPreference), useValue: { findOne: jest.fn() }, }, + { + provide: getRepositoryToken(DeadLetteredEmail), + useValue: deadLetterRepository, + }, ], }).compile(); @@ -351,10 +369,20 @@ describe('EmailService — queueing and preferences', () => { let preferencesRepository: jest.Mocked< Pick, 'findOne'> >; + /** `create` passes the object straight through, so `save` receives the row + * the service built and tests can assert on it directly. */ + let deadLetterRepository: { + create: jest.Mock; + save: jest.Mock; + }; beforeEach(async () => { userRepository = { findOne: jest.fn() }; preferencesRepository = { findOne: jest.fn() }; + deadLetterRepository = { + create: jest.fn((row: unknown) => row), + save: jest.fn((row: unknown) => Promise.resolve(row)), + }; const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -374,6 +402,10 @@ describe('EmailService — queueing and preferences', () => { provide: getRepositoryToken(NotificationCategoryPreference), useValue: { findOne: jest.fn() }, }, + { + provide: getRepositoryToken(DeadLetteredEmail), + useValue: deadLetterRepository, + }, ], }).compile(); @@ -451,3 +483,170 @@ describe('EmailService — queueing and preferences', () => { ).rejects.toThrow(/Missing required email template variables/); }); }); + +// --------------------------------------------------------------------------- +// Dead-letter queue: what happens to a message the provider will not accept +// --------------------------------------------------------------------------- + +describe('EmailService — dead-letter queue', () => { + let service: EmailService; + let deadLetterRepository: { create: jest.Mock; save: jest.Mock }; + + /** Runs the queue timer far enough to cover three attempts and both backoffs. */ + const drainQueue = () => jest.advanceTimersByTimeAsync(30_000); + + const spyOnDeliver = () => + jest.spyOn( + service as unknown as { + deliverEmail: (e: QueuedEmail) => Promise; + }, + 'deliverEmail', + ); + + beforeEach(async () => { + jest.useFakeTimers(); + + deadLetterRepository = { + create: jest.fn((row: unknown) => row), + save: jest.fn((row: unknown) => Promise.resolve(row)), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + EmailService, + { + provide: ConfigService, + useValue: { + get: jest.fn((key: string) => { + if (key === 'EMAIL_RETRY_MAX_ATTEMPTS') return '3'; + if (key === 'EMAIL_RETRY_BASE_DELAY_MS') return '1000'; + if (key === 'SENDGRID_API_KEY') return 'test-api-key'; + return undefined; + }), + }, + }, + { provide: getRepositoryToken(User), useValue: { findOne: jest.fn() } }, + { + provide: getRepositoryToken(UserPreferences), + useValue: { findOne: jest.fn() }, + }, + { + provide: getRepositoryToken(NotificationCategoryPreference), + useValue: { findOne: jest.fn() }, + }, + { + provide: getRepositoryToken(DeadLetteredEmail), + useValue: deadLetterRepository, + }, + ], + }).compile(); + + service = module.get(EmailService); + service.onModuleInit(); + }); + + afterEach(() => { + service.onModuleDestroy(); + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + async function queueOne(): Promise { + await service.queueEmail({ + to: 'recipient@example.com', + subject: 'Test Subject', + html: '

Hello

', + text: 'Hello', + }); + } + + it('retries a transient failure and does not dead-letter once it succeeds', async () => { + const deliver = spyOnDeliver() + .mockRejectedValueOnce(new Error('SendGrid error (503): Service Unavailable')) + .mockResolvedValueOnce(undefined); + + await queueOne(); + await drainQueue(); + + expect(deliver).toHaveBeenCalledTimes(2); + expect(deadLetterRepository.save).not.toHaveBeenCalled(); + expect(service.getDeliveryCounters()).toEqual({ + sent: 1, + retried: 1, + deadLettered: 0, + }); + }); + + it('dead-letters a permanently rejected message without retrying it', async () => { + const deliver = spyOnDeliver().mockRejectedValue( + new Error('SendGrid error (400): Bad Request'), + ); + + await queueOne(); + await drainQueue(); + + // 4xx is a verdict, not a hiccup — one attempt only. + expect(deliver).toHaveBeenCalledTimes(1); + expect(deadLetterRepository.save).toHaveBeenCalledTimes(1); + + const row = deadLetterRepository.save.mock.calls[0][0]; + expect(row).toEqual( + expect.objectContaining({ + recipient: 'recipient@example.com', + subject: 'Test Subject', + body_text: 'Hello', + reason: DeadLetterReason.PERMANENT, + attempts: 1, + }), + ); + expect(row.failure_message).toContain('400'); + expect(service.getDeliveryCounters()).toEqual({ + sent: 0, + retried: 0, + deadLettered: 1, + }); + }); + + it('dead-letters as retries_exhausted once the attempt budget runs out', async () => { + const deliver = spyOnDeliver().mockRejectedValue( + new Error('SendGrid error (503): Service Unavailable'), + ); + + await queueOne(); + await drainQueue(); + + expect(deliver).toHaveBeenCalledTimes(3); + expect(deadLetterRepository.save).toHaveBeenCalledTimes(1); + expect(deadLetterRepository.save.mock.calls[0][0]).toEqual( + expect.objectContaining({ + reason: DeadLetterReason.RETRIES_EXHAUSTED, + attempts: 3, + }), + ); + // Two retries for three attempts. + expect(service.getDeliveryCounters()).toEqual({ + sent: 0, + retried: 2, + deadLettered: 1, + }); + }); + + it('keeps processing when the dead-letter write itself fails', async () => { + spyOnDeliver().mockRejectedValue( + new Error('SendGrid error (400): Bad Request'), + ); + deadLetterRepository.save.mockRejectedValue(new Error('db unavailable')); + + await queueOne(); + + // The failed write must not escape processQueue and kill the timer. + await expect(drainQueue()).resolves.toBeUndefined(); + expect(service.getQueueLength()).toBe(0); + }); + + it('hands out a copy of the counters, not the live object', () => { + const counters = service.getDeliveryCounters(); + counters.sent = 999; + expect(service.getDeliveryCounters().sent).toBe(0); + }); +}); From b499f60ad490da890d40370453f03b41afdad2d0 Mon Sep 17 00:00:00 2001 From: bilhokista <59991975+bilhokista@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:44:22 +0700 Subject: [PATCH 5/5] feat(notifications): dead-letter undeliverable emails with counters --- backend/src/notifications/notifications.module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/notifications/notifications.module.ts b/backend/src/notifications/notifications.module.ts index 6042a228..41c2f9c0 100644 --- a/backend/src/notifications/notifications.module.ts +++ b/backend/src/notifications/notifications.module.ts @@ -4,6 +4,7 @@ import { Notification } from './entities/notification.entity'; import { NotificationDigestState } from './entities/notification-digest-state.entity'; import { NotificationPreference } from './entities/notification-preference.entity'; import { NotificationCategoryPreference } from './entities/notification-category-preference.entity'; +import { DeadLetteredEmail } from './entities/dead-lettered-email.entity'; import { NotificationsService } from './notifications.service'; import { NotificationsController } from './notifications.controller'; import { EmailService } from './email.service'; @@ -24,6 +25,7 @@ import { WebsocketModule } from '../websocket/websocket.module'; NotificationDigestState, NotificationPreference, NotificationCategoryPreference, + DeadLetteredEmail, User, UserPreferences, CreatorEvent,