Merge pull request #2276 from CapSoftware/codex/loops-live-safety-alert

fix: harden Loops delivery monitoring and cutover sync

Author: richiemcilroy

.github/workflows/loops-safety.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
         with:
           bun-version: "1.4.0"
-      - name: Hold lifecycle delivery if synchronization is unhealthy
+      - name: Check lifecycle delivery and report required manual pauses
         env:
           LOOPS_API_KEY: ${{ secrets.LOOPS_API_KEY }}
           LOOPS_HEALTH_SECRET: ${{ secrets.LOOPS_HEALTH_SECRET }}

packages/database/loops/lifecycle.ts

@@ -22,6 +22,19 @@ export type LoopsRuntimeConfig = {
 	teammateJoinedAt?: string | null;
 };
 
+export function enrollmentWindow(signupAt: string, config: LoopsRuntimeConfig) {
+	return {
+		recentSignup:
+			config.enrollmentEnabled &&
+			Date.parse(signupAt) >= config.enrollmentAfter.getTime(),
+		recentJoin:
+			config.enrollmentEnabled &&
+			Boolean(config.teammateJoinedAt) &&
+			Date.parse(isoDate(config.teammateJoinedAt ?? "")) >=
+				config.enrollmentAfter.getTime(),
+	};
+}
+
 export function lifecycleUpdate(
 	source: LoopsProfileSource,
 	remote: LifecycleContact,
@@ -42,20 +55,15 @@ export function lifecycleUpdate(
 		!["unsubscribed", "suppressed"].includes(remote.capConsent ?? "");
 	const subscribed =
 		globallySubscribed && remote.mailingLists[config.listId] === true;
-	const recentSignup =
-		Date.parse(profile.capSignupAt) >= config.enrollmentAfter.getTime();
+	const enrollment = enrollmentWindow(profile.capSignupAt, config);
 	const imported = Boolean(remote.capImportedAt);
-	const recentJoin =
-		teammate &&
-		Boolean(config.teammateJoinedAt) &&
-		Date.parse(isoDate(config.teammateJoinedAt ?? "")) >=
-			config.enrollmentAfter.getTime();
 	const eligible =
 		config.enrollmentEnabled &&
 		source.signedUp &&
 		subscribed &&
 		!source.pendingInvite &&
-		((!imported && recentSignup) || recentJoin) &&
+		((!imported && enrollment.recentSignup) ||
+			(teammate && enrollment.recentJoin)) &&
 		audience !== "unknown";
 	const {
 		subscribed: _subscribed,

packages/database/loops/worker.ts

@@ -12,6 +12,7 @@ import {
 	LoopsRequestError,
 } from "./client";
 import {
+	enrollmentWindow,
 	type LifecycleContact,
 	type LoopsRuntimeConfig,
 	lifecycleUpdate,
@@ -263,8 +264,10 @@ export async function runLoopsSync(customerCopy: CustomerCopy) {
 								profile: profileFingerprint(localProfile),
 								signedUp: source.signedUp,
 								pendingInvite: source.pendingInvite,
-								enrollmentEnabled: config.enrollmentEnabled,
-								enrollmentAfter: config.enrollmentAfter.toISOString(),
+								enrollment: enrollmentWindow(
+									localProfile.capSignupAt,
+									jobConfig,
+								),
 								listId: config.listId,
 								teammateJoinedAt: job.teammateJoinedAt,
 							}),

scripts/loops/README.md

@@ -80,21 +80,25 @@ Deploy the generated schema before code that selects the new columns. The unship
 
 With the schema available and test configuration set, `bun scripts/loops/seed.ts` reports how many completed signups would be queued. `--apply` queues them without sending anything. `bun scripts/loops/sync.ts --apply` processes a bounded batch using the same worker as the cron route. Test mode fails closed without an allowlist. Set `LOOPS_SYNC_MODE=production` only as part of the reviewed cutover. Seed old completed accounts once after the final suppression reconciliation; subsequent signups enter the queue through the application.
 
+Sync fingerprints include the contact's effective signup and teammate-join eligibility, rather than the raw enrollment switch or cutoff date. Enabling enrollment or moving a future cutoff does not invalidate every historical profile. A signup or join crossing the cutoff, or disabling an eligible contact's enrollment, still changes its fingerprint and requires synchronization.
+
 ## Independent delivery check
 
 `.github/workflows/loops-safety.yml` runs every five minutes, independently of the Cap cron worker. It is disabled until the repository variable `LOOPS_WATCHDOG_ENABLED` is explicitly set to `true`. Configure GitHub secrets `LOOPS_API_KEY` and `LOOPS_HEALTH_SECRET`, and configure the same health secret in Cap. Scheduled Actions run from the default branch, so this protection is not deployed while the PR remains unmerged.
 
 The authenticated, read-only `/api/cron/sync-loops/health` endpoint reports queue counts without contact data. It reports unhealthy when production sync or enrollment is disabled, the queue is empty, a due job is over five minutes late, or a job has failed three times. Expected holds for incomplete signup and removed/changed identities do not count as delivery failures. An unchanged successful refresh clears old failures; a newly queued change does not inherit an older attempt's retry delay.
 
-The checker rejects unavailable, malformed or stale health responses. On failure or audience drift it adds mutually exclusive subscription conditions to each journey's first audience guard, with scope set to all following nodes, then reads the guards back. One failed update does not stop attempts on the other journeys. An update failure or existing hold fails the Action so it remains visible. Assign an owner to GitHub Actions failure notifications and verify that notification before activation.
+The checker rejects unavailable, malformed or stale health responses. For draft or paused journeys, it can add mutually exclusive subscription conditions to the first audience guard, with scope set to all following nodes, then read the guards back. One failed update does not stop attempts on the other journeys. An update failure or existing hold fails the Action so it remains visible.
+
+For a running journey, Loops rejects audience edits with HTTP 400: "This operation is not allowed while the workflow is sending." The checker reports `manual-pause-required`, fails the Action, and explicitly warns that delivery has not stopped. Pause the affected workflows in the Loops UI before applying a hold or changing their guards. This monitor does not automatically stop active delivery. Assign an owner to GitHub Actions failure notifications and verify that notification before activation; do not launch on an assumption of automatic pausing.
 
 The hold persists after recovery. It does not change workflow status, contact subscriptions or campaign schedules. The current four production drafts have this hold applied and independently verified. To inspect them without writes:
 
 ```sh
 bun run emails:check-loops --structure-only --require-held
 ```
 
-For an emergency hold, with the Loops API key in the environment:
+For an emergency, first pause active workflows in the Loops UI. Then, with the Loops API key in the environment, apply persistent audience holds to the stopped workflows:
 
 ```sh
 bun scripts/loops/watchdog.ts --hold --apply
@@ -108,7 +112,7 @@ bun scripts/loops/watchdog.ts --resume --apply
 
 Resume requires a fresh authenticated health response and refuses changed audience rules. It does not start a draft or unpause a workflow. Contacts that have already exited because of a hold are not automatically replayed; assess recovery separately without bulk re-enrolling history.
 
-GitHub scheduling can be delayed, and a Loops API outage can prevent guard updates. A message already being sent may still arrive. Test a running owned-account journey across an outage in the deployed environment before relying on this protection. Manually pause Loops if the check cannot apply a hold. Campaigns require a fresh health check before scheduling and manual cancellation or pause during an outage; the checker only guards the four registered journeys.
+GitHub scheduling can be delayed, and a Loops API outage can prevent even stopped-workflow guard updates. Running workflows keep sending until they are manually paused; a message already being sent may still arrive. Verify the alert and manual-response procedure with an owned-account journey before relying on it. Campaigns require a fresh health check before scheduling and manual cancellation or pause during an outage; the checker only inspects the four registered journeys.
 
 ## Before any activation
 
@@ -125,7 +129,7 @@ Activation is deliberately outside this migration's approved scope. These gates
 1. Review the final custom drafts and audiences, and configure rotated credentials in the intended environment. September 11 controlled deliveries passed SPF, DKIM and DMARC with inherited `p=quarantine`; the earlier DMARC warning is no longer an observed blocker. Recheck sending-domain status at cutover.
 2. The native Stripe connection passed owned live-customer creation/update tests, including name sync, list assignment, global unsubscribe preservation and a recipient Preference Center mailing-list opt-out surviving a later native update. Import/reconcile the remaining Bento negative records with workflows off. Then deploy the reviewed schema/code in test mode and prove the actual signup, purchase and invite routes with owned accounts.
 3. Set a future `LOOPS_ENROLLMENT_AFTER` timestamp, verify the cron schedule and permissions in the deployed environment, and seed the completed-account sync jobs. Keep enrollment disabled while inspecting the resulting contacts. Preserve existing opt-outs and suppressions; no separate consent-capture step is needed.
-4. Configure and enable the independent delivery check above, verify its deployed health endpoint and failure notifications, and prove purchase/invite/opt-out transitions remove contacts before later promotional steps. Exercise its hold and explicit recovery with an owned-account journey. Downstream filters use the last synced fields; `capVerifiedAt` is not a native expiry guarantee. Pause workflows manually if the checker cannot reach Loops, and handle scheduled campaigns separately.
+4. Configure and enable the independent delivery check above, verify its deployed health endpoint and failure notifications, and prove purchase/invite/opt-out transitions remove contacts before later promotional steps. Exercise failure detection, manual pause, hold and explicit recovery with an owned-account journey. The September 11 live test confirmed that the API cannot change guards while Sending; automatic stopping remains unresolved. Approve an operational plan with manual pausing, or keep delivery off until a supported automatic control is available. Downstream filters use the last synced fields; `capVerifiedAt` is not a native expiry guarantee. Handle scheduled campaigns separately.
 5. Reconcile a fresh Bento delta at cutover, including all opt-outs and changed entitlements. Confirm no campaign/flow is queued to send twice, check overlap with Resend recording emails, then disable old Bento marketing automations only as part of the approved cutover. Preserve source history and suppression evidence.
 6. Test the deployed Cap signup/purchase/invite/opt-out path through Loops to an owned inbox, including a sync outage and retries. The completed synthetic profile-to-Loops tests do not replace this production integration check.
 7. After explicit activation approval, choose a future enrollment boundary with enough setup time. While that boundary is still in the future, enable production sync/enrollment and the independent delivery check, confirm healthy queue results, explicitly remove the delivery holds, and start the reviewed Loops workflows. Complete those steps before the boundary so the first eligible signup is processed by an already-running workflow; if setup overruns, move the boundary forward before allowing enrollment. This release admits new signups after the boundary, not a numerically limited cohort. Monitor deliveries, complaints, opt-outs and duplicate suppression. Do not bulk enroll imported history or replay stage changes emitted while workflows were drafts or held. Rollback stops new enrollment and pauses Loops before considering re-enabling Bento; never run both senders for the same journey. Retire Bento and rotate remaining credentials after reconciliation.

scripts/loops/lifecycle.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 import { customerCopy } from "../../emails/customer-copy";
 import type { LoopsClient } from "../../packages/database/loops/client";
 import {
+	enrollmentWindow,
 	type LifecycleContact,
 	type LoopsRuntimeConfig,
 	lifecycleUpdate,
@@ -21,6 +22,53 @@ const config: LoopsRuntimeConfig = {
 	allowedEmails: new Set([email]),
 };
 
+describe("enrollment fingerprint", () => {
+	test("historical accounts stay unchanged when enrollment is enabled or the future cutoff moves", () => {
+		const signup = "2026-09-10T10:00:00.000Z";
+		const held = enrollmentWindow(signup, {
+			...config,
+			enrollmentEnabled: false,
+		});
+		expect(held).toEqual({ recentSignup: false, recentJoin: false });
+		expect(enrollmentWindow(signup, config)).toEqual(held);
+		expect(
+			enrollmentWindow(signup, {
+				...config,
+				enrollmentAfter: new Date("2026-09-12T00:00:00Z"),
+			}),
+		).toEqual(held);
+	});
+
+	test("eligible signups change when enrollment is disabled or the cutoff crosses their signup", () => {
+		const signup = "2026-09-11T10:00:00.000Z";
+		expect(enrollmentWindow(signup, config)).toEqual({
+			recentSignup: true,
+			recentJoin: false,
+		});
+		expect(
+			enrollmentWindow(signup, { ...config, enrollmentEnabled: false }),
+		).toEqual({ recentSignup: false, recentJoin: false });
+		expect(
+			enrollmentWindow(signup, {
+				...config,
+				enrollmentAfter: new Date("2026-09-11T10:00:00.001Z"),
+			}),
+		).toEqual({ recentSignup: false, recentJoin: false });
+	});
+
+	test("new teammate joins change historical accounts at the exact cutoff", () => {
+		const signup = "2026-09-10T10:00:00.000Z";
+		const joined = { ...config, teammateJoinedAt: "2026-09-11 00:00:00" };
+		expect(enrollmentWindow(signup, joined)).toEqual({
+			recentSignup: false,
+			recentJoin: true,
+		});
+		expect(
+			enrollmentWindow(signup, { ...joined, enrollmentEnabled: false }),
+		).toEqual({ recentSignup: false, recentJoin: false });
+	});
+});
+
 function fixture(): LoopsProfileSource {
 	return {
 		input: {