Skip to content

Commit 19ee424

Browse files
devkiransteven-tey
devkiran
and
steven-tey
authored
Sync partner social statistics (#3328)
Co-authored-by: Steven Tey <[email protected]>
1 parent de33a63 commit 19ee424

File tree

20 files changed

+580
-200
lines changed

20 files changed

+580
-200
lines changed

‎apps/web/app/(ee)/api/cron/partner-platforms/route.ts‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
import {
2+
AccountNotFoundError,
3+
fetchSocialProfile,
4+
} from "@/lib/api/scrape-creators/fetch-social-profile";
5+
import { qstash } from "@/lib/cron";
6+
import { withCron } from "@/lib/cron/with-cron";
7+
import { prisma } from "@dub/prisma";
8+
import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
9+
import { subDays } from "date-fns";
10+
import * as z from "zod/v4";
11+
import { logAndRespond } from "../utils";
12+
13+
export const dynamic = "force-dynamic";
14+
15+
const BATCH_SIZE = 50;
16+
17+
const schema = z.object({
18+
startingAfter: z.string().optional(),
19+
});
20+
21+
/**
22+
* This route is used to update stats for verified Instagram, TikTok, and Twitter partners using the ScrapeCreators API
23+
* Runs once a day at 06:00 AM UTC (cron expression: 0 6 * * *)
24+
* POST /api/cron/partner-platforms
25+
*/
26+
export const POST = withCron(async ({ rawBody }) => {
27+
if (!process.env.SCRAPECREATORS_API_KEY) {
28+
throw new Error("SCRAPECREATORS_API_KEY is not defined");
29+
}
30+
31+
let { startingAfter } = schema.parse(
32+
rawBody ? JSON.parse(rawBody) : { startingAfter: undefined },
33+
);
34+
35+
const verifiedProfiles = await prisma.partnerPlatform.findMany({
36+
where: {
37+
type: {
38+
in: ["instagram", "tiktok", "twitter"],
39+
},
40+
verifiedAt: {
41+
not: null,
42+
},
43+
// only check platforms that haven't been checked in the last 7 days
44+
OR: [
45+
{
46+
lastCheckedAt: {
47+
lt: subDays(new Date(), 7),
48+
},
49+
},
50+
{
51+
lastCheckedAt: null,
52+
},
53+
],
54+
// only check partners that are discoverable in the partner network
55+
partner: {
56+
discoverableAt: {
57+
not: null,
58+
},
59+
},
60+
},
61+
take: BATCH_SIZE,
62+
...(startingAfter && {
63+
cursor: {
64+
id: startingAfter,
65+
},
66+
skip: 1,
67+
}),
68+
orderBy: {
69+
id: "asc",
70+
},
71+
});
72+
73+
if (verifiedProfiles.length === 0) {
74+
return logAndRespond(
75+
"No more verified social profiles found. Finished updating social platform stats.",
76+
);
77+
}
78+
79+
await Promise.allSettled(
80+
verifiedProfiles.map(async (verifiedProfile) => {
81+
if (!verifiedProfile.identifier || !verifiedProfile.type) {
82+
return;
83+
}
84+
85+
try {
86+
const socialProfile = await fetchSocialProfile({
87+
platform: verifiedProfile.type,
88+
handle: verifiedProfile.identifier,
89+
});
90+
91+
const newStats = {
92+
subscribers: socialProfile.subscribers,
93+
posts: socialProfile.posts,
94+
};
95+
96+
await prisma.partnerPlatform.update({
97+
where: {
98+
id: verifiedProfile.id,
99+
},
100+
data: {
101+
...newStats,
102+
lastCheckedAt: new Date(),
103+
},
104+
});
105+
106+
console.log(
107+
`Updated ${verifiedProfile.type} stats for @${verifiedProfile.identifier}`,
108+
newStats,
109+
);
110+
} catch (error) {
111+
// If account doesn't exist, unverify the platform
112+
if (error instanceof AccountNotFoundError) {
113+
await prisma.partnerPlatform.update({
114+
where: {
115+
id: verifiedProfile.id,
116+
},
117+
data: {
118+
verifiedAt: null,
119+
lastCheckedAt: new Date(),
120+
},
121+
});
122+
123+
console.log(
124+
`Account @${verifiedProfile.identifier} on ${verifiedProfile.type} no longer exists. Unverified platform.`,
125+
);
126+
return;
127+
}
128+
129+
console.error(
130+
`Error updating ${verifiedProfile.type} stats for @${verifiedProfile.identifier}:`,
131+
error,
132+
);
133+
}
134+
}),
135+
);
136+
137+
if (verifiedProfiles.length === BATCH_SIZE) {
138+
startingAfter = verifiedProfiles[verifiedProfiles.length - 1].id;
139+
140+
await qstash.publishJSON({
141+
url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-platforms`,
142+
method: "POST",
143+
body: {
144+
startingAfter,
145+
},
146+
});
147+
148+
return logAndRespond(
149+
`Processed ${BATCH_SIZE} profiles. Scheduled next batch (startingAfter: ${startingAfter}).`,
150+
);
151+
}
152+
153+
return logAndRespond(
154+
"Finished updating social platform stats for all verified profiles.",
155+
);
156+
});

apps/web/app/(ee)/api/cron/online-presence/youtube/route.ts renamed to apps/web/app/(ee)/api/cron/partner-platforms/youtube/route.ts

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { withCron } from "@/lib/cron/with-cron";
22
import { prisma } from "@dub/prisma";
33
import { PlatformType } from "@dub/prisma/client";
4-
import { chunk, deepEqual } from "@dub/utils";
5-
import { z } from "zod";
4+
import { chunk } from "@dub/utils";
5+
import * as z from "zod/v4";
66
import { logAndRespond } from "../../utils";
77

88
const youtubeChannelSchema = z.object({
@@ -16,10 +16,11 @@ const youtubeChannelSchema = z.object({
1616

1717
export const dynamic = "force-dynamic";
1818

19-
/*
20-
This route is used to update youtube stats for youtubeVerified partners
21-
Runs once a day at 06:00 AM UTC (cron expression: 0 6 * * *)
22-
*/
19+
/**
20+
* This route is used to update stats for YouTube verified partners using the YouTube API
21+
* Runs once a day at 06:00 AM UTC (cron expression: 0 6 * * *)
22+
* POST /api/cron/partner-platforms/youtube
23+
*/
2324
export const POST = withCron(async () => {
2425
if (!process.env.YOUTUBE_API_KEY) {
2526
throw new Error("YOUTUBE_API_KEY is not defined");
@@ -88,15 +89,14 @@ export const POST = withCron(async () => {
8889
views: channel.statistics.viewCount,
8990
};
9091

91-
if (deepEqual(currentStats, newStats)) {
92-
continue;
93-
}
94-
9592
await prisma.partnerPlatform.update({
9693
where: {
9794
id: partnerPlatform.id,
9895
},
99-
data: newStats,
96+
data: {
97+
...newStats,
98+
lastCheckedAt: new Date(),
99+
},
100100
});
101101

102102
console.log(

‎apps/web/app/(ee)/api/partners/online-presence/callback/route.ts‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
import { ONLINE_PRESENCE_PROVIDERS } from "@/lib/api/partner-profile/online-presence-providers";
2+
import { fetchSocialProfile } from "@/lib/api/scrape-creators/fetch-social-profile";
23
import { getSession } from "@/lib/auth/utils";
34
import { redis } from "@/lib/upstash/redis";
45
import { prisma } from "@dub/prisma";
5-
import { PlatformType } from "@dub/prisma/client";
6+
import { PartnerPlatform, PlatformType } from "@dub/prisma/client";
67
import {
78
getSearchParams,
89
PARTNERS_DOMAIN,
910
PARTNERS_DOMAIN_WITH_NGROK,
1011
} from "@dub/utils";
1112
import { cookies } from "next/headers";
1213
import { NextResponse } from "next/server";
13-
import { z } from "zod";
14+
import * as z from "zod/v4";
1415

1516
const requestSchema = z.object({
1617
code: z.string(),
@@ -148,6 +149,33 @@ export async function GET(req: Request) {
148149
return NextResponse.redirect(redirectUrl);
149150
}
150151

152+
// Sync social stats for platforms
153+
let socialStats: Pick<PartnerPlatform, "subscribers" | "posts" | "views"> = {
154+
subscribers: BigInt(0),
155+
posts: BigInt(0),
156+
views: BigInt(0),
157+
};
158+
159+
if (["tiktok", "twitter"].includes(platform)) {
160+
try {
161+
const socialProfile = await fetchSocialProfile({
162+
platform,
163+
handle: partnerPlatform.identifier,
164+
});
165+
166+
socialStats = {
167+
subscribers: socialProfile.subscribers,
168+
posts: socialProfile.posts,
169+
views: socialProfile.views,
170+
};
171+
} catch (error) {
172+
console.error(
173+
`Failed to fetch social stats for ${platform} handle @${partnerPlatform.identifier}:`,
174+
error,
175+
);
176+
}
177+
}
178+
151179
await prisma.partnerPlatform.update({
152180
where: {
153181
partnerId_type: {
@@ -157,7 +185,8 @@ export async function GET(req: Request) {
157185
},
158186
data: {
159187
verifiedAt: new Date(),
160-
metadata: metadata || undefined,
188+
...(metadata && { metadata }),
189+
...socialStats,
161190
},
162191
});
163192

‎apps/web/app/(ee)/partners.dub.co/(onboarding)/onboarding/online-presence/page.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { OnlinePresenceForm } from "@/ui/partners/online-presence-form";
66
import { prisma } from "@dub/prisma";
77
import { PlatformType } from "@dub/prisma/client";
88
import { Suspense } from "react";
9-
import { z } from "zod";
9+
import * as z from "zod/v4";
1010
import { OnlinePresencePageClient } from "./page-client";
1111

1212
export default function OnlinePresencePage() {

‎apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"use client";
22

33
import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner";
4-
import { ONLINE_PRESENCE_FIELDS } from "@/lib/partners/online-presence";
4+
import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms";
55
import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count";
66
import useWorkspace from "@/lib/swr/use-workspace";
77
import { NetworkPartnerProps } from "@/lib/types";
@@ -383,7 +383,7 @@ function PartnerCard({
383383
const onlinePresenceData = useMemo(
384384
() =>
385385
partner
386-
? ONLINE_PRESENCE_FIELDS.map((field) => ({
386+
? PARTNER_PLATFORM_FIELDS.map((field) => ({
387387
label: field.label,
388388
icon: field.icon,
389389
...field.data(partner.platforms),

‎apps/web/lib/actions/partners/start-partner-platform-verification.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { PlatformType } from "@dub/prisma/client";
1717
import { nanoid, PARTNERS_DOMAIN_WITH_NGROK } from "@dub/utils";
1818
import { cookies } from "next/headers";
1919
import { v4 as uuid } from "uuid";
20-
import { z } from "zod";
20+
import * as z from "zod/v4";
2121
import { authPartnerActionClient } from "../safe-action";
2222

2323
const startPartnerPlatformVerificationSchema = z.object({

‎apps/web/lib/actions/partners/verify-social-account-by-code.ts‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"use server";
22

3-
import { scrapeCreatorsClient } from "@/lib/api/scrapecreators/client";
3+
import { fetchSocialProfile } from "@/lib/api/scrape-creators/fetch-social-profile";
44
import { ratelimit } from "@/lib/upstash";
55
import { redis } from "@/lib/upstash/redis";
66
import { prisma } from "@dub/prisma";
@@ -74,18 +74,12 @@ export const verifySocialAccountByCodeAction = authPartnerActionClient
7474
// Verifies that a verification code exists in the account's profile bio/description.
7575
// Fetches the account profile and checks if the provided code appears in any of the
7676
// profile text fields (description, about, bio, summary).
77-
const socialProfile = await scrapeCreatorsClient.fetchSocialProfile({
77+
const socialProfile = await fetchSocialProfile({
7878
platform,
7979
handle,
8080
});
8181

82-
if (!socialProfile) {
83-
throw new Error(
84-
"We were unable to retrieve your social media profile. Please try again.",
85-
);
86-
}
87-
88-
if (!socialProfile.description) {
82+
if (!socialProfile.description || socialProfile.description.length === 0) {
8983
throw new Error(
9084
`We could not find a public ${
9185
platform === "youtube" ? "channel description" : "bio"
@@ -113,9 +107,11 @@ export const verifySocialAccountByCodeAction = authPartnerActionClient
113107
data: {
114108
verifiedAt: new Date(),
115109
platformId: socialProfile.platformId,
110+
subscribers: socialProfile.subscribers,
111+
posts: socialProfile.posts,
112+
views: socialProfile.views,
116113
},
117114
});
118115

119-
// Delete the verification code from Redis
120116
await redis.del(cacheKey);
121117
});

‎apps/web/lib/api/partner-profile/online-presence-providers.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ export const ONLINE_PRESENCE_PROVIDERS: Record<string, OnlinePresenceProvider> =
117117
},
118118
},
119119

120+
// We don't support LinkedIn verification yet
120121
linkedin: {
121122
authUrl: "https://www.linkedin.com/oauth/v2/authorization",
122123
tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",

‎apps/web/lib/api/scrape-creators/client.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { createFetch, createSchema } from "@better-fetch/fetch";
2+
import { PlatformType } from "@dub/prisma/client";
3+
import * as z from "zod/v4";
4+
import { profileResponseSchema } from "./schema";
5+
6+
export const scrapeCreatorsFetch = createFetch({
7+
baseURL: "https://api.scrapecreators.com",
8+
retry: {
9+
type: "linear",
10+
attempts: 2,
11+
delay: 3000,
12+
},
13+
headers: {
14+
"x-api-key": process.env.SCRAPECREATORS_API_KEY!,
15+
},
16+
schema: createSchema(
17+
{
18+
"/v1/:platform/:handleType": {
19+
method: "get",
20+
params: z.object({
21+
platform: z.enum(PlatformType),
22+
handleType: z.enum(["channel", "profile"]),
23+
}),
24+
query: z.object({
25+
handle: z.string(),
26+
}),
27+
output: profileResponseSchema,
28+
},
29+
},
30+
{ strict: true },
31+
),
32+
onError: ({ error }) => {
33+
console.error("[ScrapeCreators] Error", error);
34+
},
35+
// onResponse: async ({ response }) => {
36+
// if (process.env.NODE_ENV === "development") {
37+
// console.log(
38+
// "[ScrapeCreators] Response",
39+
// prettyPrint(await response.clone().json()),
40+
// );
41+
// }
42+
// },
43+
});

0 commit comments

Comments
 (0)