Skip to content

Commit 5479242

Browse files
steven-teysteven-tey
authored
Show DR for verified partner websites (#4024)
1 parent 6f902f4 commit 5479242

8 files changed

Lines changed: 563 additions & 105 deletions

File tree

‎apps/web/app/(ee)/api/cron/partner-platforms/route.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ export const POST = withCron(async ({ rawBody }) => {
5151
lastCheckedAt: null,
5252
},
5353
],
54-
// only check partners that are approved or trusted
54+
// only check partners approved/trusted in the network
5555
partner: {
5656
networkStatus: {
5757
in: ["approved", "trusted"],

‎apps/web/app/(ee)/api/cron/partner-platforms/website/get-domain-rating.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import * as z from "zod/v4";
2+
3+
const domainRatingSchema = z.object({
4+
domain_rating: z.object({
5+
domain_rating: z.number(),
6+
}),
7+
});
8+
9+
export async function getDomainRating(target: string) {
10+
const response = await fetch(
11+
`https://api.ahrefs.com/v3/public/domain-rating-free?target=${encodeURIComponent(target)}&output=json`,
12+
{
13+
headers: {
14+
Accept: "application/json",
15+
},
16+
},
17+
);
18+
19+
if (!response.ok) {
20+
throw new Error(
21+
`Failed to fetch domain rating for ${target}: ${response.status} ${await response.text()}`,
22+
);
23+
}
24+
25+
const data = await response.json();
26+
return Math.round(domainRatingSchema.parse(data).domain_rating.domain_rating);
27+
}

‎apps/web/app/(ee)/api/cron/partner-platforms/website/route.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import { qstash } from "@/lib/cron";
2+
import { withCron } from "@/lib/cron/with-cron";
3+
import { prisma } from "@dub/prisma";
4+
import { PlatformType } from "@dub/prisma/client";
5+
import { APP_DOMAIN_WITH_NGROK, chunk, getDomainWithoutWWW } from "@dub/utils";
6+
import * as z from "zod/v4";
7+
import { logAndRespond } from "../../utils";
8+
import { getDomainRating } from "./get-domain-rating";
9+
10+
export const dynamic = "force-dynamic";
11+
12+
const BATCH_SIZE = 500;
13+
const CONCURRENCY = 10;
14+
15+
const schema = z.object({
16+
startingAfter: z.string().optional(),
17+
});
18+
19+
/**
20+
* This route is used to update domain rating (DR) for verified website partners using the Ahrefs free API
21+
* Runs once a day at 05:00 AM UTC (cron expression: 0 5 * * *)
22+
* POST /api/cron/partner-platforms/website
23+
*/
24+
export const POST = withCron(async ({ rawBody }) => {
25+
let { startingAfter } = schema.parse(
26+
rawBody ? JSON.parse(rawBody) : { startingAfter: undefined },
27+
);
28+
29+
const websites = await prisma.partnerPlatform.findMany({
30+
where: {
31+
type: PlatformType.website,
32+
verifiedAt: {
33+
not: null,
34+
},
35+
},
36+
take: BATCH_SIZE,
37+
...(startingAfter && {
38+
cursor: {
39+
id: startingAfter,
40+
},
41+
skip: 1,
42+
}),
43+
orderBy: {
44+
id: "asc",
45+
},
46+
});
47+
48+
if (websites.length === 0) {
49+
return logAndRespond(
50+
"No more website platforms found. Finished updating website domain ratings.",
51+
);
52+
}
53+
54+
const websiteChunks = chunk(websites, CONCURRENCY);
55+
56+
for (const websiteChunk of websiteChunks) {
57+
await Promise.allSettled(
58+
websiteChunk.map(async (website) => {
59+
const target =
60+
getDomainWithoutWWW(website.identifier) ?? website.identifier;
61+
62+
if (!target) {
63+
console.error(
64+
`Invalid website identifier for partner platform ${website.id}: ${website.identifier}`,
65+
);
66+
return;
67+
}
68+
69+
try {
70+
const domainRating = await getDomainRating(target);
71+
72+
if (website.subscribers === BigInt(domainRating)) {
73+
console.log(
74+
`No changes to update for ${target} (DR: ${domainRating}), skipping...`,
75+
);
76+
return;
77+
}
78+
79+
await prisma.partnerPlatform.update({
80+
where: {
81+
id: website.id,
82+
},
83+
data: {
84+
subscribers: domainRating,
85+
lastCheckedAt: new Date(),
86+
},
87+
});
88+
89+
console.log(`Updated domain rating for ${target}`, {
90+
domainRating,
91+
});
92+
} catch (error) {
93+
console.error(`Error updating domain rating for ${target}:`, error);
94+
}
95+
}),
96+
);
97+
}
98+
99+
if (websites.length === BATCH_SIZE) {
100+
startingAfter = websites[websites.length - 1].id;
101+
102+
await qstash.publishJSON({
103+
url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-platforms/website`,
104+
method: "POST",
105+
body: {
106+
startingAfter,
107+
},
108+
});
109+
110+
return logAndRespond(
111+
`Processed ${BATCH_SIZE} websites. Scheduled next batch (startingAfter: ${startingAfter}).`,
112+
);
113+
}
114+
115+
return logAndRespond(
116+
`Finished updating domain ratings for ${websites.length} websites.`,
117+
);
118+
});

‎apps/web/app/(ee)/api/cron/partner-platforms/youtube/route.ts‎

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,35 @@
1+
import { qstash } from "@/lib/cron";
12
import { withCron } from "@/lib/cron/with-cron";
23
import { prisma } from "@dub/prisma";
34
import { PlatformType } from "@dub/prisma/client";
4-
import { chunk } from "@dub/utils";
5+
import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";
56
import * as z from "zod/v4";
67
import { logAndRespond } from "../../utils";
78
import { youtubeChannelSchema } from "./youtube-channel-schema";
89

910
export const dynamic = "force-dynamic";
1011

12+
const BATCH_SIZE = 1000;
13+
const YOUTUBE_API_CHUNK_SIZE = 50;
14+
15+
const schema = z.object({
16+
startingAfter: z.string().optional(),
17+
});
18+
1119
/**
1220
* This route is used to update stats for YouTube verified partners using the YouTube API
1321
* Runs once a day at 06:00 AM UTC (cron expression: 0 6 * * *)
1422
* POST /api/cron/partner-platforms/youtube
1523
*/
16-
export const POST = withCron(async () => {
24+
export const POST = withCron(async ({ rawBody }) => {
1725
if (!process.env.YOUTUBE_API_KEY) {
1826
throw new Error("YOUTUBE_API_KEY is not defined");
1927
}
2028

29+
let { startingAfter } = schema.parse(
30+
rawBody ? JSON.parse(rawBody) : { startingAfter: undefined },
31+
);
32+
2133
const youtubeChannels = await prisma.partnerPlatform.findMany({
2234
where: {
2335
type: PlatformType.youtube,
@@ -28,15 +40,25 @@ export const POST = withCron(async () => {
2840
not: null,
2941
},
3042
},
43+
take: BATCH_SIZE,
44+
...(startingAfter && {
45+
cursor: {
46+
id: startingAfter,
47+
},
48+
skip: 1,
49+
}),
50+
orderBy: {
51+
id: "asc",
52+
},
3153
});
3254

3355
if (youtubeChannels.length === 0) {
3456
return logAndRespond(
35-
"No YouTube platforms found. Skipping YouTube stats update.",
57+
"No more YouTube platforms found. Finished updating YouTube stats.",
3658
);
3759
}
3860

39-
const channelChunks = chunk(youtubeChannels, 50);
61+
const channelChunks = chunk(youtubeChannels, YOUTUBE_API_CHUNK_SIZE);
4062

4163
for (const channelChunk of channelChunks) {
4264
const channelIds = channelChunk.map((channel) => channel.platformId);
@@ -79,12 +101,27 @@ export const POST = withCron(async () => {
79101
subscribers: channel.statistics.subscriberCount,
80102
posts: channel.statistics.videoCount,
81103
views: channel.statistics.viewCount,
82-
avatarUrl: channel.snippet?.thumbnails?.default?.url,
104+
avatarUrl: channel.snippet?.thumbnails?.default?.url ?? null,
83105
...(channel.snippet?.customUrl && {
84106
identifier: channel.snippet.customUrl.replace("@", ""),
85107
}),
86108
};
87109

110+
const hasChanges =
111+
partnerPlatform.subscribers !== BigInt(newStats.subscribers) ||
112+
partnerPlatform.posts !== BigInt(newStats.posts) ||
113+
partnerPlatform.views !== BigInt(newStats.views) ||
114+
partnerPlatform.avatarUrl !== newStats.avatarUrl ||
115+
("identifier" in newStats &&
116+
partnerPlatform.identifier !== newStats.identifier);
117+
118+
if (!hasChanges) {
119+
console.log(
120+
`No changes to update for @${partnerPlatform.identifier}, skipping...`,
121+
);
122+
return;
123+
}
124+
88125
await prisma.partnerPlatform.update({
89126
where: {
90127
id: partnerPlatform.id,
@@ -104,7 +141,23 @@ export const POST = withCron(async () => {
104141
}
105142
}
106143

144+
if (youtubeChannels.length === BATCH_SIZE) {
145+
startingAfter = youtubeChannels[youtubeChannels.length - 1].id;
146+
147+
await qstash.publishJSON({
148+
url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-platforms/youtube`,
149+
method: "POST",
150+
body: {
151+
startingAfter,
152+
},
153+
});
154+
155+
return logAndRespond(
156+
`Processed ${BATCH_SIZE} YouTube channels. Scheduled next batch (startingAfter: ${startingAfter}).`,
157+
);
158+
}
159+
107160
return logAndRespond(
108-
`YouTube stats updated for ${youtubeChannels.length} partners`,
161+
`Finished updating YouTube stats for ${youtubeChannels.length} partners.`,
109162
);
110163
});

0 commit comments

Comments
 (0)