diff --git a/apps/bot/compose.yaml b/apps/bot/compose.yaml index ba312b4..1a171d7 100644 --- a/apps/bot/compose.yaml +++ b/apps/bot/compose.yaml @@ -27,6 +27,8 @@ services: EMBEDLY_API_DOMAIN: ${EMBEDLY_API_DOMAIN} EMBEDLY_AUTH_SECRET: ${EMBEDLY_AUTH_SECRET} EMBED_USER_AGENT: ${EMBED_USER_AGENT} + POSTHOG_API_KEY: ${POSTHOG_API_KEY:-} + POSTHOG_HOST: ${POSTHOG_HOST:-https://us.i.posthog.com} MESSAGE_CACHE_TTL_SECONDS: ${MESSAGE_CACHE_TTL_SECONDS:-86400} OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT} OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-embedly-bot} diff --git a/apps/bot/package.json b/apps/bot/package.json index a1be8c3..8bf9f07 100644 --- a/apps/bot/package.json +++ b/apps/bot/package.json @@ -25,6 +25,7 @@ "@sapphire/framework": "^5.5.0", "discord.js": "~14.26.3", "hono": "^4.12.15", + "posthog-node": "^5.51.6", "redis": "^6.0.0" }, "devDependencies": { diff --git a/apps/bot/src/lib/builder.ts b/apps/bot/src/lib/builder.ts index 3559b53..41dc558 100644 --- a/apps/bot/src/lib/builder.ts +++ b/apps/bot/src/lib/builder.ts @@ -59,7 +59,11 @@ function buildMediaEmbed(media: NormalizedPost["media"], spoiler?: EmbedFlags["S const gallery = new MediaGalleryBuilder(); gallery.addItems( - media.slice(0, MAX_GALLERY_ITEMS).map((m) => ({ media: { url: m.url }, spoiler })), + media.slice(0, MAX_GALLERY_ITEMS).map((m) => ({ + media: { url: m.url }, + description: m.description?.slice(0, 1024) || undefined, + spoiler, + })), ); return gallery.toJSON(); @@ -79,7 +83,11 @@ function addPostComponents(embed: ContainerBuilder, post: PostData, headingPrefi post.author.handle && post.author.url ? `(${hyperlink(`@${post.author.handle}`, post.author.url)})` : ""; - const authorHeading = [headingPrefix, post.author.name, authorHandle].filter(Boolean).join(" "); + const authorName = + post.platform === "FacebookMarketplace" + ? truncate(escapeMarkdown(post.author.name), 250) + : post.author.name; + const authorHeading = [headingPrefix, authorName, authorHandle].filter(Boolean).join(" "); embed.addSectionComponents((section) => { section @@ -87,6 +95,10 @@ function addPostComponents(embed: ContainerBuilder, post: PostData, headingPrefi .addTextDisplayComponents((author) => author.setContent(heading(authorHeading, HeadingLevel.Three)), ); + if (post.platform === "FacebookMarketplace" && post.price) { + const price = post.price; + section.addTextDisplayComponents((display) => display.setContent(escapeMarkdown(price))); + } if (post.text && post.text.length > 0) { section.addTextDisplayComponents((text) => text.setContent( @@ -121,6 +133,31 @@ function addPostComponents(embed: ContainerBuilder, post: PostData, headingPrefi if (post.media.length > 0) { embed.addMediaGalleryComponents(buildMediaEmbed(post.media)!); } + if (post.platform === "FacebookMarketplace") { + const { location, map } = post; + if (location) { + embed.addTextDisplayComponents((display) => + display.setContent(`${escapeMarkdown(location)} · Location is approximate`), + ); + } + if (map) { + embed.addMediaGalleryComponents((gallery) => + gallery.addItems({ + media: { url: map }, + description: (location + ? `Map showing approximate listing location in ${location}.` + : "Map showing approximate listing location." + ).slice(0, 1024), + }), + ); + } + embed.addTextDisplayComponents((footer) => + footer.setContent( + `${time(post.timestamp, TimestampStyles.RelativeTime)} • ${hyperlink("View on Facebook Marketplace", post.url)}`, + ), + ); + return; + } embed .addSeparatorComponents((sep) => sep.setDivider(false).setSpacing(SeparatorSpacingSize.Small)) .addTextDisplayComponents( @@ -140,6 +177,7 @@ function addPostComponents(embed: ContainerBuilder, post: PostData, headingPrefi } export function buildEmbed(post: PostData, flags?: Partial) { + if (post.platform === "FacebookMarketplace" && post.media.length === 0) return null; if (flags?.MediaOnly) { return buildMediaEmbed(post.media, flags?.Spoiler); } diff --git a/apps/bot/src/lib/client.ts b/apps/bot/src/lib/client.ts index 8adc7f6..8cf7a9d 100644 --- a/apps/bot/src/lib/client.ts +++ b/apps/bot/src/lib/client.ts @@ -4,6 +4,7 @@ import type { AppType } from "@embedly/api"; import { container, SapphireClient } from "@sapphire/framework"; import { ActivityType, GatewayIntentBits, Partials, PresenceUpdateStatus } from "discord.js"; import { hc } from "hono/client"; +import { PostHog } from "posthog-node"; import { MessageCache } from "./messageCache"; export class EmbedlyClient extends SapphireClient { @@ -33,11 +34,17 @@ export class EmbedlyClient extends SapphireClient { public override async login(token?: string) { container.api = hc(process.env.EMBEDLY_API_DOMAIN ?? "http://localhost:8787"); container.messageCache = await MessageCache.connect(); + if (process.env.POSTHOG_API_KEY) { + container.posthog = new PostHog(process.env.POSTHOG_API_KEY, { + host: process.env.POSTHOG_HOST, + }); + } return super.login(token); } public override async destroy() { await container.messageCache?.close(); + await container.posthog?.shutdown(); return super.destroy(); } } @@ -46,5 +53,6 @@ declare module "@sapphire/framework" { interface Container { api: ReturnType>; messageCache: MessageCache; + posthog?: PostHog; } } diff --git a/apps/bot/src/lib/handleUrls.ts b/apps/bot/src/lib/handleUrls.ts index 131949a..19a5ee9 100644 --- a/apps/bot/src/lib/handleUrls.ts +++ b/apps/bot/src/lib/handleUrls.ts @@ -186,7 +186,7 @@ export async function handleUrls( guild_id: interaction!.guildId ?? "dm", user_id: interaction!.user.id, }; - const matches = ( + let matches = ( await Promise.all( urls.map(async (request, requestIndex) => { if (options.updateTargets && !options.updateTargets.has(requestIndex)) return null; @@ -217,6 +217,28 @@ export async function handleUrls( await reactToFailure(); } + if (matches.some((match) => match.platform === "FacebookMarketplace")) { + let hasAccess = false; + if (container.posthog) { + try { + const flags = await container.posthog.evaluateFlags(matchContext.user_id, { + flagKeys: ["facebook-marketplace"], + }); + hasAccess = flags.getFlag("facebook-marketplace") === true; + } catch (error) { + container.logger.warn( + formatLog("warn", EmbedlyErrors.FeatureFlagFailed, { + ...matchContext, + ...getErrorContext(error), + }), + ); + } + } + if (!hasAccess) { + matches = matches.filter((match) => match.platform !== "FacebookMarketplace"); + } + } + if (interaction && matches.length === 0) { const requestId = `${embedSource}:${interaction.id}`; const error = diff --git a/packages/logging/src/main.ts b/packages/logging/src/main.ts index ed5bc3a..361405a 100644 --- a/packages/logging/src/main.ts +++ b/packages/logging/src/main.ts @@ -39,6 +39,12 @@ export function defineError(event: EmbedlyErrorEvent) { } export const EmbedlyErrors = { + FeatureFlagFailed: defineError({ + type: "feature_flag.failed", + title: "Access check failed.", + detail: "Could not check Marketplace access.", + status: 502, + }), NoUrlsFound: defineError({ type: "embed.no_urls_found", title: "No URLs Found.", diff --git a/packages/platforms/src/platforms/facebook-marketplace.d.ts b/packages/platforms/src/platforms/facebook-marketplace.d.ts new file mode 100644 index 0000000..fbb06d6 --- /dev/null +++ b/packages/platforms/src/platforms/facebook-marketplace.d.ts @@ -0,0 +1,30 @@ +export interface MarketplaceListing { + id: string; + marketplace_listing_title: string; + redacted_description?: { text: string } | null; + creation_time: number; + formatted_price?: { text: string } | null; + listing_price?: { formatted_amount_zeros_stripped?: string } | null; + location_text?: { text: string } | null; + location?: { latitude: number; longitude: number } | null; +} + +export interface MarketplacePhotos { + id: string; + listing_photos: Array<{ + image: { uri: string }; + accessibility_caption?: string; + }>; +} + +export interface MarketplaceData { + listing: MarketplaceListing; + photos: MarketplacePhotos; + mapTemplate?: string; +} + +export interface MarketplaceMeta { + price?: string; + location?: string; + map?: string; +} diff --git a/packages/platforms/src/platforms/facebook-marketplace.ts b/packages/platforms/src/platforms/facebook-marketplace.ts new file mode 100644 index 0000000..7f5c2f4 --- /dev/null +++ b/packages/platforms/src/platforms/facebook-marketplace.ts @@ -0,0 +1,126 @@ +import * as cheerio from "cheerio"; + +import type { Platform } from "../types"; +import type { + MarketplaceData, + MarketplaceListing, + MarketplaceMeta, + MarketplacePhotos, +} from "./facebook-marketplace.d"; + +const MATCH_RE = + /^(?:https?:\/\/)?(?:www\.|m\.)?facebook\.com\/marketplace\/item\/(\d+)\/?(?:[?#].*)?$/; +const LISTING_PREFIX = "adp_MarketplacePDPContainerQueryRelayPreloader_"; +const PHOTOS_PREFIX = "adp_MarketplacePDPC2CMediaViewerWithImagesQueryRelayPreloader_"; + +export const FacebookMarketplace: Platform< + "FacebookMarketplace", + MarketplaceData, + MarketplaceMeta +> = { + type: "FacebookMarketplace", + async match(url) { + return url.match(MATCH_RE)?.[1] ?? null; + }, + async fetch(id, env) { + if (!/^\d+$/.test(id)) { + throw { code: 400, message: "Invalid Marketplace listing ID" }; + } + const response = await fetch(`https://www.facebook.com/marketplace/item/${id}/`, { + headers: { + "User-Agent": env?.EMBED_USER_AGENT ?? "", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + }, + }); + if (!response.ok) throw { code: response.status, message: response.statusText }; + + const $ = cheerio.load(await response.text()); + let listing: MarketplaceListing | undefined; + let photos: MarketplacePhotos | undefined; + let mapTemplate: string | undefined; + + for (const element of $('script[type="application/json"][data-sjs]').toArray()) { + const text = $(element).text(); + if ( + !text.includes(LISTING_PREFIX) && + !text.includes(PHOTOS_PREFIX) && + !text.includes('"TilesMapConfig"') + ) + continue; + let data; + try { + data = JSON.parse(text); + } catch { + throw { code: 500, message: "Failed to parse Marketplace data" }; + } + for (const requireItem of data.require ?? []) { + if (requireItem[0] !== "ScheduledServerJS") continue; + for (const payload of requireItem[3] ?? []) { + const box = payload.__bbox; + for (const definition of box?.define ?? []) { + if (definition[0] === "TilesMapConfig") { + mapTemplate = definition[2]?.STATIC_MAP_URL_TEMPLATE; + } + } + for (const item of box?.require ?? []) { + if (item[0] !== "RelayPrefetchedStreamCache") continue; + const [key, value] = item[3]; + const target = + value?.__bbox?.result?.data?.viewer?.marketplace_product_details_page?.target; + if (target?.id !== id) continue; + if (key.startsWith(LISTING_PREFIX)) { + // SAFETY: this preloader's target matches the requested listing ID and observed listing shape. + listing = target as MarketplaceListing; + } + if (key.startsWith(PHOTOS_PREFIX)) { + // SAFETY: this preloader's target matches the requested listing ID and observed photo shape. + photos = target as MarketplacePhotos; + } + } + } + } + } + if (!listing || !photos) { + throw { code: 500, message: "Marketplace listing data is unavailable" }; + } + return { listing, photos, mapTemplate }; + }, + async transform({ listing, photos, mapTemplate }) { + const media = photos.listing_photos.map((photo) => ({ + url: photo.image.uri, + type: "photo", + description: photo.accessibility_caption, + })); + let map: string | undefined; + if (mapTemplate && listing.location) { + const url = new URL(mapTemplate); + url.searchParams.set("size", "600x180"); + url.searchParams.set("scale", "2"); + url.searchParams.set("zoom", "11"); + url.searchParams.set("language", "en_US"); + url.searchParams.set("center", `${listing.location.latitude},${listing.location.longitude}`); + url.searchParams.set( + "circle", + `weight:2|color:0x4D6AA47f|fillcolor:0x4D6AA41c|${listing.location.latitude},${listing.location.longitude}|2k`, + ); + map = url.href; + } + return { + platform: this.type, + author: { name: listing.marketplace_listing_title, avatar: media[0]?.url ?? "" }, + url: `https://www.facebook.com/marketplace/item/${listing.id}/`, + text: listing.redacted_description?.text, + timestamp: listing.creation_time, + price: + listing.formatted_price?.text ?? listing.listing_price?.formatted_amount_zeros_stripped, + location: listing.location_text?.text, + map, + media, + }; + }, +}; diff --git a/packages/platforms/src/platforms/index.ts b/packages/platforms/src/platforms/index.ts index 4565694..019dee1 100644 --- a/packages/platforms/src/platforms/index.ts +++ b/packages/platforms/src/platforms/index.ts @@ -3,3 +3,4 @@ export { Bluesky } from "./bluesky"; export { Instagram } from "./instagram"; export { TikTok } from "./tiktok"; export { Threads } from "./threads"; +export { FacebookMarketplace } from "./facebook-marketplace"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e672fff..46df1b7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,6 +144,9 @@ importers: hono: specifier: ^4.12.15 version: 4.12.15 + posthog-node: + specifier: ^5.51.6 + version: 5.51.6 redis: specifier: ^6.0.0 version: 6.0.0(@opentelemetry/api@1.9.1) @@ -1986,6 +1989,12 @@ packages: '@poppinss/exception@1.2.3': resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + '@posthog/core@1.50.5': + resolution: {integrity: sha512-afEchuShDaVIoxAIj76kDZQ1DhfesDmgfVp+mtTzsA3wlc8DF5uoz8YjuTjnxOicWkpP5HCDqYidK/1kT125Cg==} + + '@posthog/types@1.409.0': + resolution: {integrity: sha512-239umoaZVb2GBaXeEyJpwFvjhrrChJH8NHCwiao23EBSu3NA6EN0MTMoaHSmMEf4yjiXEFFoYJA6FjJAXF5HGA==} + '@protobufjs/aspromise@1.1.2': resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} @@ -4480,6 +4489,15 @@ packages: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} + posthog-node@5.51.6: + resolution: {integrity: sha512-r+Ge3p0OnOcOWeTnvKZmAtDwECeIoyNTyxBxuZc8wM6RHCQ9j2Xrhogrf39HSq1Y2gIOpWLqIDRmWDcu7xmqcA==} + engines: {node: ^20.20.0 || >=22.22.0} + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true + prettier@2.8.8: resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} engines: {node: '>=10.13.0'} @@ -6789,6 +6807,12 @@ snapshots: '@poppinss/exception@1.2.3': {} + '@posthog/core@1.50.5': + dependencies: + '@posthog/types': 1.409.0 + + '@posthog/types@1.409.0': {} + '@protobufjs/aspromise@1.1.2': {} '@protobufjs/base64@1.1.2': {} @@ -9585,6 +9609,10 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + posthog-node@5.51.6: + dependencies: + '@posthog/core': 1.50.5 + prettier@2.8.8: {} prettier@3.8.3: {}