feat: add Facebook Marketplace embeds

apps/bot/compose.yaml

@@ -27,6 +27,8 @@ services:
       EMBEDLY_API_DOMAIN: ${EMBEDLY_API_DOMAIN}
       EMBEDLY_AUTH_SECRET: ${EMBEDLY_AUTH_SECRET}
       EMBED_USER_AGENT: ${EMBED_USER_AGENT}
+      POSTHOG_API_KEY: ${POSTHOG_API_KEY:-}
+      POSTHOG_HOST: ${POSTHOG_HOST:-https://us.i.posthog.com}
       MESSAGE_CACHE_TTL_SECONDS: ${MESSAGE_CACHE_TTL_SECONDS:-86400}
       OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
       OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-embedly-bot}

apps/bot/package.json

@@ -25,6 +25,7 @@
     "@sapphire/framework": "^5.5.0",
     "discord.js": "~14.26.3",
     "hono": "^4.12.15",
+    "posthog-node": "^5.51.6",
     "redis": "^6.0.0"
   },
   "devDependencies": {

apps/bot/src/lib/builder.ts

@@ -59,7 +59,11 @@ function buildMediaEmbed(media: NormalizedPost["media"], spoiler?: EmbedFlags["S
 
   const gallery = new MediaGalleryBuilder();
   gallery.addItems(
-    media.slice(0, MAX_GALLERY_ITEMS).map((m) => ({ media: { url: m.url }, spoiler })),
+    media.slice(0, MAX_GALLERY_ITEMS).map((m) => ({
+      media: { url: m.url },
+      description: m.description?.slice(0, 1024) || undefined,
+      spoiler,
+    })),
   );
 
   return gallery.toJSON();
@@ -79,14 +83,22 @@ function addPostComponents(embed: ContainerBuilder, post: PostData, headingPrefi
     post.author.handle && post.author.url
       ? `(${hyperlink(`@${post.author.handle}`, post.author.url)})`
       : "";
-  const authorHeading = [headingPrefix, post.author.name, authorHandle].filter(Boolean).join(" ");
+  const authorName =
+    post.platform === "FacebookMarketplace"
+      ? truncate(escapeMarkdown(post.author.name), 250)
+      : post.author.name;
+  const authorHeading = [headingPrefix, authorName, authorHandle].filter(Boolean).join(" ");
 
   embed.addSectionComponents((section) => {
     section
       .setThumbnailAccessory((thumbnail) => thumbnail.setURL(post.author.avatar))
       .addTextDisplayComponents((author) =>
         author.setContent(heading(authorHeading, HeadingLevel.Three)),
       );
+    if (post.platform === "FacebookMarketplace" && post.price) {
+      const price = post.price;
+      section.addTextDisplayComponents((display) => display.setContent(escapeMarkdown(price)));
+    }
     if (post.text && post.text.length > 0) {
       section.addTextDisplayComponents((text) =>
         text.setContent(
@@ -121,6 +133,31 @@ function addPostComponents(embed: ContainerBuilder, post: PostData, headingPrefi
   if (post.media.length > 0) {
     embed.addMediaGalleryComponents(buildMediaEmbed(post.media)!);
   }
+  if (post.platform === "FacebookMarketplace") {
+    const { location, map } = post;
+    if (location) {
+      embed.addTextDisplayComponents((display) =>
+        display.setContent(`${escapeMarkdown(location)} · Location is approximate`),
+      );
+    }
+    if (map) {
+      embed.addMediaGalleryComponents((gallery) =>
+        gallery.addItems({
+          media: { url: map },
+          description: (location
+            ? `Map showing approximate listing location in ${location}.`
+            : "Map showing approximate listing location."
+          ).slice(0, 1024),
+        }),
+      );
+    }
+    embed.addTextDisplayComponents((footer) =>
+      footer.setContent(
+        `${time(post.timestamp, TimestampStyles.RelativeTime)} • ${hyperlink("View on Facebook Marketplace", post.url)}`,
+      ),
+    );
+    return;
+  }
   embed
     .addSeparatorComponents((sep) => sep.setDivider(false).setSpacing(SeparatorSpacingSize.Small))
     .addTextDisplayComponents(
@@ -140,6 +177,7 @@ function addPostComponents(embed: ContainerBuilder, post: PostData, headingPrefi
 }
 
 export function buildEmbed(post: PostData, flags?: Partial<EmbedFlags>) {
+  if (post.platform === "FacebookMarketplace" && post.media.length === 0) return null;
   if (flags?.MediaOnly) {
     return buildMediaEmbed(post.media, flags?.Spoiler);
   }

apps/bot/src/lib/client.ts

@@ -4,6 +4,7 @@ import type { AppType } from "@embedly/api";
 import { container, SapphireClient } from "@sapphire/framework";
 import { ActivityType, GatewayIntentBits, Partials, PresenceUpdateStatus } from "discord.js";
 import { hc } from "hono/client";
+import { PostHog } from "posthog-node";
 
 import { MessageCache } from "./messageCache";
 export class EmbedlyClient extends SapphireClient {
@@ -33,11 +34,17 @@ export class EmbedlyClient extends SapphireClient {
   public override async login(token?: string) {
     container.api = hc<AppType>(process.env.EMBEDLY_API_DOMAIN ?? "http://localhost:8787");
     container.messageCache = await MessageCache.connect();
+    if (process.env.POSTHOG_API_KEY) {
+      container.posthog = new PostHog(process.env.POSTHOG_API_KEY, {
+        host: process.env.POSTHOG_HOST,
+      });
+    }
     return super.login(token);
   }
 
   public override async destroy() {
     await container.messageCache?.close();
+    await container.posthog?.shutdown();
     return super.destroy();
   }
 }
@@ -46,5 +53,6 @@ declare module "@sapphire/framework" {
   interface Container {
     api: ReturnType<typeof hc<AppType>>;
     messageCache: MessageCache;
+    posthog?: PostHog;
   }
 }

apps/bot/src/lib/handleUrls.ts

@@ -186,7 +186,7 @@ export async function handleUrls(
         guild_id: interaction!.guildId ?? "dm",
         user_id: interaction!.user.id,
       };
-  const matches = (
+  let matches = (
     await Promise.all(
       urls.map(async (request, requestIndex) => {
         if (options.updateTargets && !options.updateTargets.has(requestIndex)) return null;
@@ -217,6 +217,28 @@ export async function handleUrls(
     await reactToFailure();
   }
 
+  if (matches.some((match) => match.platform === "FacebookMarketplace")) {
+    let hasAccess = false;
+    if (container.posthog) {
+      try {
+        const flags = await container.posthog.evaluateFlags(matchContext.user_id, {
+          flagKeys: ["facebook-marketplace"],
+        });
+        hasAccess = flags.getFlag("facebook-marketplace") === true;
+      } catch (error) {
+        container.logger.warn(
+          formatLog("warn", EmbedlyErrors.FeatureFlagFailed, {
+            ...matchContext,
+            ...getErrorContext(error),
+          }),
+        );
+      }
+    }
+    if (!hasAccess) {
+      matches = matches.filter((match) => match.platform !== "FacebookMarketplace");
+    }
+  }
+
   if (interaction && matches.length === 0) {
     const requestId = `${embedSource}:${interaction.id}`;
     const error =

packages/logging/src/main.ts

@@ -39,6 +39,12 @@ export function defineError(event: EmbedlyErrorEvent) {
 }
 
 export const EmbedlyErrors = {
+  FeatureFlagFailed: defineError({
+    type: "feature_flag.failed",
+    title: "Access check failed.",
+    detail: "Could not check Marketplace access.",
+    status: 502,
+  }),
   NoUrlsFound: defineError({
     type: "embed.no_urls_found",
     title: "No URLs Found.",

packages/platforms/src/platforms/facebook-marketplace.d.ts

@@ -0,0 +1,30 @@
+export interface MarketplaceListing {
+  id: string;
+  marketplace_listing_title: string;
+  redacted_description?: { text: string } | null;
+  creation_time: number;
+  formatted_price?: { text: string } | null;
+  listing_price?: { formatted_amount_zeros_stripped?: string } | null;
+  location_text?: { text: string } | null;
+  location?: { latitude: number; longitude: number } | null;
+}
+
+export interface MarketplacePhotos {
+  id: string;
+  listing_photos: Array<{
+    image: { uri: string };
+    accessibility_caption?: string;
+  }>;
+}
+
+export interface MarketplaceData {
+  listing: MarketplaceListing;
+  photos: MarketplacePhotos;
+  mapTemplate?: string;
+}
+
+export interface MarketplaceMeta {
+  price?: string;
+  location?: string;
+  map?: string;
+}

packages/platforms/src/platforms/facebook-marketplace.ts

@@ -0,0 +1,126 @@
+import * as cheerio from "cheerio";
+
+import type { Platform } from "../types";
+import type {
+  MarketplaceData,
+  MarketplaceListing,
+  MarketplaceMeta,
+  MarketplacePhotos,
+} from "./facebook-marketplace.d";
+
+const MATCH_RE =
+  /^(?:https?:\/\/)?(?:www\.|m\.)?facebook\.com\/marketplace\/item\/(\d+)\/?(?:[?#].*)?$/;
+const LISTING_PREFIX = "adp_MarketplacePDPContainerQueryRelayPreloader_";
+const PHOTOS_PREFIX = "adp_MarketplacePDPC2CMediaViewerWithImagesQueryRelayPreloader_";
+
+export const FacebookMarketplace: Platform<
+  "FacebookMarketplace",
+  MarketplaceData,
+  MarketplaceMeta
+> = {
+  type: "FacebookMarketplace",
+  async match(url) {
+    return url.match(MATCH_RE)?.[1] ?? null;
+  },
+  async fetch(id, env) {
+    if (!/^\d+$/.test(id)) {
+      throw { code: 400, message: "Invalid Marketplace listing ID" };
+    }
+    const response = await fetch(`https://www.facebook.com/marketplace/item/${id}/`, {
+      headers: {
+        "User-Agent": env?.EMBED_USER_AGENT ?? "",
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
+      },
+    });
+    if (!response.ok) throw { code: response.status, message: response.statusText };
+
+    const $ = cheerio.load(await response.text());
+    let listing: MarketplaceListing | undefined;
+    let photos: MarketplacePhotos | undefined;
+    let mapTemplate: string | undefined;
+
+    for (const element of $('script[type="application/json"][data-sjs]').toArray()) {
+      const text = $(element).text();
+      if (
+        !text.includes(LISTING_PREFIX) &&
+        !text.includes(PHOTOS_PREFIX) &&
+        !text.includes('"TilesMapConfig"')
+      )
+        continue;
+      let data;
+      try {
+        data = JSON.parse(text);
+      } catch {
+        throw { code: 500, message: "Failed to parse Marketplace data" };
+      }
+      for (const requireItem of data.require ?? []) {
+        if (requireItem[0] !== "ScheduledServerJS") continue;
+        for (const payload of requireItem[3] ?? []) {
+          const box = payload.__bbox;
+          for (const definition of box?.define ?? []) {
+            if (definition[0] === "TilesMapConfig") {
+              mapTemplate = definition[2]?.STATIC_MAP_URL_TEMPLATE;
+            }
+          }
+          for (const item of box?.require ?? []) {
+            if (item[0] !== "RelayPrefetchedStreamCache") continue;
+            const [key, value] = item[3];
+            const target =
+              value?.__bbox?.result?.data?.viewer?.marketplace_product_details_page?.target;
+            if (target?.id !== id) continue;
+            if (key.startsWith(LISTING_PREFIX)) {
+              // SAFETY: this preloader's target matches the requested listing ID and observed listing shape.
+              listing = target as MarketplaceListing;
+            }
+            if (key.startsWith(PHOTOS_PREFIX)) {
+              // SAFETY: this preloader's target matches the requested listing ID and observed photo shape.
+              photos = target as MarketplacePhotos;
+            }
+          }
+        }
+      }
+    }
+    if (!listing || !photos) {
+      throw { code: 500, message: "Marketplace listing data is unavailable" };
+    }
+    return { listing, photos, mapTemplate };
+  },
+  async transform({ listing, photos, mapTemplate }) {
+    const media = photos.listing_photos.map((photo) => ({
+      url: photo.image.uri,
+      type: "photo",
+      description: photo.accessibility_caption,
+    }));
+    let map: string | undefined;
+    if (mapTemplate && listing.location) {
+      const url = new URL(mapTemplate);
+      url.searchParams.set("size", "600x180");
+      url.searchParams.set("scale", "2");
+      url.searchParams.set("zoom", "11");
+      url.searchParams.set("language", "en_US");
+      url.searchParams.set("center", `${listing.location.latitude},${listing.location.longitude}`);
+      url.searchParams.set(
+        "circle",
+        `weight:2|color:0x4D6AA47f|fillcolor:0x4D6AA41c|${listing.location.latitude},${listing.location.longitude}|2k`,
+      );
+      map = url.href;
+    }
+    return {
+      platform: this.type,
+      author: { name: listing.marketplace_listing_title, avatar: media[0]?.url ?? "" },
+      url: `https://www.facebook.com/marketplace/item/${listing.id}/`,
+      text: listing.redacted_description?.text,
+      timestamp: listing.creation_time,
+      price:
+        listing.formatted_price?.text ?? listing.listing_price?.formatted_amount_zeros_stripped,
+      location: listing.location_text?.text,
+      map,
+      media,
+    };
+  },
+};

packages/platforms/src/platforms/index.ts

@@ -3,3 +3,4 @@ export { Bluesky } from "./bluesky";
 export { Instagram } from "./instagram";
 export { TikTok } from "./tiktok";
 export { Threads } from "./threads";
+export { FacebookMarketplace } from "./facebook-marketplace";