feat: sample storage and report page

Author: sarod

browser-extension/entrypoints/background/index.ts

@@ -1,3 +1,4 @@
+import { storePost } from "../shared/storage/posts-storage";
 import { getCurrentTab } from "../shared/utils/getCurrentTab";
 import { scrapTab as scrapPostFromTab } from "./scraping/scrap-tab";
 
@@ -24,16 +25,9 @@ async function scrapActiveTab() {
   const tab = await getCurrentTab();
 
   if (tab) {
+    console.log("Scraping post from active tab");
     const socialNetworkPost = await scrapPostFromTab(tab);
-
-    if (socialNetworkPost.comments.length > 0) {
-      const screenshotDataUrl: string =
-        socialNetworkPost.comments[0].screenshotDataUrl;
-      browser.downloads.download({
-        url: screenshotDataUrl, // The object URL can be used as download URL
-        filename: "screenshot.png",
-        //...
-      });
-    }
+    console.log("Storing post to local storage");
+    await storePost(socialNetworkPost);
   }
 }

browser-extension/entrypoints/background/model.ts

@@ -3,7 +3,7 @@ import {
   ExtensionTransport,
   Browser as PuppeteerBrowser,
 } from "puppeteer-core/lib/esm/puppeteer/puppeteer-core-browser.js";
-import type { Post } from "../model";
+import type { Post } from "../../shared/model/post";
 
 export abstract class BaseScraper {
   browser?: PuppeteerBrowser;

browser-extension/entrypoints/background/scraping/base-scraper.ts

@@ -1,13 +1,13 @@
-import { SocialNetworkName } from "@/entrypoints/shared/social-network-url";
+import { SocialNetworkName } from "@/entrypoints/shared/model/social-network-name";
 import { BaseScraper } from "./base-scraper";
 import { InstagramScraper } from "./instagram/instagram-scraper";
 import { YoutubeScraper } from "./youtube/youtube-scraper";
 
 export function createScraper(sn: SocialNetworkName): BaseScraper {
   switch (sn) {
-    case "youtube":
+    case "YOUTUBE":
       return new YoutubeScraper();
-    case "instagram":
+    case "INSTAGRAM":
       return new InstagramScraper();
   }
 }

browser-extension/entrypoints/background/scraping/create-scraper.ts

@@ -1,12 +1,26 @@
 import { ElementHandle } from "puppeteer-core/lib/esm/puppeteer/puppeteer-core-browser.js";
 import { BaseScraper } from "../base-scraper";
-import { type Autor, type Post, type Comment } from "../../model";
+import {
+  type Author,
+  type Post,
+  type Comment,
+} from "../../../shared/model/post";
+import { parseSocialNetworkUrl } from "@/entrypoints/shared/social-network-url";
+import { currentIsoDate } from "../utils/current-iso-date";
 
 //TODO: gérer le scroll et le chargement des commentaires
 //TODO: gérer le scraping des réponses aux commentaires
 export class InstagramScraper extends BaseScraper {
   private INSTAGRAM_URL = "https://www.instagram.com/";
 
+  extractPostId(url: string): string {
+    const parsed = parseSocialNetworkUrl(url);
+    if (!parsed) {
+      throw new Error("Unexpected");
+    }
+    return parsed.postId;
+  }
+
   async scrapTab(tab: Browser.tabs.Tab): Promise<Post> {
     const page = await this.getBrowserPageFromTab(tab);
 
@@ -64,18 +78,22 @@ export class InstagramScraper extends BaseScraper {
       );
       // gérer ici le ce scraping des réponses aux commentaires
     }
+
     return {
+      postId: this.extractPostId(tab.url!),
+      socialNetwork: "INSTAGRAM",
       url: tab.url!,
       author: auteur,
-      publishedAt: new Date(date_publication),
+      scrapTimestamp: new Date().toISOString(),
+      publishedAt: new Date(date_publication).toISOString(),
       text: texte_publication,
       comments: await Promise.all(commentaires),
     };
   }
 
   private async get_auteur_from_span(
     span_element: ElementHandle<Element>
-  ): Promise<Autor> {
+  ): Promise<Author> {
     const auteur_elem = (await span_element.$("::-p-xpath(.//a)"))!;
     const auteur_href = (await auteur_elem.$eval("::-p-xpath(.)", (node) =>
       node.getAttribute("href")
@@ -86,7 +104,7 @@ export class InstagramScraper extends BaseScraper {
     ))!;
     return {
       name: auteur_name,
-      accountHref: this.urlJoin(this.INSTAGRAM_URL, auteur_href),
+      accountUrl: this.urlJoin(this.INSTAGRAM_URL, auteur_href),
     };
   }
 
@@ -109,12 +127,19 @@ export class InstagramScraper extends BaseScraper {
     } catch (_) {
       date_commentaire = undefined;
     }
+
     const screenshot = await comment_element.screenshot({ encoding: "base64" });
+    const screenshotDate = currentIsoDate();
     return {
-      autor: auteur,
-      text: await base_1.$eval("::-p-xpath(.)", (node) => node.textContent!)!,
-      commentDate: date_commentaire,
+      author: auteur,
+      commentText: await base_1.$eval(
+        "::-p-xpath(.)",
+        (node) => node.textContent!
+      )!,
+      commentDate: date_commentaire?.toISOString(),
+      commentDateRelative: false,
       screenshotDataUrl: screenshot,
+      screenshotDate,
       replies: [],
     };
   }

browser-extension/entrypoints/background/scraping/instagram/instagram-scraper.ts

@@ -1,5 +1,5 @@
 import { parseSocialNetworkUrl } from "../../shared/social-network-url";
-import { Post } from "../model";
+import { Post } from "../../shared/model/post";
 import { createScraper } from "./create-scraper";
 
 export async function scrapTab(

browser-extension/entrypoints/background/scraping/scrap-tab.ts

@@ -0,0 +1,3 @@
+export function currentIsoDate(): string {
+  return new Date(Date.now()).toISOString();
+}

browser-extension/entrypoints/background/scraping/utils/current-iso-date.ts

@@ -1,11 +1,77 @@
-import { ElementHandle } from "puppeteer-core/lib/esm/puppeteer/puppeteer-core-browser.js";
+import {
+  Page,
+  ElementHandle,
+} from "puppeteer-core/lib/esm/puppeteer/puppeteer-core-browser.js";
 import { BaseScraper } from "../base-scraper";
-import { type Post, type Comment } from "../../model";
+import { type Post, type Comment, Author } from "../../../shared/model/post";
+import { parseSocialNetworkUrl } from "@/entrypoints/shared/social-network-url";
+import { currentIsoDate } from "../utils/current-iso-date";
 
 export class YoutubeScraper extends BaseScraper {
+  extractPostId(url: string): string {
+    const parsed = parseSocialNetworkUrl(url);
+    if (!parsed) {
+      throw new Error("Unexpected");
+    }
+    return parsed.postId;
+  }
+
   async scrapTab(tab: Browser.tabs.Tab): Promise<Post> {
+    const postUrl = tab.url!;
+    const postId = this.extractPostId(postUrl);
     const page = await this.getBrowserPageFromTab(tab);
+    const scrapTimestamp = currentIsoDate();
+
+    const author = await this.scrapPostAuthor(page);
+    const { publishedAt, publishedAtRelative, text } =
+      await this.scrapPostPublishedAtAndText(page);
+    const comments: Comment[] = await this.scrapPostComments(page);
+
+    return {
+      postId: postId,
+      socialNetwork: "YOUTUBE",
+      scrapTimestamp: scrapTimestamp,
+
+      url: postUrl,
+      author: author,
+      publishedAt: publishedAt,
+      publishedAtRelative: publishedAtRelative,
+      text: text,
+      comments: comments,
+    };
+  }
+
+  private async scrapPostPublishedAtAndText(postPage: Page): Promise<{
+    publishedAt: string | undefined;
+    publishedAtRelative: boolean | undefined;
+    text: string;
+  }> {
+    const snippetText = await innertText(
+      (await postPage.$("#description #snippet-text"))!
+    );
+    const publishedAt =
+      (await ariaLabel((await postPage.$("#description #date-text"))!)) ??
+      undefined;
+    return {
+      publishedAt: publishedAt,
+      publishedAtRelative: true,
+      text: snippetText,
+    };
+  }
+
+  private async scrapPostAuthor(postPage: Page): Promise<Author> {
+    const postOwnerEl = (await postPage.$("#owner"))!;
+    const channelNameEl = (await postOwnerEl.$("#channel-name"))!;
+    const link = (await channelNameEl.$("a"))!;
+    const channelName = await innertText(link);
+    const channelUrl = await anchorHref(link);
+    return {
+      name: channelName,
+      accountUrl: channelUrl,
+    };
+  }
 
+  private async scrapPostComments(page: Page): Promise<Comment[]> {
     const commentsSectionHandle: ElementHandle = (await page.$("#comments"))!;
     commentsSectionHandle.scrollIntoView();
 
@@ -21,7 +87,6 @@ export class YoutubeScraper extends BaseScraper {
     // await loadAllTopLevelComments(commentsSectionHandle);
     // await expandReplies(commentsSectionHandle)
     // await expandLongComments(commentsSectionHandle)
-
     await commentsSectionHandle.waitForSelector("#comment-container");
     const commentContainers = await commentsSectionHandle.$$(
       "#comment-container"
@@ -43,40 +108,30 @@ export class YoutubeScraper extends BaseScraper {
             commentTextHandle
           )
         )?.trim();
-        const screenshotData = this.uintArraySCreenshotToBase64Url(
+        const screenshotData = this.uintArrayScreenshotToBase64Url(
           await commentContainer.screenshot()
         );
+        const screenshotDate = currentIsoDate();
+
         const comment: Comment = {
-          autor: {
+          author: {
             name: commentAuthor,
             // TODO extract href
           },
-          text: commentText,
+          commentText: commentText,
           screenshotDataUrl: screenshotData,
-          commentDate: new Date(Date.now()),
+          screenshotDate,
+          // TODO extrat comment relative date
           // TODO capture replies
           replies: [],
         };
         return comment;
       })
     );
-
-    console.log("comments", comments);
-
-    return {
-      url: tab.url!,
-      author: {
-        // TODO capture post author
-        name: "Unknown",
-        accountHref: "",
-      },
-      // TODO capture post text
-      publishedAt: new Date(Date.now()),
-      comments: comments,
-    };
+    return comments;
   }
 
-  private uintArraySCreenshotToBase64Url(pngData: Uint8Array): string {
+  private uintArrayScreenshotToBase64Url(pngData: Uint8Array): string {
     let binary = "";
     const len = pngData.byteLength;
     for (let i = 0; i < len; i++) {
@@ -86,3 +141,17 @@ export class YoutubeScraper extends BaseScraper {
     return "data:image/png;base64," + base64;
   }
 }
+
+async function innertText(element: ElementHandle): Promise<string> {
+  return await element.evaluate((e) => (e as HTMLElement).innerText, element);
+}
+
+async function anchorHref(
+  element: ElementHandle<HTMLAnchorElement>
+): Promise<string> {
+  return await element.evaluate((e) => e.href, element);
+}
+
+async function ariaLabel(element: ElementHandle): Promise<string | null> {
+  return await element.evaluate((e) => (e as HTMLElement).ariaLabel, element);
+}