Add e2e tests for link-shortener resolution

Alias the nginx fixture container as search.app (a known shortener) and
allow-list it for the crawler, with /shortlink and /shortlink-image
redirect endpoints. Covers both the HTML crawl path (stored url resolved)
and the asset path (sourceUrl resolved).

Author: bcamarneiro

packages/e2e_tests/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       MEILI_ADDR: http://meilisearch:7700
       BROWSER_WEB_URL: http://chrome:9222
       CRAWLER_NUM_WORKERS: 6
-      CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx
+      CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx,search.app
       CRAWLER_VIDEO_DOWNLOAD: "true"
       CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE: -1
       OPENAI_API_KEY: aimock-test-key
@@ -44,6 +44,11 @@ services:
   nginx:
     image: nginx:alpine
     restart: unless-stopped
+    networks:
+      # Alias so the crawler can reach a "known link shortener" host in tests.
+      default:
+        aliases:
+          - search.app
     volumes:
       - ./setup/html:/usr/share/nginx/html
       - ./setup/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro

packages/e2e_tests/setup/nginx/default.conf

@@ -7,6 +7,15 @@ server {
         return 302 http://127.0.0.1:80/hello.html;
     }
 
+    # Simulate a link-shortener redirect (reached via the search.app alias).
+    location = /shortlink {
+        return 302 http://nginx:80/hello.html;
+    }
+
+    location = /shortlink-image {
+        return 302 http://nginx:80/image.png;
+    }
+
     location / {
         try_files $uri =404;
     }

packages/e2e_tests/tests/workers/crawler.test.ts

@@ -143,6 +143,56 @@ describe("Crawler Tests", () => {
     expect(bookmark.content.htmlContent).toBeNull();
   });
 
+  it("resolves a known link-shortener URL to its destination", async () => {
+    // search.app is aliased to the nginx container and allow-listed as a
+    // known shortener, so /shortlink 302s to the real hello.html page.
+    let { data: bookmark } = await client.POST("/bookmarks", {
+      body: {
+        type: "link",
+        url: "http://search.app/shortlink",
+      },
+    });
+    assert(bookmark);
+
+    await waitUntil(async () => {
+      const data = await getBookmark(bookmark!.id);
+      assert(data);
+      assert(data.content.type === "link");
+      return data.content.crawledAt !== null;
+    }, "Shortened bookmark is crawled");
+
+    bookmark = await getBookmark(bookmark.id);
+    assert(bookmark && bookmark.content.type === "link");
+    // The stored URL should now be the resolved destination, not the short link.
+    expect(bookmark.content.url).not.toContain("search.app");
+    expect(bookmark.content.url).toContain("hello.html");
+    expect(bookmark.content.htmlContent).toContain("Hello World");
+  });
+
+  it("resolves a shortener that points directly to an asset", async () => {
+    let { data: bookmark } = await client.POST("/bookmarks", {
+      body: {
+        type: "link",
+        url: "http://search.app/shortlink-image",
+      },
+    });
+    assert(bookmark);
+
+    await waitUntil(async () => {
+      const data = await getBookmark(bookmark!.id);
+      assert(data);
+      return data.content.type === "asset";
+    }, "Shortened asset bookmark is converted to an image");
+
+    bookmark = await getBookmark(bookmark.id);
+    assert(bookmark && bookmark.content.type === "asset");
+    expect(bookmark.content.assetType).toBe("image");
+    // sourceUrl should be the resolved destination, not the short link.
+    expect(bookmark.content.sourceUrl).not.toContain("search.app");
+    expect(bookmark.content.sourceUrl).toContain("image.png");
+    expect(bookmark.content.fileName).toBe("image.png");
+  });
+
   it("image lings jobs be converted into images", async () => {
     let { data: bookmark } = await client.POST("/bookmarks", {
       body: {