Resolve known link-shortener URLs to their destination

When a bookmark is saved as a Google short/share link (search.app,
share.google), the crawler already follows the redirect to fetch the
real content, but the stored URL kept the opaque short link.

Overwrite the stored URL with the resolved destination, scoped to a
known-shortener allowlist and guarded by a safe-scheme check, so
ordinary redirects (http->https, tracking-param strips) never rewrite
the user's URL. Applied across all three crawl outcomes:
- HTML pages: via the browser's final URL (Phase-1 bookmarkLinks write)
- direct PDF/image links: the probe now returns its redirect target so
  the asset-bookmark path stores the resolved sourceUrl
- downstream video jobs: crawlAndParseUrl returns the effective URL

Covered by unit tests (packages/shared/utils/url.test.ts) and e2e tests
that alias the nginx fixture as search.app and assert the resolved url
and sourceUrl.

Closes #2235

Author: bcamarneiro

apps/workers/workers/crawler/crawlAndParse.ts

@@ -35,6 +35,7 @@ import serverConfig from "@karakeep/shared/config";
 import logger from "@karakeep/shared/logger";
 import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
 import type { ZReaderViewReason } from "@karakeep/shared/types/bookmarks";
+import { resolveShortenedBookmarkUrl } from "@karakeep/shared/utils/url";
 
 import type { ParseSubprocessOutput } from "../utils/parseHtmlSubprocessIpc";
 import {
@@ -51,7 +52,11 @@ import {
 } from "./assetStorage";
 import { crawlPage } from "./crawlPage";
 import { runParseSubprocess } from "./parseSubprocess";
-import { redactUrlCredentials, shouldRetryCrawlStatusCode } from "./utils";
+import {
+  redactUrlCredentials,
+  shouldRetryCrawlStatusCode,
+  truncateUrl,
+} from "./utils";
 
 const tracer = getTracer("@karakeep/workers");
 
@@ -176,7 +181,7 @@ export interface CrawlAndParseUrlArgs {
  */
 export async function crawlAndParseUrl(
   args: CrawlAndParseUrlArgs,
-): Promise<() => Promise<void>> {
+): Promise<{ runArchival: () => Promise<void>; effectiveUrl: string }> {
   const {
     url,
     userId,
@@ -329,12 +334,24 @@ export async function crawlAndParseUrl(
         }
       };
 
+      // Replace a known shortener (search.app/share.google) with its resolved
+      // destination so the bookmark reflects the real URL. See issue #2235.
+      const resolvedUrl = resolveShortenedBookmarkUrl(url, browserUrl);
+      if (resolvedUrl) {
+        logger.info(
+          `[Crawler][${jobId}] Resolved shortened URL "${truncateUrl(
+            url,
+          )}" to "${truncateUrl(resolvedUrl)}". Updating the bookmark URL.`,
+        );
+      }
+
       // Phase 1: Write metadata immediately for fast user feedback.
       // Content and asset storage happen later and can be slow (banner
       // image download, screenshot/pdf upload, etc.).
       await db
         .update(bookmarkLinks)
         .set({
+          ...(resolvedUrl ? { url: resolvedUrl } : {}),
           title: meta.title,
           description: meta.description,
           // Don't store data URIs as they're not valid URLs and are usually quite large
@@ -489,7 +506,7 @@ export async function crawlAndParseUrl(
       // Delete the old assets if any
       await Promise.all(assetDeletionTasks);
 
-      return async () => {
+      const runArchival = async () => {
         if (
           !precrawledArchiveAssetId &&
           (serverConfig.crawler.fullPageArchive || archiveFullPage)
@@ -531,6 +548,10 @@ export async function crawlAndParseUrl(
           }
         }
       };
+
+      // effectiveUrl reflects any shortener resolution so downstream jobs
+      // (e.g. video) use the real destination, not the opaque short link.
+      return { runArchival, effectiveUrl: resolvedUrl ?? url };
     },
   );
 }

apps/workers/workers/crawler/probe.ts

@@ -48,6 +48,11 @@ const PROBE_HTML_CONTENT_TYPES = new Set<string>([
 
 export interface UrlProbeResult {
   contentType: string | null;
+  /**
+   * The URL after any redirects the probe followed. Callers use this to
+   * resolve link-shortener URLs to their real destination.
+   */
+  finalUrl: string;
   /**
    * Resolves with the page's preview metadata (or null). The extraction runs
    * in the background so it can overlap with the browser crawl — await it
@@ -120,7 +125,11 @@ export async function getContentTypeAndMetadata(
         logger.error(
           `[Crawler][${jobId}] Failed to determine the content-type for the url ${truncateUrl(url)}: ${e}`,
         );
-        return { contentType: null, metadata: Promise.resolve(null) };
+        return {
+          contentType: null,
+          finalUrl: url,
+          metadata: Promise.resolve(null),
+        };
       }
       setSpanAttributes({
         "crawler.getContentType.statusCode": response.status,
@@ -135,7 +144,11 @@ export async function getContentTypeAndMetadata(
       );
 
       if (!contentType || !PROBE_HTML_CONTENT_TYPES.has(contentType)) {
-        return { contentType, metadata: Promise.resolve(null) };
+        return {
+          contentType,
+          finalUrl: response.url,
+          metadata: Promise.resolve(null),
+        };
       }
 
       // A previous run already extracted and stored this page's metadata; the
@@ -147,7 +160,11 @@ export async function getContentTypeAndMetadata(
         addLogFields<"crawlerWorker.run">({
           "crawler.probe.metadata": "reused_stored",
         });
-        return { contentType, metadata: Promise.resolve(null) };
+        return {
+          contentType,
+          finalUrl: response.url,
+          metadata: Promise.resolve(null),
+        };
       }
 
       // A blocked/retryable status usually means a challenge or error page
@@ -159,7 +176,11 @@ export async function getContentTypeAndMetadata(
         addLogFields<"crawlerWorker.run">({
           "crawler.probe.metadata": "blocked_status",
         });
-        return { contentType, metadata: Promise.resolve(null) };
+        return {
+          contentType,
+          finalUrl: response.url,
+          metadata: Promise.resolve(null),
+        };
       }
 
       // The response is an HTML page: parse its metadata from the body we've
@@ -205,7 +226,7 @@ export async function getContentTypeAndMetadata(
           return null;
         }
       })();
-      return { contentType, metadata };
+      return { contentType, finalUrl: response.url, metadata };
     },
   );
 }

apps/workers/workers/crawlerWorker.ts

@@ -39,6 +39,7 @@ import {
 } from "@karakeep/shared/queueing";
 import { getRateLimitClient } from "@karakeep/shared/ratelimiting";
 import { tryCatch } from "@karakeep/shared/tryCatch";
+import { resolveShortenedBookmarkUrl } from "@karakeep/shared/utils/url";
 import { WebhooksService } from "@karakeep/trpc/models/webhooks.service";
 
 import {
@@ -393,20 +394,32 @@ async function runCrawler(
   // it — reload it from the bookmark row instead.
   const reuseStoredProbeMetadata =
     job.runNumber > 0 && probeMetadataAt !== null;
-  const { contentType, metadata: probeMetadata }: UrlProbeResult =
-    precrawledArchiveAssetId
-      ? { contentType: ASSET_TYPES.TEXT_HTML, metadata: Promise.resolve(null) }
-      : await getContentTypeAndMetadata(url, jobId, job.abortSignal, runProxy, {
-          skipMetadataExtraction: reuseStoredProbeMetadata,
-        });
+  const {
+    contentType,
+    finalUrl,
+    metadata: probeMetadata,
+  }: UrlProbeResult = precrawledArchiveAssetId
+    ? {
+        contentType: ASSET_TYPES.TEXT_HTML,
+        finalUrl: url,
+        metadata: Promise.resolve(null),
+      }
+    : await getContentTypeAndMetadata(url, jobId, job.abortSignal, runProxy, {
+        skipMetadataExtraction: reuseStoredProbeMetadata,
+      });
   job.abortSignal.throwIfAborted();
 
+  // For asset bookmarks the shortener is resolved via the content-type probe's
+  // redirect target (there's no browser crawl to derive it from). The HTML path
+  // resolves separately inside crawlAndParseUrl via the browser URL.
+  const assetUrl = resolveShortenedBookmarkUrl(url, finalUrl) ?? url;
+
   // Link bookmarks get transformed into asset bookmarks if they point to a supported asset instead of a webpage
   const isPdf = contentType === ASSET_TYPES.APPLICATION_PDF;
 
   if (isPdf) {
     await handleAsAssetBookmark(
-      url,
+      assetUrl,
       "pdf",
       userId,
       jobId,
@@ -420,7 +433,7 @@ async function runCrawler(
     SUPPORTED_UPLOAD_ASSET_TYPES.has(contentType)
   ) {
     await handleAsAssetBookmark(
-      url,
+      assetUrl,
       "image",
       userId,
       jobId,
@@ -454,7 +467,7 @@ async function runCrawler(
           }
           return metadata;
         });
-    const archivalLogic = await crawlAndParseUrl({
+    const { runArchival, effectiveUrl } = await crawlAndParseUrl({
       url,
       userId,
       jobId,
@@ -475,10 +488,10 @@ async function runCrawler(
       probeMetadataPromise,
     });
 
-    await enqueuePostCrawlJobs(job, bookmarkId, userId, url);
+    await enqueuePostCrawlJobs(job, bookmarkId, userId, effectiveUrl);
 
     // Do the archival as a separate last step as it has the potential for failure
-    await archivalLogic();
+    await runArchival();
   }
 
   // Record the latency from bookmark creation to crawl completion.

packages/e2e_tests/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       MEILI_ADDR: http://meilisearch:7700
       BROWSER_WEB_URL: http://chrome:9222
       CRAWLER_NUM_WORKERS: 6
-      CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx
+      CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx,search.app
       CRAWLER_STORE_PDF: "true"
       CRAWLER_VIDEO_DOWNLOAD: "true"
       CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE: -1
@@ -48,6 +48,11 @@ services:
   nginx:
     image: nginx:alpine
     restart: unless-stopped
+    networks:
+      # Alias so the crawler can reach a "known link shortener" host in tests.
+      default:
+        aliases:
+          - search.app
     volumes:
       - ./setup/html:/usr/share/nginx/html
       - ./setup/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro

packages/e2e_tests/setup/nginx/default.conf

@@ -7,6 +7,15 @@ server {
         return 302 http://127.0.0.1:80/hello.html;
     }
 
+    # Simulate a link-shortener redirect (reached via the search.app alias).
+    location = /shortlink {
+        return 302 http://nginx:80/hello.html;
+    }
+
+    location = /shortlink-image {
+        return 302 http://nginx:80/image.png;
+    }
+
     location / {
         try_files $uri =404;
     }

packages/e2e_tests/tests/workers/crawler.test.ts

@@ -144,6 +144,56 @@ describe("Crawler Tests", () => {
     expect(bookmark.content.htmlContent).toBeNull();
   });
 
+  it("resolves a known link-shortener URL to its destination", async () => {
+    // search.app is aliased to the nginx container and allow-listed as a
+    // known shortener, so /shortlink 302s to the real hello.html page.
+    let { data: bookmark } = await client.POST("/bookmarks", {
+      body: {
+        type: "link",
+        url: "http://search.app/shortlink",
+      },
+    });
+    assert(bookmark);
+
+    await waitUntil(async () => {
+      const data = await getBookmark(bookmark!.id);
+      assert(data);
+      assert(data.content.type === "link");
+      return data.content.crawledAt !== null;
+    }, "Shortened bookmark is crawled");
+
+    bookmark = await getBookmark(bookmark.id);
+    assert(bookmark && bookmark.content.type === "link");
+    // The stored URL should now be the resolved destination, not the short link.
+    expect(bookmark.content.url).not.toContain("search.app");
+    expect(bookmark.content.url).toContain("hello.html");
+    expect(bookmark.content.htmlContent).toContain("Hello World");
+  });
+
+  it("resolves a shortener that points directly to an asset", async () => {
+    let { data: bookmark } = await client.POST("/bookmarks", {
+      body: {
+        type: "link",
+        url: "http://search.app/shortlink-image",
+      },
+    });
+    assert(bookmark);
+
+    await waitUntil(async () => {
+      const data = await getBookmark(bookmark!.id);
+      assert(data);
+      return data.content.type === "asset";
+    }, "Shortened asset bookmark is converted to an image");
+
+    bookmark = await getBookmark(bookmark.id);
+    assert(bookmark && bookmark.content.type === "asset");
+    expect(bookmark.content.assetType).toBe("image");
+    // sourceUrl should be the resolved destination, not the short link.
+    expect(bookmark.content.sourceUrl).not.toContain("search.app");
+    expect(bookmark.content.sourceUrl).toContain("image.png");
+    expect(bookmark.content.fileName).toBe("image.png");
+  });
+
   it("image lings jobs be converted into images", async () => {
     let { data: bookmark } = await client.POST("/bookmarks", {
       body: {