Restore YouTube viewing progress estimates

Author: skyhong2002

README.md

@@ -103,6 +103,7 @@ above use webp (≈10× smaller than the SVG), so this page loads fast.
 - `GET /api/wrapped/{year}.json` — machine-readable annual recap
 - `POST /api/ingest/events` — authenticated private-ingest service (JSON)
 - `POST /api/ingest/youtube/capture` — dedicated-token Chrome viewing capture
+- `POST /api/ingest/youtube/progress` — explicit history progress import
 - `POST /mcp` — stateless MCP Streamable HTTP endpoint
 - `GET /healthz` — freshness-aware health check (`healthy`, `degraded`, or `unhealthy`)
 
@@ -174,9 +175,15 @@ five non-ad playback seconds, and sends cumulative measured watch time
 every 30 seconds. Failed requests remain in `chrome.storage.local`, retry with
 bounded exponential backoff, and survive browser restarts. One session is
 idempotently updated server-side, so retries never add duplicate watch events.
-The dedicated token can only access the capture endpoint. Search terms,
-playback position, cookies, and browsing outside `www.youtube.com` are not
-collected.
+The dedicated token can only access the capture and progress endpoints. Search
+terms, cookies, and browsing outside `www.youtube.com` are not collected.
+
+The popup's explicit **Import history** action opens the signed-in YouTube
+History page and imports only video ids plus the resume/progress and duration
+shown there. It does not import titles, channels, history timestamps, searches,
+or event order. These rows remain private and contribute only aggregate
+content-coverage statistics. Automatic viewing capture does not collect
+playback position.
 
 ### Behind a reverse proxy
 

chrome-extension/background.js

@@ -3,6 +3,7 @@ import { DEFAULT_ENDPOINT, mergeQueue, retryDelayMs } from './queue.js';
 const QUEUE_KEY = 'captureQueue';
 const SETTINGS_KEY = 'captureSettings';
 const STATUS_KEY = 'captureStatus';
+const HISTORY_STATUS_KEY = 'historyImportStatus';
 let queueMutation = Promise.resolve();
 let flushPromise = null;
 
@@ -143,6 +144,122 @@ function flushQueue() {
   return flushPromise;
 }
 
+function progressEndpoint(endpoint) {
+  return endpoint.replace(/\/capture$/, '/progress');
+}
+
+async function sendProgressBatch(payload) {
+  const config = await settings();
+  if (!config.enabled || !config.token) throw new Error('Capture token is not configured');
+  let lastError = null;
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      const response = await fetch(progressEndpoint(config.endpoint), {
+        method: 'POST',
+        headers: {
+          authorization: `Bearer ${config.token}`,
+          'content-type': 'application/json',
+        },
+        body: JSON.stringify(payload),
+      });
+      if (response.ok) return await response.json();
+      const body = await response.json().catch(() => ({}));
+      throw new Error(body?.error || `Progress import failed: HTTP ${response.status}`);
+    } catch (error) {
+      lastError = error;
+      if (attempt < 2) await new Promise((resolve) => setTimeout(resolve, 1000 * (2 ** attempt)));
+    }
+  }
+  throw lastError;
+}
+
+async function historyStatus(patch) {
+  const stored = await chrome.storage.local.get(HISTORY_STATUS_KEY);
+  await chrome.storage.local.set({
+    [HISTORY_STATUS_KEY]: {
+      ...(stored[HISTORY_STATUS_KEY] ?? {}),
+      ...patch,
+    },
+  });
+}
+
+async function waitForTab(tabId) {
+  const current = await chrome.tabs.get(tabId);
+  if (current.status === 'complete') return;
+  await new Promise((resolve) => {
+    const listener = (updatedId, changeInfo) => {
+      if (updatedId !== tabId || changeInfo.status !== 'complete') return;
+      chrome.tabs.onUpdated.removeListener(listener);
+      resolve();
+    };
+    chrome.tabs.onUpdated.addListener(listener);
+  });
+}
+
+async function startHistoryImport() {
+  const config = await settings();
+  if (!config.enabled || !config.token) throw new Error('Capture token is not configured');
+  const stored = await chrome.storage.local.get(HISTORY_STATUS_KEY);
+  if (stored[HISTORY_STATUS_KEY]?.state === 'running') {
+    throw new Error('A history import is already running');
+  }
+  const scanId = crypto.randomUUID();
+  const observedAt = new Date().toISOString();
+  const tab = await chrome.tabs.create({
+    active: true,
+    url: 'https://www.youtube.com/feed/history',
+  });
+  if (!tab.id) throw new Error('Could not open YouTube History');
+  await historyStatus({
+    state: 'running',
+    scanId,
+    observedAt,
+    tabId: tab.id,
+    videos: 0,
+    pass: 0,
+    lastError: '',
+  });
+  void (async () => {
+    try {
+      await waitForTab(tab.id);
+      const result = await chrome.tabs.sendMessage(tab.id, {
+        type: 'start-history-import',
+        scanId,
+        observedAt,
+      });
+      if (!result?.ok) throw new Error(result?.error || 'YouTube History import failed');
+      await historyStatus({
+        state: 'complete',
+        videos: result.videos,
+        completedAt: new Date().toISOString(),
+        lastError: '',
+      });
+    } catch (error) {
+      const storedStatus = await chrome.storage.local.get(HISTORY_STATUS_KEY);
+      if (storedStatus[HISTORY_STATUS_KEY]?.state === 'cancelled') return;
+      await historyStatus({
+        state: 'error',
+        completedAt: new Date().toISOString(),
+        lastError: error instanceof Error ? error.message : String(error),
+      });
+    }
+  })();
+  return { scanId, observedAt, tabId: tab.id };
+}
+
+async function cancelHistoryImport() {
+  const stored = await chrome.storage.local.get(HISTORY_STATUS_KEY);
+  const status = stored[HISTORY_STATUS_KEY] ?? {};
+  if (status.tabId) {
+    await chrome.tabs.sendMessage(status.tabId, { type: 'cancel-history-import' }).catch(() => {});
+  }
+  await historyStatus({
+    state: 'cancelled',
+    completedAt: new Date().toISOString(),
+    lastError: '',
+  });
+}
+
 chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
   if (message?.type === 'capture' && message.payload) {
     enqueue(message.payload)
@@ -156,6 +273,39 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
       .catch((error) => sendResponse({ ok: false, error: String(error) }));
     return true;
   }
+  if (message?.type === 'history-import-start') {
+    startHistoryImport()
+      .then((result) => sendResponse({ ok: true, ...result }))
+      .catch((error) => {
+        sendResponse({ ok: false, error: error instanceof Error ? error.message : String(error) });
+      });
+    return true;
+  }
+  if (message?.type === 'history-import-cancel') {
+    cancelHistoryImport()
+      .then(() => sendResponse({ ok: true }))
+      .catch((error) => sendResponse({ ok: false, error: String(error) }));
+    return true;
+  }
+  if (message?.type === 'history-progress-batch' && message.payload) {
+    sendProgressBatch(message.payload)
+      .then((result) => sendResponse({ ok: true, result }))
+      .catch((error) => sendResponse({
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+      }));
+    return true;
+  }
+  if (message?.type === 'history-import-progress') {
+    historyStatus({
+      state: 'running',
+      videos: message.videos,
+      pass: message.pass,
+    })
+      .then(() => sendResponse({ ok: true }))
+      .catch((error) => sendResponse({ ok: false, error: String(error) }));
+    return true;
+  }
   return false;
 });
 

chrome-extension/content.js

@@ -6,6 +6,7 @@
   let boundVideo = null;
   let lastTickAt = performance.now();
   let lastMediaTime = 0;
+  let historyImportCancelled = false;
 
   function videoIdFromLocation() {
     const url = new URL(location.href);
@@ -158,6 +159,70 @@
     if (!video.paused) ensureSession(video);
   }
 
+  function wait(milliseconds) {
+    return new Promise((resolve) => window.setTimeout(resolve, milliseconds));
+  }
+
+  function progressFingerprint(item) {
+    return [
+      item.progressPercent ?? '',
+      item.resumeSeconds ?? '',
+      item.durationSeconds ?? '',
+    ].join(':');
+  }
+
+  async function sendProgressBatch(scanId, observedAt, items, complete = false) {
+    const response = await chrome.runtime.sendMessage({
+      type: 'history-progress-batch',
+      payload: { scanId, observedAt, items, complete },
+    });
+    if (!response?.ok) throw new Error(response?.error || 'Progress batch was rejected');
+    return response;
+  }
+
+  async function runHistoryImport(scanId, observedAt) {
+    if (location.pathname !== '/feed/history') {
+      throw new Error('History import must run on the YouTube History page');
+    }
+    historyImportCancelled = false;
+    const sent = new Map();
+    let unchangedPasses = 0;
+    let lastHeight = 0;
+    let lastItems = 0;
+    for (let pass = 0; pass < 900; pass++) {
+      if (historyImportCancelled) throw new Error('History import cancelled');
+      const items = globalThis.infovoreYoutubeHistory.collectProgress();
+      const changed = items.filter((item) => {
+        const fingerprint = progressFingerprint(item);
+        if (sent.get(item.videoId) === fingerprint) return false;
+        sent.set(item.videoId, fingerprint);
+        return true;
+      });
+      for (let index = 0; index < changed.length; index += 250) {
+        await sendProgressBatch(scanId, observedAt, changed.slice(index, index + 250));
+      }
+      await chrome.runtime.sendMessage({
+        type: 'history-import-progress',
+        scanId,
+        videos: sent.size,
+        pass,
+      });
+      const height = document.documentElement.scrollHeight;
+      if (height === lastHeight && items.length === lastItems && changed.length === 0) {
+        unchangedPasses++;
+      } else {
+        unchangedPasses = 0;
+      }
+      if (unchangedPasses >= 5) break;
+      lastHeight = height;
+      lastItems = items.length;
+      window.scrollTo({ top: height, behavior: 'instant' });
+      await wait(700);
+    }
+    await sendProgressBatch(scanId, observedAt, [], true);
+    return sent.size;
+  }
+
   function handleNavigation() {
     const videoId = videoIdFromLocation();
     if (state && state.videoId !== videoId) {
@@ -174,6 +239,22 @@
     lastMediaTime = boundVideo?.currentTime ?? 0;
   });
   window.addEventListener('pagehide', () => flush(true));
+  chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
+    if (message?.type === 'start-history-import') {
+      runHistoryImport(message.scanId, message.observedAt)
+        .then((videos) => sendResponse({ ok: true, videos }))
+        .catch((error) => sendResponse({
+          ok: false,
+          error: error instanceof Error ? error.message : String(error),
+        }));
+      return true;
+    }
+    if (message?.type === 'cancel-history-import') {
+      historyImportCancelled = true;
+      sendResponse({ ok: true });
+    }
+    return false;
+  });
   new MutationObserver(bindVideo).observe(document.documentElement, {
     childList: true,
     subtree: true,

chrome-extension/history.js

@@ -0,0 +1,74 @@
+(() => {
+  function parseDurationText(value) {
+    const parts = String(value ?? '').trim().split(':').map(Number);
+    if (parts.length < 2 || parts.length > 3 || parts.some(Number.isNaN)) return null;
+    return parts.reduce((total, part) => total * 60 + part, 0);
+  }
+
+  function progressFromLockup(root) {
+    const link = root.querySelector('h3 a[href*="/watch?v="], a[href*="/shorts/"]');
+    if (!link) return null;
+    const url = new URL(link.href, location.origin);
+    const videoId = url.searchParams.get('v') || url.pathname.match(/^\/shorts\/([^/?]+)/)?.[1];
+    if (!/^[A-Za-z0-9_-]{11}$/.test(videoId ?? '')) return null;
+    const durationText = [...root.querySelectorAll('.ytBadgeShapeText')]
+      .map((element) => element.textContent?.trim() ?? '')
+      .find((text) => /^(?:\d+:){1,2}\d{2}$/.test(text));
+    const durationSeconds = parseDurationText(durationText);
+    const progressStyle = root.querySelector(
+      '.ytThumbnailOverlayProgressBarHostWatchedProgressBarSegment'
+    )?.getAttribute('style') ?? '';
+    const progressMatch = progressStyle.match(/width:\s*([\d.]+)%/);
+    const progressPercent = progressMatch
+      ? Math.max(0, Math.min(100, Number(progressMatch[1])))
+      : null;
+    const resumeLink = root.querySelector('a[href*="/watch?v="][href*="t="]');
+    const resumeUrl = resumeLink ? new URL(resumeLink.href, location.origin) : url;
+    const resumeValue = resumeUrl.searchParams.get('t')?.replace(/s$/, '') ?? '';
+    const resumeSeconds = /^\d+$/.test(resumeValue) ? Number(resumeValue) : null;
+    if (progressPercent === null && resumeSeconds === null) return null;
+    return {
+      videoId,
+      progressPercent,
+      resumeSeconds: durationSeconds === null || resumeSeconds === null
+        ? resumeSeconds
+        : Math.min(durationSeconds, resumeSeconds),
+      durationSeconds,
+    };
+  }
+
+  function mergeProgress(current, incoming) {
+    if (!current) return incoming;
+    const progressPercent = current.progressPercent === null
+      ? incoming.progressPercent
+      : incoming.progressPercent === null
+        ? current.progressPercent
+        : Math.max(current.progressPercent, incoming.progressPercent);
+    const resumeSeconds = current.resumeSeconds === null
+      ? incoming.resumeSeconds
+      : incoming.resumeSeconds === null
+        ? current.resumeSeconds
+        : Math.max(current.resumeSeconds, incoming.resumeSeconds);
+    return {
+      videoId: current.videoId,
+      progressPercent,
+      resumeSeconds,
+      durationSeconds: current.durationSeconds ?? incoming.durationSeconds,
+    };
+  }
+
+  function collectProgress(documentRoot = document) {
+    const items = new Map();
+    for (const root of documentRoot.querySelectorAll('yt-lockup-view-model')) {
+      const item = progressFromLockup(root);
+      if (item) items.set(item.videoId, mergeProgress(items.get(item.videoId), item));
+    }
+    return [...items.values()];
+  }
+
+  globalThis.infovoreYoutubeHistory = {
+    collectProgress,
+    mergeProgress,
+    parseDurationText,
+  };
+})();

chrome-extension/manifest.json

@@ -1,8 +1,8 @@
 {
   "manifest_version": 3,
   "name": "infovore YouTube Capture",
-  "version": "1.0.0",
-  "description": "Privately records YouTube viewing sessions and measured watch time to infovore.",
+  "version": "1.1.0",
+  "description": "Privately records measured YouTube watch time and imports saved viewing progress to infovore.",
   "minimum_chrome_version": "120",
   "permissions": [
     "alarms",
@@ -26,6 +26,7 @@
         "https://www.youtube.com/*"
       ],
       "js": [
+        "history.js",
         "content.js"
       ],
       "run_at": "document_idle"