Fixing waste of tokens on browser open, force new rules on extension update, automatic reload after one-off double spend (#16)

This PR fixes an important and a minor bugs in the extensions, and improves UX:

# Bug

Before this patch, when opening the browser with the extension enabled, one token per endpoint (about 50) would be discarded automatically. This is now resolved.

# Minor bug

The way that the header rules were being applied for the /html/* search pages, was causing the wrong header being sent due to confusion between /html and /html/*.

# UX Feature 1

After extension update, new header rules were not being automatically applied and the user had to manually disable and re-enable PP mode. This is now done automatically.

# UX Feature 2

Sometimes due to a race-condition or a header rule update, the user may encounter an error message about a token having been already spent. With this update, we check if this error had already happened in the prior minute. If not, then the page displaying the error is reloaded using a new token, which likely resolves the problem. If the error had just happened (eg, if after the page reload the problem is not resolved), the new error page is left on display.

Author: fvirdia

src/background.js

@@ -82,10 +82,11 @@ chrome.runtime.onMessage.addListener(async (message, sender, sendResponse) => {
 
 chrome.runtime.onInstalled.addListener(async (details) => {
   // in install, enable the extension and fetch some tokens
+  console.log("onInstalled")
   if (details.reason == "install") {
     await chrome.storage.local.set({ 'enabled': true });
-    await setEnabled();
-    await onStart();
+    // onStart (which will be executed in approximately 1 second)
+    // will pick up 'enabled': true and use it to enable the extension
   } else if (details.reason == "update") {
     // if extension was enabled before receiving the oupdate,
     // force a disable-enable cycle in order to apply any changes
@@ -116,18 +117,35 @@ async function onStart() {
   if (VERBOSE) {
     debug_log(`onStart: ${new Date().toISOString().match(/(\d{2}:){2}\d{2}/)[0]}`);
   }
-  // reset enabled/disabled status depending on what the user left it as
-  const { enabled } = await browser.storage.local.get({ 'enabled': false });
-  if (enabled) {
+  console.log(`onStart: ${new Date().toISOString().match(/(\d{2}:){2}\d{2}/)[0]}`);
+  // The browser is being started up, or the extension being enabled.
+  // When an extension is disabled or the browser is turned off,
+  // the declarativeNetRequest rules used to send tokens to Kagi are removed.
+  // However, there is no "onBrowserClose" or "onExtensionDisable" listener
+  // allowing us to unload the tokens that were loaded in those rules.
+  // If we don't do anything, those tokens will be lost.
+  // Hence, we have to recover them from localStorage. An easy way that does not
+  // require writing any new code is to trigger the code used when the PP mode toggle
+  // is disabled.
+  const was_enabled = (await browser.storage.local.get({ 'enabled': false }))['enabled'];
+  // emulate PP mode being disabled
+  await browser.storage.local.set({ 'enabled': false })
+  await setDisabled();
+  // if the extension was last enabled, simulate the PP mode toggle being enabled
+  if (was_enabled) {
+    await browser.storage.local.set({ 'enabled': true })
     await setEnabled();
-  } else {
-    await setDisabled();
   }
+  // refresh extension icon
+  await update_extension_icon(was_enabled);
   // when coming online, send status to Kagi Search extension
   await sendPPModeStatus();
 }
 
-browser.runtime.onStartup.addListener(onStart)
+browser.runtime.onStartup.addListener(async () => {
+  // dummy operation to make sure background.js is run
+  await browser.runtime.getPlatformInfo();
+})
 
 // -- keep background.js alive (to address non-persistency of manifest V3 extensions)
 
@@ -146,3 +164,11 @@ chrome.alarms.onAlarm.addListener(async (info) => {
     await browser.runtime.getPlatformInfo();
   }
 });
+
+// init the extension
+(async () => {
+  console.log(`background.js: ${new Date().toISOString().match(/(\d{2}:){2}\d{2}/)[0]}`);
+  setTimeout(async () => {
+    await onStart();
+  }, 1000)
+})()

src/popup/enable_toggle.js

@@ -1,3 +1,7 @@
+import {
+  update_extension_icon
+} from '../scripts/icon.js'
+
 const enabled_checkbox = document.querySelector("#kagipp-enabled")
 const status_message_indicator = document.querySelector("#status-message-indicator")
 
@@ -23,6 +27,7 @@ async function is_enabled() {
   const { enabled } = await browser.storage.local.get({ 'enabled': false })
   enabled_checkbox.checked = enabled;
   await update_indicator_opacity(enabled);
+  await update_extension_icon(enabled);
 }
 
 async function set_enabled() {

src/popup/popup.js

@@ -135,7 +135,7 @@ if (enabled_checkbox) {
 
 (async () => {
   // try reading right away
-  await is_enabled()
+  await is_enabled();
   // add CSS transition style
   setTimeout(() => {
     let sheet = window.document.styleSheets[0];
@@ -145,7 +145,7 @@ if (enabled_checkbox) {
   setTimeout(() => {
     setIntervalAndFire(async () => {
       await is_enabled();
-    });
+    }, 1000);
   }, 1000)
 })()
 

src/scripts/anonymization.js

@@ -57,6 +57,8 @@ export const ACCEPT_QUICK_ANSWER_OFFSET = 152;
 export const ACCEPT_QUICK_ANSWER_DOC_OFFSET = 154;
 export const ACCEPT_TRANSLATE_JSON_OFFSET = 156;
 export const ACCEPT_TRANSLATE_TURSNTILE_OFFSET = 158;
+export const KAGI_HTML_SLASH_REDIRECT = 180;
+export const ONION_HTML_SLASH_REDIRECT = 182;
 
 const INITIAL_HTTP_AUTHORIZATION_ID = 200
 const INITIAL_NO_TOKEN_REDIRECT_ID = 400

src/scripts/config.js

@@ -40,21 +40,21 @@ const REDEMPTION_SERVICES = [
 ]
 let REDEMPTION_ENDPOINTS = [
     `${SCHEME}://${DOMAIN_PORT}/|`,
-    `${SCHEME}://${DOMAIN_PORT}/html`,
+    `${SCHEME}://${DOMAIN_PORT}/html|`,
     `${SCHEME}://${DOMAIN_PORT}/settings`,
     `${SCHEME}://${DOMAIN_PORT}/api/quick_settings/landing`,
     `${SCHEME}://${DOMAIN_PORT}/mother/context`,
     `${SCHEME}://${DOMAIN_PORT}/mother/summarize_document`,
     `${ONION_SCHEME}://${ONION_DOMAIN_PORT}/|`,
-    `${ONION_SCHEME}://${ONION_DOMAIN_PORT}/html`,
+    `${ONION_SCHEME}://${ONION_DOMAIN_PORT}/html|`,
     `${ONION_SCHEME}://${ONION_DOMAIN_PORT}/settings`,
     `${ONION_SCHEME}://${ONION_DOMAIN_PORT}/api/quick_settings/landing`,
     `${ONION_SCHEME}://${ONION_DOMAIN_PORT}/mother/context`,
     `${ONION_SCHEME}://${ONION_DOMAIN_PORT}/mother/summarize_document`
 ]
 if (STAGING) {
     REDEMPTION_ENDPOINTS.push(`${SCHEME}://stage.${DOMAIN_PORT}/|`)
-    REDEMPTION_ENDPOINTS.push(`${SCHEME}://stage.${DOMAIN_PORT}/html`)
+    REDEMPTION_ENDPOINTS.push(`${SCHEME}://stage.${DOMAIN_PORT}/html|`)
     REDEMPTION_ENDPOINTS.push(`${SCHEME}://stage.${DOMAIN_PORT}/settings`)
     REDEMPTION_ENDPOINTS.push(`${SCHEME}://stage.${DOMAIN_PORT}/api/quick_settings/landing`)
     REDEMPTION_ENDPOINTS.push(`${SCHEME}://stage.${DOMAIN_PORT}/mother/context`)
@@ -108,7 +108,7 @@ export {
 let WEBREQUEST_REDEMPTION_ENDPOINTS = []
 for (let i = 0; i < REDEMPTION_ENDPOINTS.length; i++) {
     let endpoint = REDEMPTION_ENDPOINTS[i];
-    if (endpoint.endsWith('/|')) {
+    if (endpoint.endsWith('|')) {
          // webRequest does not recognize urlFilter's |
          // so we remove the trailing |
         endpoint = endpoint.substring(0, endpoint.length-1)

src/scripts/generation_and_redemption.js

@@ -181,7 +181,7 @@ async function setPPHeadersListener(details) {
     const url = new URL(details.url);
     const scheme_domain_port = url.origin;
     const pathname = url.pathname; // comes with a leading /
-    const endpoint = (pathname == "/") ? `${scheme_domain_port}${pathname}|` : `${scheme_domain_port}${pathname}`;
+    const endpoint = (pathname == "/" || pathname.endsWith('/html')) ? `${scheme_domain_port}${pathname}|` : `${scheme_domain_port}${pathname}`;
     await setPPHeaders(endpoint);
 }
 

src/scripts/headers.js

@@ -24,6 +24,8 @@ import {
     ACCEPT_QUICK_ANSWER_OFFSET,
     ACCEPT_QUICK_ANSWER_DOC_OFFSET,
     ACCEPT_TRANSLATE_TURSNTILE_OFFSET,
+    KAGI_HTML_SLASH_REDIRECT,
+    ONION_HTML_SLASH_REDIRECT,
     ANONYMIZING_RULES_OFFSET,
     ANONYMIZING_RULESET,
     REFERER_RULES_OFFSET,
@@ -303,6 +305,54 @@ async function unsetLocaRedirectorHeader() {
     });
 }
 
+async function setHTMLIndexRedirector() {
+    if (VERBOSE) {
+        debug_log(`setHTMLIndexRedirector`)
+    }
+    const rules = {
+        addRules: [{
+            id: KAGI_HTML_SLASH_REDIRECT,
+            priority: 1,
+            condition: {
+                urlFilter: `||${DOMAIN_PORT}/html/|`,
+                resourceTypes: ["main_frame", "sub_frame"]
+            },
+            action: {
+                type: "redirect",
+                redirect: {
+                    url: `https://${DOMAIN_PORT}/html`
+                }
+            }
+        },{
+            id: ONION_HTML_SLASH_REDIRECT,
+            priority: 1,
+            condition: {
+                urlFilter: `||${ONION_DOMAIN_PORT}/html/|`,
+                resourceTypes: ["main_frame", "sub_frame"]
+            },
+            action: {
+                type: "redirect",
+                redirect: {
+                    url: `http://${ONION_DOMAIN_PORT}/html`
+                }
+            }
+        }],
+        removeRuleIds: [KAGI_HTML_SLASH_REDIRECT, ONION_HTML_SLASH_REDIRECT]
+    };
+
+    await browser.declarativeNetRequest.updateDynamicRules(rules);
+}
+
+async function unsetHTMLIndexRedirector() {
+    if (VERBOSE) {
+        debug_log("unsetHTMLIndexRedirector");
+    }
+    await chrome.declarativeNetRequest.updateDynamicRules({
+        addRules: [],
+        removeRuleIds: [KAGI_HTML_SLASH_REDIRECT, ONION_HTML_SLASH_REDIRECT]
+    });
+}
+
 async function setAuthorizationHeader(endpoint, token_tuple) {
     if (VERBOSE) {
         debug_log(`[${endpoint}] ${token_tuple[0].substring(0, 32)}`)
@@ -376,6 +426,8 @@ export {
     setAntiFingerprintingRules,
     unsetAntiFingerprintingRules,
     setNoTokensRedirect,
+    setHTMLIndexRedirector,
+    unsetHTMLIndexRedirector,
     setAuthorizationHeader,
     unsetAuthorizationHeader,
     setLocaRedirectorHeader,

src/scripts/icon.js

@@ -1,7 +1,6 @@
 async function update_extension_icon(enabled) {
   if (typeof enabled === "undefined") {
-    const { _enabled } = await browser.storage.local.get({ 'enabled': false })
-    enabled = _enabled;
+    enabled = (await browser.storage.local.get({ 'enabled': false }))['enabled'];
   }
   const path = enabled ? "enabled" : "disabled";
   await chrome.action.setIcon({

src/scripts/toggle.js

@@ -12,7 +12,9 @@ import {
     unsetAntiFingerprintingRules,
     setLocaRedirectorHeader,
     unsetLocaRedirectorHeader,
-    selfRemovingUnsetRefererHeadersListener
+    selfRemovingUnsetRefererHeadersListener,
+    setHTMLIndexRedirector,
+    unsetHTMLIndexRedirector
 } from './headers.js'
 
 import {
@@ -53,7 +55,7 @@ async function checkingDoubleSpendListener(details) {
     const url = new URL(details.url);
     const scheme_domain_port = url.origin;
     const pathname = url.pathname; // comes with a leading /
-    const endpoint = (pathname == "/") ? `${scheme_domain_port}${pathname}|` : `${scheme_domain_port}${pathname}`;
+    const endpoint = (pathname == "/" || pathname.endsWith('/html')) ? `${scheme_domain_port}${pathname}|` : `${scheme_domain_port}${pathname}`;
     if (VERBOSE) {
         debug_log(`checkingDoubleSpendListener: ${details.statusCode} ${endpoint}`)
     }
@@ -62,7 +64,42 @@ async function checkingDoubleSpendListener(details) {
         if (VERBOSE) {
             debug_log(`> loading a new token for ${endpoint}`)
         }
+        // a token was double spent, load the next one
         await setPPHeaders(endpoint);
+        /*
+         * The status at this line is:
+         * - an error page is shown (or no results displayed in case of a /socket/ request failing)
+         * - a new token is loaded for the endpoint that last failed
+         * This may be a one-off error, eg due to an update in header rules, or a race condition.
+         * If that's the case, we would rather reload the page now that a new token was set.
+         * If the error has happened repeatedly though, this is not a good strategy,
+         * as it may single out that there is a failing user making repeated queries, and also
+         * loop infinitely.
+         * Our approach here is the following:
+         * - load the time of the last recorded double-spend
+         * - if very recent, then this is a repeated failure, don't reload;
+         *   just show the error telling the user to check the documentation and possibly report the failure
+         * - if the last error is not very recent, then automatically reload the page
+        */
+        const now = (new Date()).getTime(); // unix time in milliseconds
+        const { last_double_spend } = await browser.storage.local.get({ 'last_double_spend' : 0 });
+        const gap = now - last_double_spend;
+        await browser.storage.local.set({ 'last_double_spend' : now }); // update the last_double_spend information
+        if (gap > 60 * 1000) { // if the last seen doublespend was more than 1 minute ago
+            // likely one-off double-spend, reload the current page
+            if (VERBOSE) {
+                debug_log("checkingDoubleSpendListener: one-off double-spend, reloading");
+            }
+            const active_tabs_cur_window =  await chrome.tabs.query({active: true, currentWindow: true});
+            const cur_tab = active_tabs_cur_window[0];
+            if (cur_tab && cur_tab.id && cur_tab.id == details.tabId) {
+                await browser.tabs.reload(details.tabId, { bypassCache: true });
+            }
+        } else {
+            if (VERBOSE) {
+                debug_log("checkingDoubleSpendListener: repeated double-spend, not reloading");
+            }
+        }
     } else if (details.statusCode == 403) {
         // let the user know that their tokens are stale
         // realistically, this should only happen to devs debugging against staging
@@ -96,6 +133,7 @@ async function setEnabled() {
     await setRefererRules();
     await setAntiFingerprintingRules();
     await setLocaRedirectorHeader();
+    await setHTMLIndexRedirector();
     for (let i = 0; i < REDEMPTION_ENDPOINTS.length; i++) {
         let endpoint = REDEMPTION_ENDPOINTS[i];
         await setPPHeaders(endpoint);
@@ -123,6 +161,7 @@ async function setDisabled() {
     }
     await unsetAntiFingerprintingRules();
     await unsetLocaRedirectorHeader();
+    await unsetHTMLIndexRedirector();
     for (let i = 0; i < REDEMPTION_ENDPOINTS.length; i++) {
         let endpoint = REDEMPTION_ENDPOINTS[i];
         await unsetPPHeaders(endpoint);