Make built-in AI extension work with multimodal files

built-in-ai-extension/eslint.config.js

@@ -0,0 +1,33 @@
+import js from '@eslint/js';
+import google from 'eslint-config-google';
+import prettier from 'eslint-config-prettier';
+import globals from 'globals';
+
+export default [
+  js.configs.recommended,
+  {
+    rules: {
+      ...google.rules,
+      ...prettier.rules,
+      'require-jsdoc': 'off',
+      'valid-jsdoc': 'off',
+      'max-len': ['warn', { code: 100, ignoreUrls: true }],
+      'no-unused-vars': 'warn',
+      'no-undef': 'error',
+    },
+  },
+  {
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+      globals: {
+        ...globals.browser,
+        ...globals.webextensions,
+        chrome: 'readonly',
+      },
+    },
+  },
+  {
+    ignores: ['dist/**', 'node_modules/**'],
+  },
+];

built-in-ai-extension/offscreen/offscreen.js

@@ -5,15 +5,61 @@
 
 // offscreen/offscreen.js
 
-let sessions = new Map(); // requestId -> session instance
-let controllers = new Map(); // callId -> AbortController
+const sessions = new Map(); // requestId -> session instance
+const controllers = new Map(); // callId -> AbortController
 
 chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
   if (message.target !== 'offscreen') return;
 
   (async () => {
+    let processedMessage;
     try {
-      if (message.type === 'create-session') {
+      // Recursively find and process blobURLs into Blobs
+      const processBlobURLs = async (obj) => {
+        if (!obj || typeof obj !== 'object') return obj;
+
+        // More robust identification for the specific bridge descriptor
+        if (obj.__extension_blob_url__) {
+          const url = obj.__extension_blob_url__;
+          try {
+            const response = await fetch(url);
+            const blob = await response.blob();
+            URL.revokeObjectURL(url);
+            return blob;
+          } catch (e) {
+            console.error(`[Offscreen] Failed to fetch blobURL (${url}):`, e);
+            URL.revokeObjectURL(url);
+            throw e;
+          }
+        }
+
+        // Avoid recursing into Blobs, ArrayBuffers, or other special objects
+        const constructorName = obj.constructor?.name;
+        if (
+          constructorName === 'Blob' ||
+          constructorName === 'File' ||
+          constructorName === 'ArrayBuffer' ||
+          ArrayBuffer.isView(obj)
+        ) {
+          return obj;
+        }
+
+        if (Array.isArray(obj)) {
+          for (let i = 0; i < obj.length; i++) {
+            obj[i] = await processBlobURLs(obj[i]);
+          }
+        } else {
+          for (const key of Object.keys(obj)) {
+            obj[key] = await processBlobURLs(obj[key]);
+          }
+        }
+        return obj;
+      };
+
+      processedMessage = await processBlobURLs(message);
+      const { type } = processedMessage;
+
+      if (type === 'create-session') {
         const {
           config,
           options,
@@ -23,16 +69,14 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
           backend,
           senderTabId,
           senderFrameId,
-        } = message;
+        } = processedMessage;
 
         // Apply external configuration to globals for the polyfills to find.
         setupConfigs(backend, config);
 
-        const [promptApiModule] = await Promise.all([
-          import('prompt-api-polyfill'),
-        ]);
+        const promptApiModule = await import('prompt-api-polyfill');
 
-        const ApiClass = getApiClass(promptApiModule, apiType, config);
+        const ApiClass = getApiClass(promptApiModule, apiType, config, backend);
 
         if (!ApiClass) {
           throw new Error(
@@ -101,28 +145,26 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
         } finally {
           if (callId) controllers.delete(callId);
         }
-      } else if (message.type === 'availability') {
-        const { config, backend, apiType, options } = message;
+      } else if (processedMessage.type === 'availability') {
+        const { config, backend, apiType, options } = processedMessage;
 
         setupConfigs(backend, config);
 
-        const [promptApiModule] = await Promise.all([
-          import('prompt-api-polyfill'),
-        ]);
+        const promptApiModule = await import('prompt-api-polyfill');
 
-        const ApiClass = getApiClass(promptApiModule, apiType, config);
+        const ApiClass = getApiClass(promptApiModule, apiType, config, backend);
 
         const result = await ApiClass.availability(options);
         sendResponse({ success: true, result });
-      } else if (message.type === 'clone-session') {
+      } else if (processedMessage.type === 'clone-session') {
         const {
           sourceRequestId,
           requestId,
           callId,
           options,
           senderTabId,
           senderFrameId,
-        } = message;
+        } = processedMessage;
         const sourceSession = sessions.get(sourceRequestId);
         if (!sourceSession) {
           throw new Error(
@@ -174,7 +216,7 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
         } finally {
           if (callId) controllers.delete(callId);
         }
-      } else if (message.type === 'list-models') {
+      } else if (processedMessage.type === 'list-models') {
         const cache = await caches.open('transformers-cache');
         const keys = await cache.keys();
         const models = new Set();
@@ -184,8 +226,8 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
           if (match) models.add(match[1]);
         }
         sendResponse({ success: true, models: Array.from(models) });
-      } else if (message.type === 'delete-model') {
-        const { modelName } = message;
+      } else if (processedMessage.type === 'delete-model') {
+        const { modelName } = processedMessage;
         const cache = await caches.open('transformers-cache');
         const keys = await cache.keys();
         let deletedCount = 0;
@@ -208,27 +250,20 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
         keysToRemove.forEach((k) => localStorage.removeItem(k));
         sendResponse({ success: true, deletedCount });
       } else if (
-        message.type === 'prompt' ||
-        message.type === 'execute' ||
-        message.type === 'append'
+        processedMessage.type === 'prompt' ||
+        processedMessage.type === 'execute' ||
+        processedMessage.type === 'append'
       ) {
-        const {
-          requestId,
-          callId,
-          text,
-          method,
-          options,
-          senderTabId,
-          senderFrameId,
-        } = message;
+        const { requestId, callId, text, method, options } = processedMessage;
+
         const session = sessions.get(requestId);
         if (!session)
           throw new Error('No active session for requestId: ' + requestId);
 
         const controller = new AbortController();
         if (callId) controllers.set(callId, controller);
         try {
-          const result = await session[method || message.type](text, {
+          const result = await session[method || processedMessage.type](text, {
             ...options,
             signal: controller.signal,
           });
@@ -244,8 +279,8 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
           if (callId) controllers.delete(callId);
         }
       } else if (
-        message.type === 'prompt-streaming' ||
-        message.type === 'execute-streaming'
+        processedMessage.type === 'prompt-streaming' ||
+        processedMessage.type === 'execute-streaming'
       ) {
         const {
           requestId,
@@ -255,7 +290,7 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
           options,
           senderTabId,
           senderFrameId,
-        } = message;
+        } = processedMessage;
         const session = sessions.get(requestId);
         if (!session)
           throw new Error('No active session for requestId: ' + requestId);
@@ -323,14 +358,14 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
             contextWindow: session.contextWindow,
           },
         });
-      } else if (message.type === 'destroy-session') {
-        const { requestId } = message;
+      } else if (processedMessage.type === 'destroy-session') {
+        const { requestId } = processedMessage;
         const session = sessions.get(requestId);
         if (session && session.destroy) session.destroy();
         sessions.delete(requestId);
         sendResponse({ success: true });
-      } else if (message.type === 'abort-request') {
-        const { callId } = message;
+      } else if (processedMessage.type === 'abort-request') {
+        const { callId } = processedMessage;
         const controller = controllers.get(callId);
         if (controller) {
           controller.abort();
@@ -340,7 +375,14 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
       }
     } catch (err) {
       console.error('Offscreen execution error:', err);
-      sendResponse({ success: false, error: err.message, name: err.name });
+      // Use message instead of processedMessage here because processedMessage
+      // might be undefined if processBlobURLs failed
+      const apiType = processedMessage?.apiType ?? message.apiType;
+      const errorMsg =
+        apiType && !err.message.includes(apiType)
+          ? `[${apiType}] ${err.message}`
+          : err.message;
+      sendResponse({ success: false, error: errorMsg, name: err.name });
     }
   })();
 
@@ -397,7 +439,7 @@ function setupConfigs(backend, config) {
   }
 }
 
-function getApiClass(promptApiModule, apiType, config) {
+function getApiClass(promptApiModule, apiType, config, backend) {
   const nativeClass = window[apiType || 'LanguageModel'];
 
   const ApiClass =
@@ -406,9 +448,11 @@ function getApiClass(promptApiModule, apiType, config) {
     promptApiModule.default ||
     nativeClass;
 
-  // If forceInjection is false and we have a native class, prefer it
+  // If forceInjection is false and we have a native class, prefer it,
+  // BUT only if the requested backend is 'native'.
   if (
     !config.forceInjection &&
+    backend === 'native' &&
     nativeClass &&
     typeof nativeClass.create === 'function'
   ) {