Skip downloading transformer models if they already exist.

Implement a caching mechanism using a JSON file to track the state of
downloaded models. This avoids redundant downloads during prebuild if the
model configuration has not changed.

A --force flag is also added to allow users to manually bypass the cache
and re-download all models when necessary.

Author: brendandahl

resources/transformers-js/src/download-models.mjs

@@ -48,6 +48,30 @@ const KOKORO_FILES = [
    'tokenizer_config.json',
 ];
 
+const MARGO_MODELS_TO_DOWNLOAD = [
+    {
+        class: 'SiglipVisionModel',
+        dtype: 'bnb4',
+    },
+    {
+        class: 'AutoImageProcessor',
+    },
+    {
+        class: 'SiglipTextModel',
+        dtype: 'bnb4',
+    },
+    {
+        class: 'AutoTokenizer',
+        dtype: 'bnb4',
+    }
+];
+const MARGO_NAME_TO_CLASS = {
+    'SiglipVisionModel': SiglipVisionModel,
+    'AutoImageProcessor': AutoImageProcessor,
+    'SiglipTextModel': SiglipTextModel,
+    'AutoTokenizer': AutoTokenizer,
+};
+
 function getHuggingFaceUrl(repo, filename, branch = 'main') {
     if(filename.endsWith('.onnx')) {
         return `https://huggingface.co/${repo}/resolve/${branch}/onnx/${filename}`;
@@ -56,6 +80,28 @@ function getHuggingFaceUrl(repo, filename, branch = 'main') {
 }
 
 async function downloadModels() {
+    const CACHE_FILE = path.join(MODEL_DIR, 'cache.json');
+    const currentConfig = {
+        MODELS_TO_DOWNLOAD,
+        KOKORO_REPO,
+        KOKORO_FILES,
+        MARGO_MODELS_TO_DOWNLOAD,
+    };
+
+    const force = process.argv.includes('--force');
+
+    if (!force && fs.existsSync(CACHE_FILE)) {
+        try {
+            const cachedConfig = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'));
+            if (JSON.stringify(cachedConfig) === JSON.stringify(currentConfig)) {
+                console.log('Models are already up to date. Skipping download. Use --force to override.');
+                return;
+            }
+        } catch (err) {
+            console.warn(`Warning: Could not read cache file ${CACHE_FILE}:`, err.message);
+        }
+    }
+
     if (!fs.existsSync(MODEL_DIR)) {
         console.log(`Creating directory: ${MODEL_DIR}`);
         fs.mkdirSync(MODEL_DIR, { recursive: true }); 
@@ -86,20 +132,12 @@ async function downloadModels() {
 
         // Download Marqo/marqo-fashionSigLIP model
         console.log(`Downloading files for Marqo/marqo-fashionSigLIP (zero-shot-image-classification, dtype: bnb4)...`);
-        await SiglipVisionModel.from_pretrained("Marqo/marqo-fashionSigLIP" ,{ 
-                    cache_dir: env.localModelPath,
-                    dtype: 'bnb4'
-                });
-        await AutoImageProcessor.from_pretrained("Marqo/marqo-fashionSigLIP", {
-            cache_dir: env.localModelPath,
-        });
-        await SiglipTextModel.from_pretrained("Marqo/marqo-fashionSigLIP", {
-            cache_dir: env.localModelPath,
-            dtype: 'bnb4'
-        });
-        await AutoTokenizer.from_pretrained("Marqo/marqo-fashionSigLIP", {
-            cache_dir: env.localModelPath,
-        });
+        for (const modelInfo of MARGO_MODELS_TO_DOWNLOAD) {
+            await MARGO_NAME_TO_CLASS[modelInfo.class].from_pretrained("Marqo/marqo-fashionSigLIP", {
+                cache_dir: env.localModelPath,
+                dtype: modelInfo.dtype
+            });
+        }
         console.log(`Successfully downloaded and cached Marqo/marqo-fashionSigLIP`);
 
         // Download onnx-community/Kokoro-82M-v1.0-ONNX model
@@ -142,6 +180,10 @@ async function downloadModels() {
         }
         console.log(`Successfully downloaded all files for ${KOKORO_REPO}`);
 
+        // Update cache file
+        fs.writeFileSync(CACHE_FILE, JSON.stringify(currentConfig, null, 2));
+        console.log(`Updated cache file: ${CACHE_FILE}\n`);
+
     } catch (err) {
         console.error("Model download failed:", err);
         env.allowRemoteModels = originalAllowRemote;