Add local file path and network path options.

Author: vkirkl

docs/environment.rst

@@ -35,8 +35,22 @@ and configurations.
       to no debug logging.  Setting this value to anything will turn on debug logging
       (e.g., ``VERBOSE=1``).
    - ``SCOUTBOT_MODEL_URL`` (default: https://wildbookiarepository.azureedge.net/models)
-      The base URL for downloading ONNX model files. This allows organizations to host models on their
-      own infrastructure instead of using the default CDN.
+      The base URL or path for accessing ONNX model files. This can be:
+
+      * An HTTP/HTTPS URL (e.g., ``https://example.com/models``)
+      * A local file path (e.g., ``/opt/models`` or ``C:\models``)
+      * A network path (e.g., ``//server/share/models`` or ``\\server\share\models``)
+
+      When using file paths, models will be copied to a local cache for consistency.
+      This allows organizations to host models on their own infrastructure, CDN, or
+      local/network storage.
    - ``SCOUTBOT_DATA_URL`` (default: https://wildbookiarepository.azureedge.net/data)
-      The base URL for downloading test data files. This allows organizations to host test data on their
-      own infrastructure instead of using the default CDN.
+      The base URL or path for downloading test data files.
+
+      As with SCOUTBOT_MODEL_URL, this can be:
+      * An HTTP/HTTPS URL or
+      * A local file path or
+      * A network path
+
+      This allows organizations to host test data on their own infrastructure, CDN, or
+      local/network storage.

scoutbot/__init__.py

@@ -51,7 +51,10 @@
 '''
 import os
 import cv2
+import shutil
 from os.path import exists
+from urllib.parse import urlparse
+from pathlib import Path
 
 import pooch
 import utool as ut
@@ -69,9 +72,34 @@
     'SCOUTBOT_DATA_URL', 'https://wildbookiarepository.azureedge.net/data'
 ).rstrip('/')
 
+# Validate MODEL_BASE_URL
+parsed = urlparse(MODEL_BASE_URL)
+if parsed.scheme in ('http', 'https', 'ftp'):
+    # It's a URL, no further validation needed
+    pass
+elif parsed.scheme and parsed.scheme not in ('file', ''):
+    log.warning(f"Unrecognized scheme in MODEL_BASE_URL: {parsed.scheme}")
+else:
+    # It's a path - check if it exists
+    model_path = Path(MODEL_BASE_URL)
+    if not model_path.exists():
+        log.warning(f"Model source path does not exist: {MODEL_BASE_URL}")
+
+# Validate DATA_BASE_URL
+parsed = urlparse(DATA_BASE_URL)
+if parsed.scheme in ('http', 'https', 'ftp'):
+    # It's a URL, no further validation needed
+    pass
+elif parsed.scheme and parsed.scheme not in ('file', ''):
+    log.warning(f"Unrecognized scheme in DATA_BASE_URL: {parsed.scheme}")
+else:
+    # It's a path - check if it exists
+    data_path = Path(DATA_BASE_URL)
+    if not data_path.exists():
+        log.warning(f"Data source path does not exist: {DATA_BASE_URL}")
 
 from scoutbot import agg, loc, tile, wic, tile_batched  # NOQA
-from scoutbot.loc import CONFIGS as LOC_CONFIGS # NOQA
+from scoutbot.loc import CONFIGS as LOC_CONFIGS  # NOQA
 
 # from tile_batched.models import Yolov8DetectionModel
 # from tile_batched import get_sliced_prediction_batched
@@ -81,6 +109,68 @@
 __version__ = VERSION
 
 
+def get_resource_from_source(resource_name, resource_hash, source_base, resource_type='model', use_cache=True):
+    """
+    Retrieve a resource (model or data) from URL, local path, or network path.
+
+    Args:
+        resource_name: Name of the resource file
+        resource_hash: Expected hash of the resource (used for URL downloads)
+        source_base: Base URL or path (from MODEL_BASE_URL or DATA_BASE_URL)
+        resource_type: Type of resource ('model' or 'data') for cache organization
+        use_cache: Whether to use cached version for URLs
+
+    Returns:
+        str: Path to the resource file
+
+    Raises:
+        FileNotFoundError: If the resource file doesn't exist at the specified path
+    """
+    # Parse the source to determine if it's a URL or path
+    parsed = urlparse(source_base)
+
+    if parsed.scheme in ('http', 'https', 'ftp'):
+        # It's a URL - use pooch as before
+        resource_url = f'{source_base}/{resource_name}'
+        return pooch.retrieve(
+            url=resource_url,
+            known_hash=resource_hash,
+            progressbar=not QUIET,
+        )
+    else:
+        # It's a local or network path
+        source_path = Path(source_base) / resource_name
+
+        if not source_path.exists():
+            raise FileNotFoundError(f"{resource_type.capitalize()} file not found: {source_path}")
+
+        if use_cache:
+            # Copy to cache directory to maintain consistency
+            cache_dir = Path(pooch.os_cache(f"scoutbot/{resource_type}s"))
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            cache_path = cache_dir / resource_name
+
+            # Only copy if not already cached or file has changed
+            if not cache_path.exists() or os.path.getmtime(source_path) > os.path.getmtime(cache_path):
+                log.debug(f"Copying {resource_type} from {source_path} to cache {cache_path}")
+                shutil.copy2(source_path, cache_path)
+
+            return str(cache_path)
+        else:
+            # Use directly from source
+            return str(source_path)
+
+
+def get_model_from_source(model_name, model_hash, source_base, use_cache=True):
+    """Backward compatibility wrapper for get_resource_from_source"""
+    return get_resource_from_source(model_name, model_hash, source_base, 'model', use_cache)
+
+
+def get_data_from_source(data_name, data_hash, source_base, use_cache=True):
+    """Convenience wrapper for get_resource_from_source for data files"""
+    return get_resource_from_source(data_name, data_hash, source_base, 'data', use_cache)
+
+
 def fetch(pull=False, config=None):
     """
     Fetch the WIC and Localizer ONNX model files from a CDN if they do not exist locally.
@@ -108,15 +198,15 @@ def fetch(pull=False, config=None):
 
 
 def pipeline(
-    filepath,
-    config=None,
-    backend_device='cuda:0',
-    wic_thresh=wic.CONFIGS[None]['thresh'],
-    loc_thresh=loc.CONFIGS[None]['thresh'],
-    loc_nms_thresh=loc.CONFIGS[None]['nms'],
-    agg_thresh=agg.CONFIGS[None]['thresh'],
-    agg_nms_thresh=agg.CONFIGS[None]['nms'],
-    clean=True,
+        filepath,
+        config=None,
+        backend_device='cuda:0',
+        wic_thresh=wic.CONFIGS[None]['thresh'],
+        loc_thresh=loc.CONFIGS[None]['thresh'],
+        loc_nms_thresh=loc.CONFIGS[None]['nms'],
+        agg_thresh=agg.CONFIGS[None]['thresh'],
+        agg_nms_thresh=agg.CONFIGS[None]['nms'],
+        clean=True,
 ):
     """
     Run the ML pipeline on a given image filepath and return the detections
@@ -202,17 +292,17 @@ def pipeline(
 
 
 def pipeline_v3(
-    filepath,
-    config,
-    batched_detection_model=None,
-    backend_device='cuda:0',
-    loc_thresh=0.45,
-    slice_height=512,
-    slice_width=512,
-    overlap_height_ratio=0.25,
-    overlap_width_ratio=0.25,
-    perform_standard_pred=False,
-    postprocess_class_agnostic=True,
+        filepath,
+        config,
+        batched_detection_model=None,
+        backend_device='cuda:0',
+        loc_thresh=0.45,
+        slice_height=512,
+        slice_width=512,
+        overlap_height_ratio=0.25,
+        overlap_width_ratio=0.25,
+        perform_standard_pred=False,
+        postprocess_class_agnostic=True,
 ):
     """
     Run the ML pipeline on a given image filepath and return the detections
@@ -283,15 +373,15 @@ def pipeline_v3(
 
 
 def batch(
-    filepaths,
-    config=None,
-    backend_device='cuda:0',
-    wic_thresh=wic.CONFIGS[None]['thresh'],
-    loc_thresh=loc.CONFIGS[None]['thresh'],
-    loc_nms_thresh=loc.CONFIGS[None]['nms'],
-    agg_thresh=agg.CONFIGS[None]['thresh'],
-    agg_nms_thresh=agg.CONFIGS[None]['nms'],
-    clean=True,
+        filepaths,
+        config=None,
+        backend_device='cuda:0',
+        wic_thresh=wic.CONFIGS[None]['thresh'],
+        loc_thresh=loc.CONFIGS[None]['thresh'],
+        loc_nms_thresh=loc.CONFIGS[None]['nms'],
+        agg_thresh=agg.CONFIGS[None]['thresh'],
+        agg_nms_thresh=agg.CONFIGS[None]['nms'],
+        clean=True,
 ):
     """
     Run the ML pipeline on a given batch of image filepaths and return the detections
@@ -390,7 +480,7 @@ def batch(
     assert len(loc_tile_grids) == len(loc_outputs)
 
     for filepath, loc_tile_grid, loc_output in zip(
-        loc_tile_img_filepaths, loc_tile_grids, loc_outputs
+            loc_tile_img_filepaths, loc_tile_grids, loc_outputs
     ):
         batch[filepath]['loc']['grids'].append(loc_tile_grid)
         batch[filepath]['loc']['outputs'].append(loc_output)
@@ -429,16 +519,16 @@ def batch(
 
 
 def batch_v3(
-    filepaths,
-    config,
-    backend_device,
-    loc_thresh=0.45,
-    slice_height=512,
-    slice_width=512,
-    overlap_height_ratio=0.25,
-    overlap_width_ratio=0.25,
-    perform_standard_pred=False,
-    postprocess_class_agnostic=True,
+        filepaths,
+        config,
+        backend_device,
+        loc_thresh=0.45,
+        slice_height=512,
+        slice_width=512,
+        overlap_height_ratio=0.25,
+        overlap_width_ratio=0.25,
+        perform_standard_pred=False,
+        postprocess_class_agnostic=True,
 ):
     yolov8_model_path = loc.fetch(config=config)
 
@@ -499,10 +589,11 @@ def example():
         '786a940b062a90961f409539292f09144c3dbdbc6b6faa64c3e764d63d55c988'  # NOQA
     )
 
-    img_filepath = pooch.retrieve(
-        url=f'{DATA_BASE_URL}/{TEST_IMAGE}',
-        known_hash=TEST_IMAGE_HASH,
-        progressbar=True,
+    img_filepath = get_data_from_source(
+        data_name=TEST_IMAGE,
+        data_hash=TEST_IMAGE_HASH,
+        source_base=DATA_BASE_URL,
+        use_cache=True
     )
     assert exists(img_filepath)
 

scoutbot/loc/__init__.py

@@ -21,7 +21,7 @@
 import tqdm
 import utool as ut
 
-from scoutbot import MODEL_BASE_URL, QUIET, log
+from scoutbot import log
 from scoutbot.loc.transforms import (
     Compose,
     GetBoundingBoxes,
@@ -144,13 +144,13 @@
 
 def fetch(pull=False, config=DEFAULT_CONFIG):
     """
-    Fetch the Localizer ONNX model file from a CDN if it does not exist locally.
+    Fetch the Localizer ONNX model file from a CDN, local path, or network path.
 
     This function will throw an AssertionError if the download fails or the
     file otherwise does not exists locally on disk.
 
     Args:
-        pull (bool, optional): If :obj:`True`, force using the downloaded versions
+        pull (bool, optional): If :obj:`True`, force using the downloaded/copied versions
             stored in the local system's cache.  Defaults to :obj:`False`.
         config (str or None, optional): the configuration to use, one of ``phase1``
             or ``mvp``.  Defaults to :obj:`None`.
@@ -160,6 +160,7 @@ def fetch(pull=False, config=DEFAULT_CONFIG):
 
     Raises:
         AssertionError: If the model cannot be fetched.
+        FileNotFoundError: If the model file doesn't exist at the specified path.
     """
     if config is None:
         config = DEFAULT_CONFIG
@@ -171,10 +172,14 @@ def fetch(pull=False, config=DEFAULT_CONFIG):
     if not pull and exists(model_path):
         onnx_model = model_path
     else:
-        onnx_model = pooch.retrieve(
-            url=f'{MODEL_BASE_URL}/{model_name}',
-            known_hash=model_hash,
-            progressbar=not QUIET,
+        # Import the utility function from parent module
+        from scoutbot import MODEL_BASE_URL, QUIET, get_model_from_source
+
+        onnx_model = get_model_from_source(
+            model_name=model_name,
+            model_hash=model_hash,
+            source_base=MODEL_BASE_URL,
+            use_cache=True
         )
         assert exists(onnx_model)
 
@@ -256,6 +261,9 @@ def predict(gen):
     """
     log.debug('Running LOC inference')
 
+    # Import QUIET from parent module
+    from scoutbot import QUIET
+
     ort_sessions = {}
 
     for chunk, sizes, trim, config in tqdm.tqdm(gen, disable=QUIET):