Optimize photoset downloads with bulk metadata fetch

- Use extras parameter in getPhotos() to fetch URLs and metadata in bulk
- Add helper functions to extract URLs from extras and download directly
- Skip per-photo API calls when pre-fetched data is available
- Reduces API calls from 3-4 per photo to 0 additional calls

This significantly speeds up large photoset downloads and reduces
the likelihood of hitting API rate limits.

Closes #64

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: beaufour

flickr_download/flick_download.py

@@ -12,6 +12,7 @@
 from typing import Any, Dict, Optional
 
 import flickr_api as Flickr
+import requests
 import yaml
 from flickr_api.flickrerrors import FlickrAPIError, FlickrError
 from flickr_api.objects import Person, Photo, Photoset, Walker
@@ -37,6 +38,23 @@
 CONFIG_FILE = "~/.flickr_download"
 OAUTH_TOKEN_FILE = "~/.flickr_token"
 
+# Extras to fetch in bulk API calls for optimization (reduces per-photo API calls)
+# See: https://www.flickr.com/services/api/flickr.photosets.getPhotos.html
+EXTRAS_FOR_BULK_FETCH = "url_o,url_l,url_c,url_z,url_m,url_s,original_format,date_taken,media"
+
+# Mapping from size labels to extras URL keys (in preference order for fallback)
+SIZE_TO_URL_KEY: Dict[str, str] = {
+    "Original": "url_o",
+    "Large": "url_l",
+    "Medium 800": "url_c",
+    "Medium 640": "url_z",
+    "Medium": "url_m",
+    "Small": "url_s",
+}
+
+# All URL keys in descending size order for fallback
+URL_KEYS_BY_SIZE = ["url_o", "url_l", "url_c", "url_z", "url_m", "url_s"]
+
 
 def _init(key: str, secret: str, oauth: bool) -> bool:
     """Initialize API.
@@ -93,6 +111,57 @@ def _load_defaults() -> Dict[str, Any]:
     return {}
 
 
+def _get_url_from_extras(photo: Photo, size_label: Optional[str]) -> Optional[str]:
+    """Get the best available URL from photo extras data.
+
+    :param photo: photo object with extras data
+    :param size_label: requested size label (or None for largest)
+    :returns: URL string or None if not available in extras
+    """
+    if size_label:
+        # Try to get the specific size requested
+        url_key = SIZE_TO_URL_KEY.get(size_label)
+        if url_key:
+            url = photo.get(url_key)
+            if url:
+                return str(url)
+    else:
+        # No specific size requested, find largest available
+        for url_key in URL_KEYS_BY_SIZE:
+            url = photo.get(url_key)
+            if url:
+                return str(url)
+    return None
+
+
+def _get_extension_from_url(url: str) -> str:
+    """Extract file extension from URL.
+
+    :param url: URL string
+    :returns: extension including dot (e.g., '.jpg')
+    """
+    # Remove query string if present
+    path = url.split("?")[0]
+    # Get extension from path
+    ext = os.path.splitext(path)[1]
+    return ext if ext else ".jpg"
+
+
+def _download_file(url: str, fname: str) -> None:
+    """Download a file directly from URL.
+
+    :param url: URL to download from
+    :param fname: local filename to save to
+    :raises IOError: on download failure
+    """
+    response = requests.get(url, stream=True, timeout=60)
+    response.raise_for_status()
+
+    with open(fname, "wb") as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+
+
 def _get_metadata_db(dirname: str) -> sqlite3.Connection:
     conn = sqlite3.connect(Path(dirname) / ".metadata.db")
     conn.execute(
@@ -142,7 +211,9 @@ def download_list(
     :param save_json: save photo info as .json file
     """
 
-    photos = Walker(pset.getPhotos)
+    # Use extras parameter to fetch metadata in bulk (reduces per-photo API calls)
+    # This fetches URLs, original format, date taken, etc. in the initial paginated call
+    photos = Walker(pset.getPhotos, extras=EXTRAS_FOR_BULK_FETCH)
 
     suffix = f" ({size_label})" if size_label else ""
 
@@ -215,12 +286,22 @@ def do_download_photo(
             logging.info("Skipping download of already downloaded photo with ID: %s", photo.id)
             return
 
+    # Try to get URL from extras first (avoids API call for file extension)
+    prefetched_url = _get_url_from_extras(photo, size_label)
+
     fname = get_full_path(dirname, get_filename(pset, photo, suffix))
-    try:
-        fname = photo._getOutputFilename(fname, size_label)
-    except (OSError, FlickrError) as ex:
-        logging.error("Error getting photo info for %s: %s", photo.id, ex)
-        return
+    if prefetched_url:
+        # We can determine extension from URL without an API call
+        ext = _get_extension_from_url(prefetched_url)
+        fname = fname + ext
+        logging.debug("Using extension %s from pre-fetched URL for %s", ext, photo.id)
+    else:
+        # Fall back to API call to get filename/extension
+        try:
+            fname = photo._getOutputFilename(fname, size_label)
+        except (OSError, FlickrError) as ex:
+            logging.error("Error getting photo info for %s: %s", photo.id, ex)
+            return
     json_fname = fname + ".json"
 
     if not photo["loaded"]:
@@ -252,7 +333,9 @@ def do_download_photo(
         except Exception:
             logging.warning("Trouble saving photo info: %s", sys.exc_info())
 
-    if not size_label:
+    if not size_label and not prefetched_url:
+        # Only check size label if we don't have a pre-fetched URL
+        # (pre-fetched URL means we already have a valid download URL)
         try:
             largest_size = photo._getLargestSizeLabel()
         except (OSError, FlickrError) as ex:
@@ -275,12 +358,22 @@ def do_download_photo(
             return
 
         try:
-            photo.save(fname, size_label)
+            # Use pre-fetched URL if available (avoids additional API calls)
+            if prefetched_url:
+                logging.debug("Using pre-fetched URL for %s", photo.id)
+                _download_file(prefetched_url, fname)
+            else:
+                # Fall back to standard save method (makes additional API calls)
+                logging.debug("Falling back to standard save for %s", photo.id)
+                photo.save(fname, size_label)
         except IOError as ex:
-            logging.error("IO error saving photo: %s", ex)
+            logging.error("IO error saving photo %s: %s", fname, ex)
             return
         except FlickrError as ex:
-            logging.error("Flickr error saving photo: %s", ex)
+            logging.error("Flickr error saving photo %s: %s", fname, ex)
+            return
+        except requests.RequestException as ex:
+            logging.error("Network error saving photo %s: %s", fname, ex)
             return
 
         # Set file times to when the photo was taken

tests/test_flick_download.py

@@ -203,6 +203,8 @@ def test_skip_existing_file(self) -> None:
             mock_photo.__getitem__ = Mock(return_value=True)  # loaded = True
             mock_photo._getOutputFilename = Mock(return_value=str(existing_file))
             mock_photo.save = Mock()
+            # Return None for URL extras to trigger fallback to _getOutputFilename
+            mock_photo.get = Mock(return_value=None)
 
             mock_pset = Mock()
             mock_pset.title = "Test Set"