Fixing top n coins to download.

Author: yohplala

docs/DATA_SOURCES.md

@@ -192,10 +192,10 @@ The pipeline generates a `data_status.html` page (at `site/data_status.html`) th
 
 | Card | Description |
 |------|-------------|
-| **Coins Requested** | Number requested from API (1200), with sublabel showing breakdown: "886 USD + 490 no-USD" |
-| **Coins Accepted** | Total coins accepted for download, with sublabel showing how many from no-USD source |
+| **Coins Requested** | Number requested from API (1200), also the cap on total accepted coins |
+| **Coins Accepted** | Total coins accepted for download (≤ requested), with sublabel showing how many from no-USD source |
 | **Coins Downloaded** | Coins that have downloaded price data in cache |
-| **Skipped / Failed** | Filtered coins (stablecoins, wrapped) + failed downloads (no BTC pair) |
+| **Skipped / Failed** | Filtered coins (stablecoins, wrapped) + failed downloads (no BTC pair) + capped no-USD coins |
 | **Total Pairs** | Sum of all quote pairs (BTC + USD) across all coins |
 
 ### Downloaded Coins Table
@@ -458,20 +458,22 @@ This error occurs when a coin doesn't have a direct trading pair on CryptoCompar
 - Coin may be too new (created after requested start date)
 - Check CryptoCompare directly: `https://min-api.cryptocompare.com/data/v2/histoday?fsym=ETH&tsym=BTC&limit=10`
 
-### Discrepancy between requested and returned coins
+### Requested vs. returned coins
 
 The CryptoCompare market cap API returns coins in two categories:
 
 1. **Coins WITH USD data** (~886 of 1200): Have market cap, price, and volume data
-2. **Coins WITHOUT USD data** (~490 of 1200): Returned by API but missing USD price data
+2. **Coins WITHOUT USD data** (~490): Returned by API but missing USD price data
 
 Lower-ranked coins (smaller market cap) are more likely to lack USD data on CryptoCompare. These coins often still have BTC trading pairs available via the `histoday` endpoint.
 
-**Halvix now processes both categories:**
-- Filters both (removes stablecoins, wrapped, etc.)
+**Halvix processes both categories with a total cap:**
+- USD coins have priority (they have actual market cap data for ranking)
+- No-USD coins fill remaining slots up to the requested limit
+- Total accepted coins will never exceed `coins_requested`
+- Both are filtered (removes stablecoins, wrapped, etc.)
 - Marks each coin with `has_usd_data: true/false` in `coins_to_download.json`
-- Downloads BTC pairs for all altcoins (no change in behavior)
-- Shows "BTC-only" source in the data status page for coins without USD data
+- No-USD coins are sorted by their API position (CryptoCompare's internal ranking)
 
 The `fetch_metadata.json` file records the full breakdown:
 ```json
@@ -480,12 +482,19 @@ The `fetch_metadata.json` file records the full breakdown:
   "coins_fetched": 886,
   "coins_no_usd_data": 490,
   "coins_no_usd_filtered": 7,
-  "coins_no_usd_accepted": 483,
+  "coins_no_usd_accepted": 283,
+  "coins_no_usd_capped": 200,
   "coins_filtered": 31,
-  "coins_accepted": 1338
+  "coins_accepted": 1138
 }
 ```
 
+In this example:
+- 886 coins had USD data, 31 were filtered → 855 USD coins accepted
+- 490 coins had no USD data, 7 were filtered → 483 passed filtering
+- But only 283 no-USD coins were included (200 capped to meet the 1200 limit)
+- Total: 855 + 283 = 1138 ≤ 1200
+
 ---
 
 *Last updated: 2025-12-24*

src/api/cryptocompare.py

@@ -188,6 +188,12 @@ def __init__(
             headers["authorization"] = f"Apikey {api_key}"
         self.session.headers.update(headers)
 
+        # Rate limit logging: track calls for periodic status logging
+        self._calls_since_last_log = 0
+        self._calls_log_interval = 50  # Log rate limit status every N calls
+        self._last_status_log_time: float | None = None
+        self._status_log_interval = 60.0  # Or every N seconds
+
     def get_rate_limit_status(self, use_cache: bool = True) -> RateLimitStatus:
         """
         Get current rate limit status from the API.
@@ -266,6 +272,53 @@ def extract_calls(period_data: dict) -> tuple[int, int]:
             logger.warning("Error checking rate limit status: %s", e)
             return RateLimitStatus()
 
+    def _log_rate_limit_status_if_needed(self, status: RateLimitStatus) -> None:
+        """Log rate limit status periodically (every N calls or N seconds)."""
+        self._calls_since_last_log += 1
+
+        should_log = False
+        reason = ""
+
+        # Log every N calls
+        if self._calls_since_last_log >= self._calls_log_interval:
+            should_log = True
+            reason = f"every {self._calls_log_interval} calls"
+
+        # Or log every N seconds
+        elif self._last_status_log_time is not None:
+            elapsed = time.time() - self._last_status_log_time
+            if elapsed >= self._status_log_interval:
+                should_log = True
+                reason = f"every {self._status_log_interval:.0f}s"
+
+        # Or log on first call (no previous log time)
+        elif self._last_status_log_time is None:
+            should_log = True
+            reason = "initial status"
+
+        if should_log:
+            self._calls_since_last_log = 0
+            self._last_status_log_time = time.time()
+
+            # Calculate totals for clearer logging
+            total_second = status.calls_made_second + status.calls_left_second
+            total_minute = status.calls_made_minute + status.calls_left_minute
+            total_hour = status.calls_made_hour + status.calls_left_hour
+            total_month = status.calls_made_month + status.calls_left_month
+
+            logger.info(
+                "Rate limit status (%s): " "second %d/%d, minute %d/%d, hour %d/%d, month %d/%d",
+                reason,
+                status.calls_made_second,
+                total_second,
+                status.calls_made_minute,
+                total_minute,
+                status.calls_made_hour,
+                total_hour,
+                status.calls_made_month,
+                total_month,
+            )
+
     def _wait_for_rate_limit(self) -> None:
         """
         Wait if necessary to respect rate limits.
@@ -281,6 +334,10 @@ def _wait_for_rate_limit(self) -> None:
 
         # Check rate limit status periodically
         status = self.get_rate_limit_status(use_cache=True)
+
+        # Log status periodically for visibility
+        self._log_rate_limit_status_if_needed(status)
+
         if status.is_near_limit:
             wait_time = status.recommended_wait_seconds
             if wait_time > 0:

src/data/fetcher.py

@@ -54,13 +54,14 @@ class FetchResult:
 
     success: bool
     message: str
-    coins_requested: int = 0  # How many coins we asked the API for
+    coins_requested: int = 0  # How many coins we asked the API for (also the cap)
     coins_fetched: int = 0  # How many coins had USD data and were returned
     coins_no_usd_data: int = 0  # How many coins were missing USD data from API
     coins_no_usd_filtered: int = 0  # How many no-USD coins were filtered (stablecoins, etc.)
-    coins_no_usd_accepted: int = 0  # How many no-USD coins were accepted (BTC pairs only)
+    coins_no_usd_accepted: int = 0  # How many no-USD coins were included (after cap)
+    coins_no_usd_capped: int = 0  # How many no-USD coins were excluded due to cap
     coins_filtered: int = 0  # How many USD coins were filtered out (stablecoins, wrapped, etc.)
-    coins_accepted: int = 0  # Total coins accepted for download (USD + no-USD)
+    coins_accepted: int = 0  # Total coins accepted for download (USD + no-USD, capped at requested)
     errors: list[str] | None = None
 
 
@@ -213,8 +214,23 @@ def fetch_and_filter_coins(
             for coin in no_usd_accepted:
                 coin["has_usd_data"] = False
 
-            # Combine both lists
-            all_coins_to_download = coins_to_download + no_usd_accepted
+            # --- Cap total coins at n ---
+            # USD coins have priority (they have actual market cap data)
+            # No-USD coins fill remaining slots up to the requested limit
+            remaining_slots = max(0, n - len(coins_to_download))
+            no_usd_included = no_usd_accepted[:remaining_slots]
+            no_usd_capped = len(no_usd_accepted) - len(no_usd_included)
+
+            if no_usd_capped > 0:
+                logger.info(
+                    "Capped no-USD coins: %d included, %d excluded to meet limit of %d",
+                    len(no_usd_included),
+                    no_usd_capped,
+                    n,
+                )
+
+            # Combine both lists (respecting the cap)
+            all_coins_to_download = coins_to_download + no_usd_included
 
             # Export skipped coins for review (from USD coins only, main source)
             if export_skipped:
@@ -233,7 +249,8 @@ def fetch_and_filter_coins(
                 coins_fetched=len(all_coins),
                 coins_no_usd_data=len(coins_without_usd),
                 coins_no_usd_filtered=no_usd_filtered,
-                coins_no_usd_accepted=len(no_usd_accepted),
+                coins_no_usd_accepted=len(no_usd_included),  # Only those actually included
+                coins_no_usd_capped=no_usd_capped,  # How many were excluded by cap
                 coins_filtered=usd_coins_filtered,
                 coins_accepted=len(all_coins_to_download),
             )

src/main.py

@@ -1751,9 +1751,13 @@ def cmd_list_coins(args: argparse.Namespace) -> int:
     logger.info("")
     logger.info("  Without USD data: %d (BTC pairs only)", result.coins_no_usd_data)
     logger.info("    - Filtered:     %d", result.coins_no_usd_filtered)
+    if result.coins_no_usd_capped > 0:
+        logger.info("    - Capped:       %d (excluded to meet limit)", result.coins_no_usd_capped)
     logger.info("    - Accepted:     %d", result.coins_no_usd_accepted)
     logger.info("")
-    logger.info("  Total accepted:   %d coins", result.coins_accepted)
+    logger.info(
+        "  Total accepted:   %d coins (capped at %d)", result.coins_accepted, result.coins_requested
+    )
 
     # Print filter breakdown for USD coins
     summary = fetcher.get_filter_summary()
@@ -1777,6 +1781,7 @@ def cmd_list_coins(args: argparse.Namespace) -> int:
             "coins_no_usd_data": result.coins_no_usd_data,
             "coins_no_usd_filtered": result.coins_no_usd_filtered,
             "coins_no_usd_accepted": result.coins_no_usd_accepted,
+            "coins_no_usd_capped": result.coins_no_usd_capped,
             "coins_filtered": result.coins_filtered,
             "coins_accepted": result.coins_accepted,
             "timestamp": datetime.now().isoformat(),