api_scraper.py

import requests
import time
import random
import json
import signal
import argparse
from datetime import datetime
from urllib.parse import urljoin, urlparse, parse_qs, urlencode
from bs4 import BeautifulSoup
import os
from typing import Set, List, Optional


class NHLAPIScraper:
    def __init__(self, checkpoint_file: Optional[str] = None, checkpoint_interval: int = 100):
        self.base_url = "https://www.nhl.com"
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        })
        
        # NHL domain list for crawling
        self.nhl_domains = {
            'www.nhl.com', 'nhl.com', 'api-web.nhle.com', 'api.nhle.com',
            'records.nhl.com', 'statsapi.web.nhl.com', 'stats.nhl.com',
            'content.nhl.com', 'media.nhl.com', 'cms.nhl.com',
            'forge-dapi.d3.nhle.com', 'api-web.nhle.com'
        }
        
        self.visited_urls: Set[str] = set()
        self.to_visit: List[str] = [self.base_url]
        self.checkpoint_interval = checkpoint_interval
        self.processed_count = 0
        self.saved_api_count = 0
        
        if checkpoint_file:
            self._load_checkpoint(checkpoint_file)
        else:
            self.output_file = self._create_output_file()
            
        self.checkpoint_file = self._create_checkpoint_file()
        
        # Setup graceful shutdown
        signal.signal(signal.SIGINT, self._signal_handler)
        
    def _create_output_file(self) -> str:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"runs/{timestamp}_api_urls.txt"
        return filename
    
    def _create_checkpoint_file(self) -> str:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"runs/{timestamp}_api_checkpoint.json"
        return filename
    
    def _load_checkpoint(self, checkpoint_file: str):
        try:
            with open(checkpoint_file, 'r', encoding='utf-8') as f:
                data = json.load(f)
            
            self.visited_urls = set(data.get('visited', []))
            self.to_visit = data.get('queue', [self.base_url])
            self.output_file = data.get('output_file', self._create_output_file())
            self.processed_count = data.get('processed_count', 0)
            self.saved_api_count = data.get('saved_api_count', 0)
            
            print(f"Loaded checkpoint: {len(self.visited_urls)} visited, {len(self.to_visit)} queued, {self.saved_api_count} API URLs saved")
            
        except (FileNotFoundError, json.JSONDecodeError, KeyError) as e:
            print(f"Error loading checkpoint {checkpoint_file}: {e}")
            print("Starting fresh...")
            self.visited_urls = set()
            self.to_visit = [self.base_url]
            self.output_file = self._create_output_file()
            self.processed_count = 0
            self.saved_api_count = 0
    
    def _save_checkpoint(self):
        checkpoint_data = {
            'visited': list(self.visited_urls),
            'queue': self.to_visit,
            'output_file': self.output_file,
            'processed_count': self.processed_count,
            'saved_api_count': self.saved_api_count,
            'timestamp': datetime.now().isoformat()
        }
        
        try:
            with open(self.checkpoint_file, 'w', encoding='utf-8') as f:
                json.dump(checkpoint_data, f, indent=2)
            print(f"Checkpoint saved: {self.checkpoint_file}")
        except Exception as e:
            print(f"Error saving checkpoint: {e}")
    
    def _signal_handler(self, signum, frame):
        print("\nReceived interrupt signal. Saving checkpoint...")
        self._save_checkpoint()
        print("Checkpoint saved. Exiting...")
        exit(0)
    
    def _is_api_url(self, url: str) -> bool:
        """Check if URL contains 'api' or 'nhle' anywhere in the full URL"""
        url_lower = url.lower()
        return 'api' in url_lower or 'nhle' in url_lower
    
    def _save_url(self, url: str):
        """Only save URLs that contain 'api' or 'nhle'"""
        if self._is_api_url(url):
            with open(self.output_file, 'a', encoding='utf-8') as f:
                f.write(f"{url}\n")
                f.flush()
            self.saved_api_count += 1
            print(f"  -> SAVED API URL: {url}")
    
    def _make_request(self, url: str, max_retries: int = 3) -> requests.Response:
        for attempt in range(max_retries):
            try:
                delay = random.uniform(1.5, 2.0)
                time.sleep(delay)
                
                response = self.session.get(url, timeout=10)
                
                if response.status_code == 429:
                    retry_delay = 2 ** attempt * 5 + random.uniform(1, 3)
                    print(f"Rate limited. Waiting {retry_delay:.1f}s before retry {attempt + 1}/{max_retries}")
                    time.sleep(retry_delay)
                    continue
                
                response.raise_for_status()
                return response
                
            except requests.RequestException as e:
                if attempt == max_retries - 1:
                    print(f"Failed to fetch {url} after {max_retries} attempts: {e}")
                    raise
                else:
                    retry_delay = 2 ** attempt + random.uniform(1, 2)
                    print(f"Request failed, retrying in {retry_delay:.1f}s: {e}")
                    time.sleep(retry_delay)
    
    def _normalize_url(self, url: str) -> str:
        """Normalize URL to avoid duplicates from minor variations"""
        parsed = urlparse(url)
        
        # Remove trailing slash from path (except root)
        path = parsed.path.rstrip('/') if parsed.path != '/' else '/'
        
        # Sort query parameters for consistent ordering
        query = ''
        if parsed.query:
            params = parse_qs(parsed.query, keep_blank_values=True)
            sorted_params = sorted(params.items())
            query = urlencode(sorted_params, doseq=True)
        
        # Rebuild URL
        normalized = f"{parsed.scheme}://{parsed.netloc}{path}"
        if query:
            normalized += f"?{query}"
            
        return normalized
    
    def _extract_urls(self, html: str, base_url: str) -> List[str]:
        soup = BeautifulSoup(html, 'html.parser')
        urls = []
        
        for tag in soup.find_all(['a', 'link', 'script']):
            # Check href attribute
            href = tag.get('href')
            if href:
                full_url = urljoin(base_url, href)
                parsed = urlparse(full_url)
                
                # Accept URLs from any NHL-related domain
                if any(domain in parsed.netloc for domain in ['nhl.com', 'nhle.com']):
                    normalized_url = self._normalize_url(full_url)
                    urls.append(normalized_url)
            
            # Check src attribute (for script tags, etc.)
            src = tag.get('src')
            if src:
                full_url = urljoin(base_url, src)
                parsed = urlparse(full_url)
                
                if any(domain in parsed.netloc for domain in ['nhl.com', 'nhle.com']):
                    normalized_url = self._normalize_url(full_url)
                    urls.append(normalized_url)
        
        # Also check for API URLs in script content
        for script in soup.find_all('script'):
            if script.string:
                # Look for URL patterns in JavaScript
                import re
                url_pattern = r'https?://[^\s"\'<>]+(?:nhl\.com|nhle\.com)[^\s"\'<>]*'
                matches = re.findall(url_pattern, script.string)
                for match in matches:
                    normalized_url = self._normalize_url(match)
                    urls.append(normalized_url)
        
        return urls
    
    def scrape(self):
        print(f"Starting NHL API URL scraper...")
        print(f"Output file: {self.output_file}")
        print(f"Target: URLs containing 'api' or 'nhle'")
        
        while self.to_visit:
            current_url = self.to_visit.pop(0)
            
            if current_url in self.visited_urls:
                continue
                
            try:
                print(f"Scraping: {current_url}")
                response = self._make_request(current_url)
                
                self.visited_urls.add(current_url)
                # Save URL only if it contains 'api' or 'nhle'
                self._save_url(current_url)
                self.processed_count += 1
                
                if response.headers.get('content-type', '').startswith('text/html'):
                    new_urls = self._extract_urls(response.text, current_url)
                    
                    for url in new_urls:
                        if url not in self.visited_urls and url not in self.to_visit:
                            self.to_visit.append(url)
                
                print(f"Found {len(new_urls)} new URLs. Queue: {len(self.to_visit)}, Visited: {len(self.visited_urls)}, API URLs Saved: {self.saved_api_count}")
                
                # Save checkpoint periodically
                if self.processed_count % self.checkpoint_interval == 0:
                    self._save_checkpoint()
                
            except Exception as e:
                print(f"Error processing {current_url}: {e}")
                continue
        
        print(f"Scraping complete! Total URLs visited: {len(self.visited_urls)}")
        print(f"API URLs saved: {self.saved_api_count}")
        print(f"Results saved to: {self.output_file}")
        
        # Save final checkpoint
        self._save_checkpoint()


def main():
    parser = argparse.ArgumentParser(description='NHL API URL Scraper with checkpoint support')
    parser.add_argument('--checkpoint', '-c', help='Resume from checkpoint file')
    parser.add_argument('--checkpoint-interval', '-i', type=int, default=100, 
                       help='Save checkpoint every N URLs (default: 100)')
    
    args = parser.parse_args()
    
    scraper = NHLAPIScraper(
        checkpoint_file=args.checkpoint,
        checkpoint_interval=args.checkpoint_interval
    )
    scraper.scrape()


if __name__ == "__main__":
    main()