add pipeline and retry logic for rate limiting

Author: whatimdoingwithmylife-sudo

.github/workflows/ci.yml

@@ -0,0 +1,32 @@
+
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[dev]
+        
+    - name: Run tests
+      run: |
+        pytest -v tests/

.kiro/specs/real-api-tests/requirements.md

@@ -92,6 +92,7 @@ with R34Client() as client:
 | `timeout` | float | 30.0 | Request timeout in seconds |
 | `posts_per_page` | int | 42 | Posts per page for pagination |
 | `headers` | dict | Browser-like headers | Custom HTTP headers |
+| `max_retries` | int | 5 | Max retries for rate limits (429) |
 
 ## License
 

README.md

@@ -26,12 +26,14 @@ classifiers = [
 dependencies = [
     "httpx>=0.24.0",
     "selectolax>=0.3.0",
+    "tenacity>=8.2.0",
 ]
 
 [project.optional-dependencies]
 dev = [
     "pytest>=7.0",
     "pytest-asyncio>=0.21",
+    "respx>=0.20.0",
 ]
 
 [project.urls]

pyproject.toml

@@ -1,7 +1,7 @@
 """RHI API Wrapper - A simple wrapper for rule34 image board."""
 
 from .models import Post, Tag, PostComment, PostDetails, UserProfile
-from .client import R34Client, AsyncR34Client
+from .client import R34Client, AsyncR34Client, RateLimitError
 from .parser import (
     PostParser,
     SidebarParser,
@@ -18,6 +18,7 @@
     "UserProfile",
     "R34Client",
     "AsyncR34Client",
+    "RateLimitError",
     "PostParser",
     "SidebarParser",
     "PostDetailsParser",

rule34scraper/__init__.py

@@ -1,13 +1,35 @@
 """HTTP client for RHI API."""
 
+import logging
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
 import httpx
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+    before_sleep_log,
+)
 
 from .models import Post, Tag, PostDetails, UserProfile
 from .parser import PostParser, SidebarParser, PostDetailsParser, UserProfileParser
 
+logger = logging.getLogger(__name__)
+
+
+class RateLimitError(Exception):
+    """Raised when the server returns a 429 Too Many Requests response."""
+    pass
+
+
+def _check_rate_limit(response: httpx.Response) -> httpx.Response:
+    """Check response for rate limiting and raise if detected."""
+    if response.status_code == 429:
+        raise RateLimitError(f"Rate limited: {response.status_code}")
+    return response
+
 DEFAULT_BASE_URL = "https://rule34.xxx/index.php"
 DEFAULT_POSTS_PER_PAGE = 42
 
@@ -27,6 +49,7 @@ def __init__(
         timeout: float = 30.0,
         posts_per_page: int = DEFAULT_POSTS_PER_PAGE,
         headers: Optional[Dict[str, str]] = None,
+        max_retries: int = 5,
     ):
         """Initialize the client.
 
@@ -35,12 +58,14 @@ def __init__(
             timeout: Request timeout in seconds.
             posts_per_page: Number of posts per page (for pagination offset).
             headers: Custom headers to use (merges with defaults).
+            max_retries: Maximum number of retry attempts for rate-limited requests.
         """
         self.base_url = base_url.rstrip("/")
         self.posts_per_page = posts_per_page
         self._timeout = timeout
         self._headers = {**DEFAULT_HEADERS, **(headers or {})}
         self._client: Optional[httpx.Client] = None
+        self._max_retries = max_retries
 
     @property
     def client(self) -> httpx.Client:
@@ -49,6 +74,21 @@ def client(self) -> httpx.Client:
             self._client = httpx.Client(timeout=self._timeout, headers=self._headers)
         return self._client
 
+    def _get(self, url: str, params: Dict = None) -> httpx.Response:
+        """Execute GET request with retry logic for rate limits."""
+        @retry(
+            stop=stop_after_attempt(self._max_retries),
+            wait=wait_exponential(multiplier=1, min=1.0, max=60.0),
+            retry=retry_if_exception_type((RateLimitError, httpx.TransportError)),
+            before_sleep=before_sleep_log(logger, logging.WARNING),
+            reraise=True,
+        )
+        def _request() -> httpx.Response:
+            response = self.client.get(url, params=params)
+            return _check_rate_limit(response)
+        
+        return _request()
+
     def close(self) -> None:
         """Close the HTTP client."""
         if self._client:
@@ -74,7 +114,7 @@ def get_posts(self, tags: str = "", page: int = 1) -> Tuple[List[Post], List[Tag
         offset = (page - 1) * self.posts_per_page
         params = {"page": "post", "s": "list", "tags": tags, "pid": offset}
 
-        response = self.client.get(self.base_url, params=params)
+        response = self._get(self.base_url, params=params)
         response.raise_for_status()
 
         html = response.text
@@ -96,14 +136,14 @@ def get_sidebar_tags(self, tags: str = "") -> List[Tag]:
     def get_post_details(self, post_id: int) -> Optional[PostDetails]:
         """Fetch detailed info for a specific post."""
         params = {"page": "post", "s": "view", "id": post_id}
-        response = self.client.get(self.base_url, params=params)
+        response = self._get(self.base_url, params=params)
         response.raise_for_status()
         return PostDetailsParser.parse_html(response.text)
 
     def get_user_profile(self, username: str) -> Optional[UserProfile]:
         """Fetch user profile by username."""
         params = {"page": "account", "s": "profile", "uname": username}
-        response = self.client.get(self.base_url, params=params)
+        response = self._get(self.base_url, params=params)
         response.raise_for_status()
         return UserProfileParser.parse_html(response.text, self.base_url)
 
@@ -170,6 +210,7 @@ def __init__(
         timeout: float = 30.0,
         posts_per_page: int = DEFAULT_POSTS_PER_PAGE,
         headers: Optional[Dict[str, str]] = None,
+        max_retries: int = 5,
     ):
         """Initialize the async client.
 
@@ -178,12 +219,14 @@ def __init__(
             timeout: Request timeout in seconds.
             posts_per_page: Number of posts per page.
             headers: Custom headers to use.
+            max_retries: Maximum number of retry attempts for rate-limited requests.
         """
         self.base_url = base_url.rstrip("/")
         self.posts_per_page = posts_per_page
         self._timeout = timeout
         self._headers = {**DEFAULT_HEADERS, **(headers or {})}
         self._client: Optional[httpx.AsyncClient] = None
+        self._max_retries = max_retries
 
     @property
     def client(self) -> httpx.AsyncClient:
@@ -192,6 +235,21 @@ def client(self) -> httpx.AsyncClient:
             self._client = httpx.AsyncClient(timeout=self._timeout, headers=self._headers)
         return self._client
 
+    async def _get(self, url: str, params: Dict = None) -> httpx.Response:
+        """Execute GET request with retry logic for rate limits."""
+        @retry(
+            stop=stop_after_attempt(self._max_retries),
+            wait=wait_exponential(multiplier=1, min=1.0, max=60.0),
+            retry=retry_if_exception_type((RateLimitError, httpx.TransportError)),
+            before_sleep=before_sleep_log(logger, logging.WARNING),
+            reraise=True,
+        )
+        async def _request() -> httpx.Response:
+            response = await self.client.get(url, params=params)
+            return _check_rate_limit(response)
+        
+        return await _request()
+
     async def close(self) -> None:
         """Close the HTTP client."""
         if self._client:
@@ -209,7 +267,7 @@ async def get_posts(self, tags: str = "", page: int = 1) -> Tuple[List[Post], Li
         offset = (page - 1) * self.posts_per_page
         params = {"page": "post", "s": "list", "tags": tags, "pid": offset}
 
-        response = await self.client.get(self.base_url, params=params)
+        response = await self._get(self.base_url, params=params)
         response.raise_for_status()
 
         html = response.text
@@ -226,14 +284,14 @@ async def search(self, tags: str, page: int = 1) -> List[Post]:
     async def get_post_details(self, post_id: int) -> Optional[PostDetails]:
         """Fetch detailed info for a specific post."""
         params = {"page": "post", "s": "view", "id": post_id}
-        response = await self.client.get(self.base_url, params=params)
+        response = await self._get(self.base_url, params=params)
         response.raise_for_status()
         return PostDetailsParser.parse_html(response.text)
 
     async def get_user_profile(self, username: str) -> Optional[UserProfile]:
         """Fetch user profile by username."""
         params = {"page": "account", "s": "profile", "uname": username}
-        response = await self.client.get(self.base_url, params=params)
+        response = await self._get(self.base_url, params=params)
         response.raise_for_status()
         return UserProfileParser.parse_html(response.text, self.base_url)
 

rule34scraper/client.py

@@ -0,0 +1,141 @@
+"""Integration tests against the live Rule34 API."""
+
+import pytest
+from rule34scraper.client import AsyncR34Client, R34Client
+
+
+class TestSyncClient:
+    """Tests for the synchronous R34Client."""
+
+    def test_get_posts_returns_results(self):
+        """Fetching posts should return a non-empty list."""
+        with R34Client() as client:
+            posts, tags = client.get_posts(tags="solo", page=1)
+
+            assert len(posts) > 0
+            assert len(tags) > 0
+
+    def test_post_has_required_fields(self):
+        """Each post should have all required fields populated."""
+        with R34Client() as client:
+            posts, _ = client.get_posts(tags="solo", page=1)
+            post = posts[0]
+
+            assert post.id > 0
+            assert post.preview_url.startswith("http")
+            assert len(post.tags) > 0
+            assert post.rating in ("safe", "questionable", "explicit")
+            assert post.detail_url.startswith("http")
+
+    def test_search_returns_posts(self):
+        """Search method should return posts matching tags."""
+        with R34Client() as client:
+            posts = client.search(tags="1girl", page=1)
+
+            assert len(posts) > 0
+
+    def test_get_post_details(self):
+        """Fetching post details should return complete info."""
+        with R34Client() as client:
+            posts, _ = client.get_posts(tags="solo", page=1)
+            post_id = posts[0].id
+
+            details = client.get_post_details(post_id)
+
+            assert details is not None
+            assert details.id == post_id
+            assert details.image_url.startswith("http")
+            assert details.width > 0
+            assert details.height > 0
+            assert details.rating in ("safe", "questionable", "explicit")
+
+    def test_get_sidebar_tags(self):
+        """Sidebar tags should have name, count, and type."""
+        with R34Client() as client:
+            tags = client.get_sidebar_tags(tags="solo")
+
+            assert len(tags) > 0
+            tag = tags[0]
+            assert tag.name
+            assert tag.count >= 0
+            assert tag.type
+
+    def test_pagination_returns_different_posts(self):
+        """Different pages should return different posts."""
+        with R34Client() as client:
+            posts_page1, _ = client.get_posts(tags="solo", page=1)
+            posts_page2, _ = client.get_posts(tags="solo", page=2)
+
+            ids_page1 = {p.id for p in posts_page1}
+            ids_page2 = {p.id for p in posts_page2}
+
+            assert ids_page1 != ids_page2
+
+
+@pytest.mark.asyncio
+class TestAsyncClient:
+    """Tests for the asynchronous AsyncR34Client."""
+
+    async def test_get_posts_returns_results(self):
+        """Fetching posts should return a non-empty list."""
+        async with AsyncR34Client() as client:
+            posts, tags = await client.get_posts(tags="solo", page=1)
+
+            assert len(posts) > 0
+            assert len(tags) > 0
+
+    async def test_post_has_required_fields(self):
+        """Each post should have all required fields populated."""
+        async with AsyncR34Client() as client:
+            posts, _ = await client.get_posts(tags="solo", page=1)
+            post = posts[0]
+
+            assert post.id > 0
+            assert post.preview_url.startswith("http")
+            assert len(post.tags) > 0
+            assert post.rating in ("safe", "questionable", "explicit")
+            assert post.detail_url.startswith("http")
+
+    async def test_search_returns_posts(self):
+        """Search method should return posts matching tags."""
+        async with AsyncR34Client() as client:
+            posts = await client.search(tags="1girl", page=1)
+
+            assert len(posts) > 0
+
+    async def test_get_post_details(self):
+        """Fetching post details should return complete info."""
+        async with AsyncR34Client() as client:
+            posts, _ = await client.get_posts(tags="solo", page=1)
+            post_id = posts[0].id
+
+            details = await client.get_post_details(post_id)
+
+            assert details is not None
+            assert details.id == post_id
+            assert details.image_url.startswith("http")
+            assert details.width > 0
+            assert details.height > 0
+            assert details.rating in ("safe", "questionable", "explicit")
+
+    async def test_get_sidebar_tags(self):
+        """Sidebar tags should have name, count, and type."""
+        async with AsyncR34Client() as client:
+            posts, tags = await client.get_posts(tags="solo", page=1)
+
+            assert len(tags) > 0
+            tag = tags[0]
+            assert tag.name
+            assert tag.count >= 0
+            assert tag.type
+
+    async def test_pagination_returns_different_posts(self):
+        """Different pages should return different posts."""
+        async with AsyncR34Client() as client:
+            posts_page1, _ = await client.get_posts(tags="solo", page=1)
+            posts_page2, _ = await client.get_posts(tags="solo", page=2)
+
+            ids_page1 = {p.id for p in posts_page1}
+            ids_page2 = {p.id for p in posts_page2}
+
+            assert ids_page1 != ids_page2