feat: Add test suite for CloudflareBypasser, including cookie generation, HTML retrieval, and server endpoints

Author: sarperavci

.github/workflows/tests.yml

@@ -0,0 +1,174 @@
+name: CloudflareBypassForScraping Tests
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+  workflow_dispatch:  # Allow manual triggers
+  schedule:
+    # Run tests daily at 2 AM UTC
+    - cron: '0 2 * * *'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11']
+      fail-fast: false
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          libnss3 \
+          libnspr4 \
+          libatk1.0-0 \
+          libatk-bridge2.0-0 \
+          libcups2 \
+          libdrm2 \
+          libxkbcommon0 \
+          libxcomposite1 \
+          libxdamage1 \
+          libxfixes3 \
+          libxrandr2 \
+          libgbm1 \
+          libasound2 \
+          libpango-1.0-0 \
+          libcairo2
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        pip install -r requirements.txt
+        pip install -r server_requirements.txt
+        pip install -r tests/test-requirements.txt
+    
+    - name: Show installed packages
+      run: |
+        pip list
+    
+    - name: Run Cookie Generation Tests
+      run: |
+        python -m pytest -c tests/pytest.ini tests/test_cookies.py -v --tb=short --timeout=120
+      timeout-minutes: 30
+    
+    - name: Run HTML Retrieval Tests
+      run: |
+        python -m pytest -c tests/pytest.ini tests/test_html.py -v --tb=short --timeout=120
+      timeout-minutes: 30
+    
+    - name: Start Server for Integration Tests
+      run: |
+        python server.py &
+        SERVER_PID=$!
+        echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV
+        sleep 10
+        curl -f http://localhost:8000/cache/stats || exit 1
+    
+    - name: Run Server Endpoint Tests
+      run: |
+        python -m pytest -c tests/pytest.ini tests/test_server.py -v --tb=short --timeout=120
+      timeout-minutes: 20
+    
+    - name: Run Mirror Request Tests
+      run: |
+        python -m pytest -c tests/pytest.ini tests/test_mirror.py -v --tb=short --timeout=120
+      timeout-minutes: 30
+    
+    - name: Stop Server
+      if: always()
+      run: |
+        if [ ! -z "$SERVER_PID" ]; then
+          kill $SERVER_PID || true
+        fi
+        pkill -f "python.*server.py" || true
+    
+    - name: Generate Test Report
+      if: always()
+      run: |
+        echo "## Test Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "Tests completed for Python ${{ matrix.python-version }}" >> $GITHUB_STEP_SUMMARY
+    
+    - name: Show logs on failure
+      if: failure()
+      run: |
+        echo "=== Recent logs ==="
+        tail -n 100 ~/.local/share/camoufox/logs/* || true
+        echo "=== System info ==="
+        uname -a
+        python --version
+        pip list
+
+  test-quick:
+    name: Quick Tests (Single Request)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+    
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libasound2
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r server_requirements.txt
+        pip install -r tests/test-requirements.txt
+    
+    - name: Run Quick Tests
+      run: |
+        python -m pytest -c tests/pytest.ini tests/test_cookies.py::test_single_cookie_generation -v
+        python -m pytest -c tests/pytest.ini tests/test_html.py::test_single_html_retrieval -v
+      timeout-minutes: 15
+
+  test-parallel:
+    name: Parallel Tests (Concurrency)
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+    
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libasound2
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r server_requirements.txt
+        pip install -r tests/test-requirements.txt
+    
+    - name: Test Parallel Cookie Generation
+      run: |
+        python -m pytest -c tests/pytest.ini tests/test_cookies.py -k "parallel" -v --timeout=120
+      timeout-minutes: 35

tests/__init__.py

@@ -0,0 +1,38 @@
+import pytest
+import asyncio
+from typing import AsyncGenerator
+
+# Test configuration
+TEST_URL = "https://challenge.sarper.me"
+EXPECTED_SUCCESS_TEXT = "cloudflare challenged passed. You are now a free bot."
+TEST_TIMEOUT = 60  # seconds per test
+
+
+@pytest.fixture(scope="session")
+def event_loop():
+    """Create an instance of the default event loop for the test session."""
+    loop = asyncio.get_event_loop_policy().new_event_loop()
+    yield loop
+    loop.close()
+
+
+@pytest.fixture
+def test_url():
+    """Provide the test URL with Cloudflare challenge."""
+    return TEST_URL
+
+
+@pytest.fixture
+def expected_text():
+    """Provide the expected success text after bypass."""
+    return EXPECTED_SUCCESS_TEXT
+
+
+@pytest.fixture
+async def bypasser():
+    """Create a CamoufoxBypasser instance for testing."""
+    from cf_bypasser.core.bypasser import CamoufoxBypasser
+    instance = CamoufoxBypasser(max_retries=5, log=True)
+    yield instance
+    # Cleanup is handled per-request now, but just in case
+    await instance.cleanup()

tests/conftest.py

@@ -0,0 +1,42 @@
+[pytest]
+# Pytest configuration for CloudflareBypassForScraping
+
+# Test discovery patterns
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+
+# Async support
+asyncio_mode = auto
+asyncio_default_fixture_loop_scope = function
+
+# Show extra test summary info
+addopts = 
+    -v
+    --tb=short
+    --strict-markers
+    --disable-warnings
+    -p no:warnings
+    --color=yes
+    --durations=10
+    -p no:seleniumbase
+
+# Test markers
+markers =
+    asyncio: marks tests as async (deselect with '-m "not asyncio"')
+    slow: marks tests as slow (deselect with '-m "not slow"')
+    parallel: marks tests that run in parallel
+    
+# Logging
+log_cli = true
+log_cli_level = INFO
+log_cli_format = %(asctime)s [%(levelname)8s] %(message)s
+log_cli_date_format = %H:%M:%S
+
+# Coverage options
+[coverage:run]
+source = cf_bypasser
+omit = 
+    */tests/*
+    */test_*.py
+    */__pycache__/*

tests/pytest.ini

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+Runs all tests
+"""
+import subprocess
+import sys
+import time
+import signal
+import os
+from pathlib import Path
+
+
+def print_banner(text):
+    """Print a formatted banner."""
+    print("\n" + "=" * 80)
+    print(f"  {text}")
+    print("=" * 80 + "\n")
+
+
+def start_server():
+    """Start the server in the background."""
+    print("Starting server...")
+    
+    # Start server process
+    process = subprocess.Popen(
+        [sys.executable, "server.py"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        cwd=Path(__file__).parent.parent
+    )
+    
+    # Wait for server to start
+    print("⏳ Waiting for server to initialize...")
+    time.sleep(5)
+    
+    # Check if server started successfully
+    if process.poll() is not None:
+        print("❌ Server failed to start!")
+        return None
+    
+    print("✅ Server started successfully\n")
+    return process
+
+
+def stop_server(process):
+    """Stop the server process."""
+    if process:
+        print("\n🛑 Stopping server...")
+        process.send_signal(signal.SIGTERM)
+        try:
+            process.wait(timeout=10)
+            print("✅ Server stopped\n")
+        except subprocess.TimeoutExpired:
+            print("⚠️ Server didn't stop gracefully, forcing...")
+            process.kill()
+            process.wait()
+
+
+def run_tests(test_args=None):
+    """Run pytest with specified arguments."""
+    cmd = [sys.executable, "-m", "pytest"]
+    
+    if test_args:
+        cmd.extend(test_args)
+    else:
+        # Default: run all tests with verbose output
+        cmd.extend([
+            "-c", "tests/pytest.ini",
+            "tests/",
+            "-v",
+            "--tb=short",
+            "--color=yes"
+        ])
+    
+    print_banner("Running Tests")
+    print(f"Command: {' '.join(cmd)}\n")
+    
+    result = subprocess.run(cmd, cwd=Path(__file__).parent.parent)
+    return result.returncode
+
+
+def main():
+    """Main test runner."""
+    print_banner("CloudflareBypassForScraping Test Suite")
+    
+    # Check if pytest is installed
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "pytest", "--version"],
+            check=True,
+            capture_output=True
+        )
+    except subprocess.CalledProcessError:
+        print("pytest is not installed!")
+        print("Install test requirements with: pip install -r tests/test-requirements.txt")
+        return 1
+    
+    server_process = None
+    
+    try:
+        # Start server
+        server_process = start_server()
+        if not server_process:
+            return 1
+        
+        # Run tests
+        test_args = sys.argv[1:] if len(sys.argv) > 1 else None
+        exit_code = run_tests(test_args)
+        
+        # Print summary
+        print_banner("Test Results")
+        if exit_code == 0:
+            print("All tests passed!\n")
+        else:
+            print(f"Tests failed with exit code {exit_code}\n")
+        
+        return exit_code
+        
+    except KeyboardInterrupt:
+        print("\n\n Tests interrupted by user")
+        return 130
+        
+    except Exception as e:
+        print(f"\n Error running tests: {e}")
+        return 1
+        
+    finally:
+        # Always stop the server
+        stop_server(server_process)
+
+
+if __name__ == "__main__":
+    sys.exit(main())