feat: enhance proxy scraper with additional sources and improve logging system

Author: ArnabXD

CLAUDE.md

@@ -65,8 +65,8 @@ docker-compose up
 
 ### Core Components
 
-- **Scraper** (`pkg/scraper/`): Fetches proxy lists from multiple sources (ProxyScrape, FreeProxyList)
-- **Checker** (`pkg/checker/`): Validates proxy health with SQLite caching and intelligent check intervals
+- **Scraper** (`pkg/scraper/`): Fetches proxy lists from multiple sources (ProxyScrape, FreeProxyList, Geonode, ProxyListOrg, GitHub)
+- **Checker** (`pkg/checker/`): Validates proxy health with SQLite caching, intelligent check intervals, and unified logging
 - **Manager** (`pkg/manager/`): Manages proxy pool with database persistence, in-memory cache, and auto-refresh
 - **Database** (`internal/database/`): SQLite-based persistent storage with Jet ORM for type-safe queries
 - **Proxy Server** (`pkg/proxy/`): HTTP/HTTPS proxy server with privacy features
@@ -121,6 +121,13 @@ AProxy uses **Viper** for advanced configuration management with validation:
 3. **Config files**: YAML, JSON, TOML supported (searches `./`, `./config/`, `/etc/aproxy/`)
 4. **Defaults**: Sensible defaults for all settings
 
+**Supported Scraper Sources:**
+- `proxyscrape`: ProxyScrape API
+- `freeproxylist`: FreeProxyList scraper
+- `geonode`: Geonode API scraper  
+- `proxylistorg`: ProxyListOrg scraper
+- `github`: GitHub proxy list scraper (proxifly/free-proxy-list)
+
 **Configuration Management:**
 ```bash
 # Generate sample config file
@@ -172,6 +179,11 @@ curl -x http://localhost:8080 \
 - `checker.max_workers`: Concurrent health check workers (default: `50`)
 - `checker.test_url`: URL used to test proxy health (default: `http://icanhazip.com`)
 
+**Scraper Configuration Options:**
+- `scraper.sources`: List of proxy sources to use (default: `["proxyscrape", "freeproxylist", "geonode", "github"]`)
+- `scraper.timeout`: Request timeout for scraping (default: `30s`)
+- `scraper.user_agent`: User agent string for scraper requests
+
 **Logging Configuration:**
 - Currently logs to stdout only in JSON format
 - Log level can be controlled via command line or environment variables
@@ -205,6 +217,17 @@ The SQLite database includes:
 
 ## Recent Improvements
 
+### GitHub Proxy Scraper (v1.2)
+- **New GitHub source**: Added scraper for proxifly/free-proxy-list GitHub repository
+- **Enhanced source variety**: Now supports 5 different proxy sources for better diversity
+- **Configuration validation**: Added `github` to allowed scraper sources in config validation
+
+### Logging System Improvements (v1.2)
+- **Unified logging**: Standardized all checker logging to use internal logger package
+- **Reduced verbosity**: Removed verbose individual proxy failure debugging output
+- **Consistent log levels**: Proper use of InfoBg/WarnBg throughout checker components
+- **Better performance**: Less logging overhead during proxy health checks
+
 ### Configuration System (v1.1)
 - **Migrated to Viper**: Replaced manual config parsing with Viper library
 - **Added validation**: All config values validated using `go-playground/validator`

README.md

@@ -110,7 +110,7 @@ docker-compose logs -f
 
 ## How It Works
 
-1. **Scraper** fetches proxy lists from multiple free sources (ProxyScrape, FreeProxyList)
+1. **Scraper** fetches proxy lists from multiple free sources (ProxyScrape, FreeProxyList, Geonode, ProxyListOrg, GitHub)
 2. **Health Checker** validates proxies using configurable test URLs
 3. **Database** caches proxy health status to avoid redundant checks
 4. **Manager** maintains pool of healthy proxies with automatic rotation
@@ -160,6 +160,12 @@ docker-compose logs -f
 - `checker.timeout` - Proxy test timeout (default: `15s`)
 - `checker.batch_size` - Proxies per batch (default: `50`)
 - `checker.batch_delay` - Delay between batches (default: `30s`)
+- `checker.background_enabled` - Enable background proxy checking (default: `true`)
+
+### Scraper Sources
+- `scraper.sources` - Proxy sources to use: `proxyscrape`, `freeproxylist`, `geonode`, `proxylistorg`, `github`
+- `scraper.timeout` - Scraper request timeout (default: `30s`)
+- `scraper.user_agent` - User agent for scraping requests
 
 ### Database
 - `database.path` - SQLite file location (default: `./data/aproxy.db`)

internal/config/config.go

@@ -87,7 +87,7 @@ func setDefaults() {
 	// Scraper defaults
 	viper.SetDefault("scraper.timeout", "30s")
 	viper.SetDefault("scraper.user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
-	viper.SetDefault("scraper.sources", []string{"proxyscrape", "freeproxylist", "geonode"})
+	viper.SetDefault("scraper.sources", []string{"proxyscrape", "freeproxylist", "geonode", "github"})
 
 	// Checker defaults
 	viper.SetDefault("checker.test_url", "http://icanhazip.com")

pkg/checker/checker.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"aproxy/internal/logger"
 	"aproxy/pkg/scraper"
 	netproxy "golang.org/x/net/proxy"
 )
@@ -53,6 +54,7 @@ type Checker struct {
 	timeout    time.Duration
 	maxWorkers int
 	userAgent  string
+	logger     *logger.Logger
 }
 
 type CheckerConfig struct {
@@ -68,6 +70,7 @@ func NewChecker() *Checker {
 		timeout:    20 * time.Second,
 		maxWorkers: 20,
 		userAgent:  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+		logger:     logger.New("checker"),
 	}
 }
 
@@ -77,6 +80,7 @@ func NewCheckerWithConfig(config CheckerConfig) *Checker {
 		timeout:    config.Timeout,
 		maxWorkers: config.MaxWorkers,
 		userAgent:  config.UserAgent,
+		logger:     logger.New("checker"),
 	}
 }
 
@@ -150,35 +154,16 @@ func (c *Checker) CheckProxies(ctx context.Context, proxies []scraper.Proxy) []C
 
 	var results []CheckResult
 	healthyCount := 0
-	failureCounts := make(map[string]int)
 
 	for result := range resultQueue {
 		results = append(results, result)
 		if result.Status == StatusHealthy {
 			healthyCount++
-		} else {
-			// Count failures by type
-			errType := result.Status.String()
-			if result.Error != nil && strings.Contains(result.Error.Error(), "SOCKS proxy not supported") {
-				errType = "socks_skipped"
-			}
-			failureCounts[errType]++
-
-			// Log first few failures for debugging (but not SOCKS)
-			if errType != "socks_skipped" && failureCounts[errType] <= 3 {
-				fmt.Printf("DEBUG: Proxy %s (%s) failed: %s (error: %v)\n",
-					result.Proxy.Address(), result.Proxy.Type, result.Status.String(), result.Error)
-			}
 		}
 	}
 
-	// Summary of results
 	if len(results) > 0 {
-		fmt.Printf("DEBUG: Results - Healthy: %d", healthyCount)
-		for errType, count := range failureCounts {
-			fmt.Printf(", %s: %d", errType, count)
-		}
-		fmt.Printf(" (total: %d)\n", len(results))
+		c.logger.InfoBg("Proxy check completed: %d healthy out of %d total", healthyCount, len(results))
 	}
 
 	return results
@@ -471,11 +456,11 @@ func TestSingleProxy(host string, port int) {
 	checker := NewChecker()
 	ctx := context.Background()
 
-	fmt.Printf("Testing proxy %s:%d...\n", host, port)
+	checker.logger.InfoBg("Testing proxy %s:%d", host, port)
 	result := checker.CheckProxy(ctx, proxy)
 
-	fmt.Printf("Result: %s (took %v)\n", result.Status.String(), result.ResponseTime)
+	checker.logger.InfoBg("Result: %s (took %v)", result.Status.String(), result.ResponseTime)
 	if result.Error != nil {
-		fmt.Printf("Error: %v\n", result.Error)
+		checker.logger.WarnBg("Error: %v", result.Error)
 	}
 }

pkg/checker/db_checker.go

@@ -3,11 +3,11 @@ package checker
 import (
 	"context"
 	"fmt"
-	"log"
 	"time"
 
 	"aproxy/internal/database"
 	"aproxy/internal/database/models/model"
+	"aproxy/internal/logger"
 	"aproxy/pkg/scraper"
 )
 
@@ -18,6 +18,7 @@ type DBChecker struct {
 	checkInterval time.Duration
 	batchSize     int
 	batchDelay    time.Duration
+	logger        *logger.Logger
 }
 
 // NewDBChecker creates a new database-backed checker
@@ -28,6 +29,7 @@ func NewDBChecker(dbService *database.Service, checkInterval time.Duration, batc
 		checkInterval: checkInterval,
 		batchSize:     batchSize,
 		batchDelay:    batchDelay,
+		logger:        logger.New("db-checker"),
 	}
 }
 
@@ -39,6 +41,7 @@ func NewDBCheckerWithConfig(dbService *database.Service, checkerConfig CheckerCo
 		checkInterval: checkInterval,
 		batchSize:     batchSize,
 		batchDelay:    batchDelay,
+		logger:        logger.New("db-checker"),
 	}
 }
 
@@ -48,7 +51,7 @@ func (c *DBChecker) CheckProxiesWithCaching(ctx context.Context, proxies []scrap
 		return nil
 	}
 
-	log.Printf("Checking %d proxies with caching (skip if checked within %v)", len(proxies), c.checkInterval)
+	c.logger.InfoBg("Checking %d proxies with caching (skip if checked within %v)", len(proxies), c.checkInterval)
 
 	// Get addresses of all scraped proxies
 	addresses := make([]string, len(proxies))
@@ -62,12 +65,12 @@ func (c *DBChecker) CheckProxiesWithCaching(ctx context.Context, proxies []scrap
 	// Get existing proxies from database
 	existingProxies, err := c.dbService.GetProxiesByAddresses(ctx, addresses)
 	if err != nil {
-		log.Printf("Failed to get existing proxies: %v", err)
+		c.logger.WarnBg("Failed to get existing proxies: %v", err)
 		// Fall back to checking all proxies
 		return c.Checker.CheckProxies(ctx, proxies)
 	}
 
-	log.Printf("Found %d existing proxies in database", len(existingProxies))
+	c.logger.InfoBg("Found %d existing proxies in database", len(existingProxies))
 
 	// Separate new proxies that need to be inserted vs existing ones
 	var newProxies []scraper.Proxy
@@ -88,7 +91,7 @@ func (c *DBChecker) CheckProxiesWithCaching(ctx context.Context, proxies []scrap
 	for _, proxy := range newProxies {
 		dbProxy, err := c.dbService.UpsertProxy(ctx, proxy)
 		if err != nil {
-			log.Printf("Failed to upsert new proxy %s: %v", proxy.Address(), err)
+			c.logger.WarnBg("Failed to upsert new proxy %s: %v", proxy.Address(), err)
 			continue
 		}
 		dbProxies = append(dbProxies, dbProxy)
@@ -123,7 +126,7 @@ func (c *DBChecker) CheckProxiesWithCaching(ctx context.Context, proxies []scrap
 		}
 	}
 
-	log.Printf("Found %d proxies that need checking (out of %d total)", len(proxiesToCheck), len(dbProxies))
+	c.logger.InfoBg("Found %d proxies that need checking (out of %d total)", len(proxiesToCheck), len(dbProxies))
 
 	if len(proxiesToCheck) == 0 {
 		// All proxies have been checked recently, return cached results
@@ -146,7 +149,7 @@ func (c *DBChecker) CheckProxiesWithCaching(ctx context.Context, proxies []scrap
 		proxyAddr := result.Proxy.Address()
 		dbProxy, exists := proxyMap[proxyAddr]
 		if !exists {
-			log.Printf("No database proxy found for %s", proxyAddr)
+			c.logger.WarnBg("No database proxy found for %s", proxyAddr)
 			continue
 		}
 
@@ -188,9 +191,9 @@ func (c *DBChecker) CheckProxiesWithCaching(ctx context.Context, proxies []scrap
 			// Use longer timeout for database operations
 			updateCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 			if err := c.dbService.BatchUpdateProxyHealth(updateCtx, batchUpdates); err != nil {
-				log.Printf("Failed to batch update proxy health (batch %d-%d): %v", i, end-1, err)
+				c.logger.WarnBg("Failed to batch update proxy health (batch %d-%d): %v", i, end-1, err)
 			} else {
-				log.Printf("Successfully updated %d proxy health records to database", len(batchUpdates))
+				c.logger.InfoBg("Successfully updated %d proxy health records to database", len(batchUpdates))
 			}
 			cancel()
 		}
@@ -348,14 +351,14 @@ func (c *DBChecker) checkProxiesProgressive(ctx context.Context, proxies []scrap
 	var allResults []CheckResult
 	totalBatches := (len(proxies) + c.batchSize - 1) / c.batchSize
 
-	log.Printf("Checking %d proxies in %d batches (batch size: %d, delay: %v)",
+	c.logger.InfoBg("Checking %d proxies in %d batches (batch size: %d, delay: %v)",
 		len(proxies), totalBatches, c.batchSize, c.batchDelay)
 
 	for i := 0; i < len(proxies); i += c.batchSize {
 		// Check for cancellation before starting each batch
 		select {
 		case <-ctx.Done():
-			log.Printf("Context cancelled before batch %d/%d, stopping progressive checking", i/c.batchSize+1, totalBatches)
+			c.logger.WarnBg("Context cancelled before batch %d/%d, stopping progressive checking", i/c.batchSize+1, totalBatches)
 			return allResults
 		default:
 			// Continue with batch
@@ -369,7 +372,7 @@ func (c *DBChecker) checkProxiesProgressive(ctx context.Context, proxies []scrap
 		batch := proxies[i:end]
 		batchNum := i/c.batchSize + 1
 
-		log.Printf("Checking batch %d/%d (%d proxies)", batchNum, totalBatches, len(batch))
+		c.logger.InfoBg("Checking batch %d/%d (%d proxies)", batchNum, totalBatches, len(batch))
 
 		// Check batch using original checker
 		batchResults := c.Checker.CheckProxies(ctx, batch)
@@ -380,7 +383,7 @@ func (c *DBChecker) checkProxiesProgressive(ctx context.Context, proxies []scrap
 			select {
 			case <-ctx.Done():
 				// Context cancelled, skip background save
-				log.Printf("Context cancelled, skipping database save for batch %d", batchNum)
+				c.logger.WarnBg("Context cancelled, skipping database save for batch %d", batchNum)
 			default:
 				// Context still active, save in background
 				go func(results []CheckResult, batchNumber int) {
@@ -409,7 +412,7 @@ func (c *DBChecker) checkProxiesProgressive(ctx context.Context, proxies []scrap
 						}
 						if len(updates) > 0 {
 							if err := c.dbService.BatchUpdateProxyHealth(saveCtx, updates); err == nil {
-								log.Printf("Saved batch %d results to database (%d records)", batchNumber, len(updates))
+								c.logger.InfoBg("Saved batch %d results to database (%d records)", batchNumber, len(updates))
 							}
 						}
 					}
@@ -424,13 +427,13 @@ func (c *DBChecker) checkProxiesProgressive(ctx context.Context, proxies []scrap
 				healthyCount++
 			}
 		}
-		log.Printf("Batch %d/%d complete. Total healthy so far: %d", batchNum, totalBatches, healthyCount)
+		c.logger.InfoBg("Batch %d/%d complete. Total healthy so far: %d", batchNum, totalBatches, healthyCount)
 
 		// Add delay between batches (except for the last one)
 		if end < len(proxies) {
 			select {
 			case <-ctx.Done():
-				log.Printf("Context cancelled, stopping progressive checking at batch %d/%d with %d healthy proxies found", batchNum, totalBatches, healthyCount)
+				c.logger.WarnBg("Context cancelled, stopping progressive checking at batch %d/%d with %d healthy proxies found", batchNum, totalBatches, healthyCount)
 				return allResults
 			case <-time.After(c.batchDelay):
 				// Continue to next batch
@@ -444,7 +447,7 @@ func (c *DBChecker) checkProxiesProgressive(ctx context.Context, proxies []scrap
 			healthyCount++
 		}
 	}
-	log.Printf("Progressive checking completed: checked %d proxies in %d batches, found %d healthy", len(proxies), totalBatches, healthyCount)
+	c.logger.InfoBg("Progressive checking completed: checked %d proxies in %d batches, found %d healthy", len(proxies), totalBatches, healthyCount)
 	return allResults
 }