diff --git a/client.go b/client.go index 9591e55..f393e95 100644 --- a/client.go +++ b/client.go @@ -13,9 +13,62 @@ type Error struct { Func string } +// ProxyNetwork defines the network layer (IPv4/IPv6) a proxy can support +type ProxyNetwork int + +const ( + // ProxyNetworkUnset is the zero value and must not be used - forces explicit selection + ProxyNetworkUnset ProxyNetwork = iota + // ProxyNetworkAny means the proxy can be used for both IPv4 and IPv6 connections + ProxyNetworkAny + // ProxyNetworkIPv4 means the proxy should only be used for IPv4 connections + ProxyNetworkIPv4 + // ProxyNetworkIPv6 means the proxy should only be used for IPv6 connections + ProxyNetworkIPv6 +) + +// ProxyType defines the infrastructure type of a proxy +type ProxyType int + +const ( + // ProxyTypeAny means the proxy can be used for any type of request + ProxyTypeAny ProxyType = iota + // ProxyTypeMobile means the proxy uses mobile network infrastructure + ProxyTypeMobile + // ProxyTypeResidential means the proxy uses residential IP addresses + ProxyTypeResidential + // ProxyTypeDatacenter means the proxy uses datacenter infrastructure + ProxyTypeDatacenter +) + +// ProxyConfig defines the configuration for a single proxy +type ProxyConfig struct { + // URL is the proxy URL (e.g., "socks5://proxy.example.com:1080") + URL string + // Network specifies if this proxy supports IPv4, IPv6, or both + Network ProxyNetwork + // Type specifies the infrastructure type (Mobile, Residential, Datacenter, or Any) + Type ProxyType + // AllowedDomains is a list of glob patterns for domains this proxy should handle + // Examples: "*.example.com", "api.*.org" + // If empty, the proxy can be used for any domain + AllowedDomains []string +} + +// ProxyStats holds statistics for a single proxy +type ProxyStats struct { + // RequestCount is the total number of requests made through this proxy + RequestCount atomic.Int64 + // ErrorCount is the number of failed requests/connections through this proxy + ErrorCount atomic.Int64 + // LastUsed is when this proxy was last selected (Unix nanoseconds) + LastUsed atomic.Int64 +} + type HTTPClientSettings struct { RotatorSettings *RotatorSettings - Proxy string + Proxies []ProxyConfig + AllowDirectFallback bool TempDir string DiscardHook DiscardHook DNSServers []string @@ -73,6 +126,9 @@ type CustomHTTPClient struct { CDXDedupeTotal *atomic.Int64 DoppelgangerDedupeTotal *atomic.Int64 LocalDedupeTotal *atomic.Int64 + + // ProxyStats holds per-proxy statistics, keyed by proxy URL + ProxyStats map[string]*ProxyStats } func (c *CustomHTTPClient) Close() error { @@ -103,6 +159,11 @@ func (c *CustomHTTPClient) Close() error { return nil } +// GetProxyStats returns a copy of the per-proxy statistics map +func (c *CustomHTTPClient) GetProxyStats() map[string]*ProxyStats { + return c.ProxyStats +} + func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, err error) { httpClient = new(CustomHTTPClient) @@ -216,7 +277,7 @@ func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient httpClient.ConnReadDeadline = HTTPClientSettings.ConnReadDeadline // Configure custom dialer / transport - customDialer, err := newCustomDialer(httpClient, HTTPClientSettings.Proxy, HTTPClientSettings.DialTimeout, HTTPClientSettings.DNSRecordsTTL, HTTPClientSettings.DNSResolutionTimeout, HTTPClientSettings.DNSCacheSize, HTTPClientSettings.DNSServers, HTTPClientSettings.DNSConcurrency, HTTPClientSettings.DisableIPv4, HTTPClientSettings.DisableIPv6) + customDialer, err := newCustomDialer(httpClient, HTTPClientSettings.Proxies, HTTPClientSettings.AllowDirectFallback, HTTPClientSettings.DialTimeout, HTTPClientSettings.DNSRecordsTTL, HTTPClientSettings.DNSResolutionTimeout, HTTPClientSettings.DNSCacheSize, HTTPClientSettings.DNSServers, HTTPClientSettings.DNSConcurrency, HTTPClientSettings.DisableIPv4, HTTPClientSettings.DisableIPv6) if err != nil { return nil, err } diff --git a/client_test.go b/client_test.go index 99ef81e..bdd2b80 100644 --- a/client_test.go +++ b/client_test.go @@ -688,7 +688,13 @@ func TestHTTPClientWithProxy(t *testing.T) { // init the HTTP client responsible for recording HTTP(s) requests / responses httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{ RotatorSettings: rotatorSettings, - Proxy: fmt.Sprintf("socks5://%s", proxyAddr)}) + Proxies: []ProxyConfig{ + { + URL: fmt.Sprintf("socks5://%s", proxyAddr), + Network: ProxyNetworkAny, + Type: ProxyTypeAny, + }, + }}) if err != nil { t.Fatalf("Unable to init WARC writing HTTP client: %s", err) } diff --git a/dialer.go b/dialer.go index c642cc5..2f42ec4 100644 --- a/dialer.go +++ b/dialer.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "net/url" + "path/filepath" "slices" "strconv" "strings" @@ -39,6 +40,12 @@ const ( // This is used internally to retrieve the wrapped connection for advanced use cases. // Use WithWrappedConnection() helper function for convenience. ContextKeyWrappedConn contextKey = "wrappedConn" + + // ContextKeyProxyType is the context key for requesting a specific proxy type. + // External callers (like Zeno) can set this to request a proxy of a specific type + // (Mobile, Residential, or Datacenter). + // Use WithProxyType() helper function for convenience. + ContextKeyProxyType contextKey = "proxyType" ) // WithFeedbackChannel adds a feedback channel to the request context. @@ -62,23 +69,47 @@ func WithWrappedConnection(ctx context.Context, wrappedConnChan chan *CustomConn return context.WithValue(ctx, ContextKeyWrappedConn, wrappedConnChan) } +// WithProxyType adds a proxy type preference to the request context. +// When set, the proxy selector will prefer proxies of the specified type +// (Mobile, Residential, or Datacenter). +// This is typically used by external callers like Zeno when they need a specific proxy type. +// +// Example: +// +// req = req.WithContext(warc.WithProxyType(req.Context(), warc.ProxyTypeMobile)) +func WithProxyType(ctx context.Context, proxyType ProxyType) context.Context { + return context.WithValue(ctx, ContextKeyProxyType, proxyType) +} + // dnsExchanger is an interface for DNS clients that can exchange messages type dnsExchanger interface { ExchangeContext(ctx context.Context, m *dns.Msg, address string) (r *dns.Msg, rtt time.Duration, err error) } +// proxyDialerInfo holds information about a configured proxy +type proxyDialerInfo struct { + dialer proxy.ContextDialer + needsHostname bool // true if proxy requires hostname (socks5h, http), false if can use IP (socks5) + proxyNetwork ProxyNetwork + proxyType ProxyType + allowedDomains []string // glob patterns + url string + stats *ProxyStats +} + type customDialer struct { - proxyDialer proxy.ContextDialer - proxyNeedsHostname bool // true if proxy requires hostname (socks5h, http), false if can use IP (socks5) - client *CustomHTTPClient - DNSConfig *dns.ClientConfig - DNSClient dnsExchanger - DNSRecords *otter.Cache[string, net.IP] + proxyDialers []proxyDialerInfo + proxyRoundRobinIndex atomic.Uint32 + allowDirectFallback bool + client *CustomHTTPClient + DNSConfig *dns.ClientConfig + DNSClient dnsExchanger + DNSRecords *otter.Cache[string, net.IP] net.Dialer - disableIPv4 bool - disableIPv6 bool - dnsConcurrency int - dnsRoundRobinIndex atomic.Uint32 + disableIPv4 bool + disableIPv6 bool + dnsConcurrency int + dnsRoundRobinIndex atomic.Uint32 } var emptyPayloadDigests = []string{ @@ -88,7 +119,7 @@ var emptyPayloadDigests = []string{ "blake3:af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", } -func newCustomDialer(httpClient *CustomHTTPClient, proxyURL string, DialTimeout, DNSRecordsTTL, DNSResolutionTimeout time.Duration, DNSCacheSize int, DNSServers []string, DNSConcurrency int, disableIPv4, disableIPv6 bool) (d *customDialer, err error) { +func newCustomDialer(httpClient *CustomHTTPClient, proxies []ProxyConfig, allowDirectFallback bool, DialTimeout, DNSRecordsTTL, DNSResolutionTimeout time.Duration, DNSCacheSize int, DNSServers []string, DNSConcurrency int, disableIPv4, disableIPv6 bool) (d *customDialer, err error) { d = new(customDialer) d.Timeout = DialTimeout @@ -96,6 +127,7 @@ func newCustomDialer(httpClient *CustomHTTPClient, proxyURL string, DialTimeout, d.disableIPv4 = disableIPv4 d.disableIPv6 = disableIPv6 d.dnsConcurrency = DNSConcurrency + d.allowDirectFallback = allowDirectFallback DNScache, err := otter.MustBuilder[string, net.IP](DNSCacheSize). // CollectStats(). // Uncomment this line to enable stats collection, can be useful later on @@ -121,29 +153,165 @@ func newCustomDialer(httpClient *CustomHTTPClient, proxyURL string, DialTimeout, Timeout: DNSResolutionTimeout, } - if proxyURL != "" { - u, err := url.Parse(proxyURL) + // Initialize proxy stats map + httpClient.ProxyStats = make(map[string]*ProxyStats) + + // Initialize all proxies + for _, proxyConfig := range proxies { + if proxyConfig.URL == "" { + continue + } + + // Validate that Network is explicitly set + if proxyConfig.Network == ProxyNetworkUnset { + return nil, fmt.Errorf("proxy %s: Network must be explicitly set to ProxyNetworkAny, ProxyNetworkIPv4, or ProxyNetworkIPv6", proxyConfig.URL) + } + + u, err := url.Parse(proxyConfig.URL) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse proxy URL %s: %w", proxyConfig.URL, err) } var proxyDialer proxy.Dialer if proxyDialer, err = proxy.FromURL(u, d); err != nil { - return nil, err + return nil, fmt.Errorf("failed to create proxy from URL %s: %w", proxyConfig.URL, err) } - d.proxyDialer = proxyDialer.(proxy.ContextDialer) - // Determine if this proxy requires hostname (remote DNS) or can use IP (local DNS) // Proxies with remote DNS: socks5h, socks4a, http, https // Proxies with local DNS: socks5, socks4 - d.proxyNeedsHostname = u.Scheme == "socks5h" || u.Scheme == "socks4a" || + needsHostname := u.Scheme == "socks5h" || u.Scheme == "socks4a" || u.Scheme == "http" || u.Scheme == "https" + + // Create and initialize stats for this proxy + stats := &ProxyStats{} + httpClient.ProxyStats[proxyConfig.URL] = stats + + d.proxyDialers = append(d.proxyDialers, proxyDialerInfo{ + dialer: proxyDialer.(proxy.ContextDialer), + needsHostname: needsHostname, + proxyNetwork: proxyConfig.Network, + proxyType: proxyConfig.Type, + allowedDomains: proxyConfig.AllowedDomains, + url: proxyConfig.URL, + stats: stats, + }) } return d, nil } +// selectProxy selects an appropriate proxy based on network type, domain, and context flags. +// Returns nil if no proxy is available or should be used (direct connection). +// Returns an error if proxies exist but none match the requirements and direct fallback is disabled. +func (d *customDialer) selectProxy(ctx context.Context, network, address string) (*proxyDialerInfo, error) { + // No proxies configured, use direct connection + if len(d.proxyDialers) == 0 { + return nil, nil + } + + // Extract hostname from address for domain matching + hostname, _, err := net.SplitHostPort(address) + if err != nil { + // If no port, treat the whole address as hostname + hostname = address + } + + // Check if a specific proxy type is requested via context + var requestedProxyType *ProxyType + if ctx.Value(ContextKeyProxyType) != nil { + if val, ok := ctx.Value(ContextKeyProxyType).(ProxyType); ok { + requestedProxyType = &val + } + } + + // Filter eligible proxies + var eligible []*proxyDialerInfo + for i := range d.proxyDialers { + proxy := &d.proxyDialers[i] + + // Filter by proxy type (Mobile, Residential, Datacenter) + // If a specific type is requested, only use matching proxies + // If no type is requested, only use ProxyTypeAny proxies + if requestedProxyType != nil { + // Specific type requested: only match that exact type + if proxy.proxyType != *requestedProxyType { + continue + } + } else { + // No type requested: only use ProxyTypeAny proxies + if proxy.proxyType != ProxyTypeAny { + continue + } + } + + // Filter by network type (IPv4/IPv6) + switch proxy.proxyNetwork { + case ProxyNetworkIPv4: + if strings.HasSuffix(network, "6") { + continue // Skip IPv6 networks + } + case ProxyNetworkIPv6: + if strings.HasSuffix(network, "4") { + continue // Skip IPv4 networks + } + case ProxyNetworkAny: + // Always eligible regardless of network type + } + + // Filter by domain patterns + if len(proxy.allowedDomains) > 0 { + matched := false + for _, pattern := range proxy.allowedDomains { + // Use filepath.Match for glob pattern matching + // Note: filepath.Match doesn't support '**' but supports '*' and '?' + if match, _ := filepath.Match(pattern, hostname); match { + matched = true + break + } + // Also check if pattern matches a subdomain pattern + // For example, "*.example.com" should match "api.example.com" + if strings.HasPrefix(pattern, "*.") { + suffix := pattern[1:] // Remove the leading '*' + if strings.HasSuffix(hostname, suffix) || hostname == suffix[1:] { + matched = true + break + } + } + } + if !matched { + continue // Skip if domain doesn't match any pattern + } + } + + eligible = append(eligible, proxy) + } + + // No eligible proxies found + if len(eligible) == 0 { + if d.allowDirectFallback { + return nil, nil // Use direct connection + } + proxyTypeStr := "any" + if requestedProxyType != nil { + proxyTypeStr = fmt.Sprintf("%v", *requestedProxyType) + } + return nil, fmt.Errorf("no eligible proxies found for network=%s, address=%s, proxyType=%s and direct fallback is disabled", network, address, proxyTypeStr) + } + + // Round-robin selection among eligible proxies + startIdx := int(d.proxyRoundRobinIndex.Add(1)-1) % len(eligible) + selectedProxy := eligible[startIdx] + + // Update proxy statistics + if selectedProxy.stats != nil { + selectedProxy.stats.RequestCount.Add(1) + selectedProxy.stats.LastUsed.Store(time.Now().UnixNano()) + } + + return selectedProxy, nil +} + type CustomConnection struct { net.Conn io.Reader @@ -235,10 +403,16 @@ func (d *customDialer) CustomDialContext(ctx context.Context, network, address s return nil, errors.New("no supported network type available") } + // Select appropriate proxy based on context, network type, and domain + selectedProxy, err := d.selectProxy(ctx, network, address) + if err != nil { + return nil, err + } + var dialAddr string var IP net.IP - if d.proxyDialer != nil && d.proxyNeedsHostname { + if selectedProxy != nil && selectedProxy.needsHostname { // Remote DNS proxy (socks5h, socks4a, http, https) // Skip DNS archiving to avoid privacy leak and ensure accuracy. // The proxy will handle DNS resolution on its end, and we don't want to: @@ -263,8 +437,11 @@ func (d *customDialer) CustomDialContext(ctx context.Context, network, address s dialAddr = net.JoinHostPort(IP.String(), port) } - if d.proxyDialer != nil { - conn, err = d.proxyDialer.DialContext(ctx, network, dialAddr) + if selectedProxy != nil { + conn, err = selectedProxy.dialer.DialContext(ctx, network, dialAddr) + if err != nil && selectedProxy.stats != nil { + selectedProxy.stats.ErrorCount.Add(1) + } } else { if d.client.randomLocalIP { localAddr := getLocalAddr(network, IP) @@ -299,11 +476,16 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres return nil, errors.New("no supported network type available") } + // Select appropriate proxy based on context, network type, and domain + selectedProxy, err := d.selectProxy(ctx, network, address) + if err != nil { + return nil, err + } + var dialAddr string var IP net.IP - var err error - if d.proxyDialer != nil && d.proxyNeedsHostname { + if selectedProxy != nil && selectedProxy.needsHostname { // Remote DNS proxy (socks5h, socks4a, http, https) // Skip DNS archiving to avoid privacy leak and ensure accuracy. // The proxy will handle DNS resolution on its end, and we don't want to: @@ -330,8 +512,11 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres var plainConn net.Conn - if d.proxyDialer != nil { - plainConn, err = d.proxyDialer.DialContext(ctx, network, dialAddr) + if selectedProxy != nil { + plainConn, err = selectedProxy.dialer.DialContext(ctx, network, dialAddr) + if err != nil && selectedProxy.stats != nil { + selectedProxy.stats.ErrorCount.Add(1) + } } else { if d.client.randomLocalIP { localAddr := getLocalAddr(network, IP) @@ -367,6 +552,10 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres defer cancel() if err := tlsConn.HandshakeContext(handshakeCtx); err != nil { + // Track TLS handshake errors for proxy connections + if selectedProxy != nil && selectedProxy.stats != nil { + selectedProxy.stats.ErrorCount.Add(1) + } closeErr := plainConn.Close() if closeErr != nil { return nil, fmt.Errorf("CustomDialTLS: TLS handshake failed and closing plain connection failed: %s", closeErr.Error()) @@ -511,7 +700,7 @@ func (d *customDialer) writeWARCFromConnection(ctx context.Context, reqPipe, res case <-ctx.Done(): return default: - if d.proxyDialer == nil { + if len(d.proxyDialers) == 0 { switch addr := conn.RemoteAddr().(type) { case *net.TCPAddr: IP := addr.IP.String() diff --git a/dialer_test.go b/dialer_test.go index 147668b..2d01563 100644 --- a/dialer_test.go +++ b/dialer_test.go @@ -2,6 +2,7 @@ package warc import ( "bytes" + "context" "io" "strings" "testing" @@ -165,3 +166,332 @@ func TestFindEndOfHeadersOffset(t *testing.T) { }) } } + +func TestProxySelection(t *testing.T) { + t.Run("NoProxies", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{}, + } + proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy != nil { + t.Error("expected nil proxy when no proxies configured") + } + }) + + t.Run("IPv4ProxyWithIPv4Network", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkIPv4, + proxyType: ProxyTypeAny, + url: "socks5://ipv4-proxy:1080", + }, + }, + } + proxy, err := d.selectProxy(context.Background(), "tcp4", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil { + t.Error("expected proxy for IPv4 network with IPv4 proxy") + } + if proxy != nil && proxy.url != "socks5://ipv4-proxy:1080" { + t.Errorf("expected socks5://ipv4-proxy:1080, got %s", proxy.url) + } + }) + + t.Run("IPv4ProxyWithIPv6Network", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkIPv4, + proxyType: ProxyTypeAny, + url: "socks5://ipv4-proxy:1080", + }, + }, + allowDirectFallback: true, + } + proxy, err := d.selectProxy(context.Background(), "tcp6", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy != nil { + t.Error("expected nil proxy for IPv6 network with IPv4 proxy") + } + }) + + t.Run("IPv6ProxyWithIPv6Network", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkIPv6, + proxyType: ProxyTypeAny, + url: "socks5://ipv6-proxy:1080", + }, + }, + } + proxy, err := d.selectProxy(context.Background(), "tcp6", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil { + t.Error("expected proxy for IPv6 network with IPv6 proxy") + } + }) + + t.Run("DomainFiltering", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeAny, + allowedDomains: []string{"*.example.com"}, + url: "socks5://domain-proxy:1080", + }, + }, + } + + // Should match subdomain + proxy, err := d.selectProxy(context.Background(), "tcp", "api.example.com:443") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil { + t.Error("expected proxy for matching domain") + } + + // Should match base domain + d.allowDirectFallback = true + proxy, err = d.selectProxy(context.Background(), "tcp", "example.com:443") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil { + t.Error("expected proxy for base domain") + } + + // Should not match different domain + proxy, err = d.selectProxy(context.Background(), "tcp", "other.com:443") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy != nil { + t.Error("expected nil proxy for non-matching domain") + } + }) + + t.Run("ProxyTypeSelection", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeAny, + url: "socks5://any-proxy:1080", + }, + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeMobile, + url: "socks5://mobile-proxy:1080", + }, + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeResidential, + url: "socks5://residential-proxy:1080", + }, + }, + } + + // Without proxy type context, should use ProxyTypeAny proxy + proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil || proxy.url != "socks5://any-proxy:1080" { + t.Error("expected any-proxy without proxy type context") + } + + // With mobile proxy type context, should use mobile proxy + ctx := WithProxyType(context.Background(), ProxyTypeMobile) + proxy, err = d.selectProxy(ctx, "tcp", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil || proxy.url != "socks5://mobile-proxy:1080" { + t.Error("expected mobile-proxy with mobile proxy type context") + } + + // With residential proxy type context, should use residential proxy + ctx = WithProxyType(context.Background(), ProxyTypeResidential) + proxy, err = d.selectProxy(ctx, "tcp", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil || proxy.url != "socks5://residential-proxy:1080" { + t.Error("expected residential-proxy with residential proxy type context") + } + }) + + t.Run("RoundRobinSelection", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeAny, + url: "socks5://proxy1:1080", + }, + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeAny, + url: "socks5://proxy2:1080", + }, + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeAny, + url: "socks5://proxy3:1080", + }, + }, + } + + // Expected order for 3 complete cycles (9 selections) + expectedOrder := []string{ + "socks5://proxy1:1080", + "socks5://proxy2:1080", + "socks5://proxy3:1080", + "socks5://proxy1:1080", + "socks5://proxy2:1080", + "socks5://proxy3:1080", + "socks5://proxy1:1080", + "socks5://proxy2:1080", + "socks5://proxy3:1080", + } + + // Select proxies 9 times and verify sequential round-robin order + for i := 0; i < 9; i++ { + proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80") + if err != nil { + t.Errorf("iteration %d: unexpected error: %v", i, err) + } + if proxy == nil { + t.Errorf("iteration %d: expected proxy, got nil", i) + } else if proxy.url != expectedOrder[i] { + t.Errorf("iteration %d: expected %s, got %s", i, expectedOrder[i], proxy.url) + } + } + }) + + t.Run("NoEligibleProxiesWithFallback", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkIPv6, + proxyType: ProxyTypeAny, + url: "socks5://ipv6-proxy:1080", + }, + }, + allowDirectFallback: true, + } + + // IPv4 network with IPv6 proxy should use direct connection + proxy, err := d.selectProxy(context.Background(), "tcp4", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy != nil { + t.Error("expected nil proxy with direct fallback") + } + }) + + t.Run("NoEligibleProxiesWithoutFallback", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkIPv6, + proxyType: ProxyTypeAny, + url: "socks5://ipv6-proxy:1080", + }, + }, + allowDirectFallback: false, + } + + // IPv4 network with IPv6 proxy and no fallback should error + proxy, err := d.selectProxy(context.Background(), "tcp4", "example.com:80") + if err == nil { + t.Error("expected error when no eligible proxies and no fallback") + } + if proxy != nil { + t.Error("expected nil proxy") + } + }) + + t.Run("ComplexFiltering", func(t *testing.T) { + d := &customDialer{ + proxyDialers: []proxyDialerInfo{ + { + proxyNetwork: ProxyNetworkIPv4, + proxyType: ProxyTypeAny, + allowedDomains: []string{"*.api.example.com"}, + url: "socks5://api-ipv4-proxy:1080", + }, + { + proxyNetwork: ProxyNetworkIPv6, + proxyType: ProxyTypeAny, + allowedDomains: []string{"*.media.example.com"}, + url: "socks5://media-ipv6-proxy:1080", + }, + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeMobile, + url: "socks5://mobile-proxy:1080", + }, + { + proxyNetwork: ProxyNetworkAny, + proxyType: ProxyTypeResidential, + url: "socks5://residential-proxy:1080", + }, + }, + } + + // Test IPv4 API domain + proxy, err := d.selectProxy(context.Background(), "tcp4", "service.api.example.com:443") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil || proxy.url != "socks5://api-ipv4-proxy:1080" { + t.Error("expected api-ipv4-proxy for IPv4 API domain") + } + + // Test IPv6 media domain + proxy, err = d.selectProxy(context.Background(), "tcp6", "cdn.media.example.com:443") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil { + t.Error("expected proxy for IPv6 media domain, got nil") + } else if proxy.url != "socks5://media-ipv6-proxy:1080" { + t.Errorf("expected media-ipv6-proxy for IPv6 media domain, got %s", proxy.url) + } + + // Test mobile proxy type context + ctx := WithProxyType(context.Background(), ProxyTypeMobile) + proxy, err = d.selectProxy(ctx, "tcp", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil || proxy.url != "socks5://mobile-proxy:1080" { + t.Error("expected mobile-proxy with mobile proxy type context") + } + + // Test residential proxy type context + ctx = WithProxyType(context.Background(), ProxyTypeResidential) + proxy, err = d.selectProxy(ctx, "tcp", "example.com:80") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if proxy == nil || proxy.url != "socks5://residential-proxy:1080" { + t.Error("expected residential-proxy with residential proxy type context") + } + }) +}