diff --git a/url_finder/src/config.rs b/url_finder/src/config.rs index 4cd9715..cca5078 100644 --- a/url_finder/src/config.rs +++ b/url_finder/src/config.rs @@ -14,7 +14,8 @@ pub struct Config { pub proxy_url: Option, pub proxy_user: Option, pub proxy_password: Option, - pub proxy_ip_count: Option, + pub proxy_ip_count: Option, + pub proxy_default_port: Option, } impl Config { @@ -39,6 +40,9 @@ impl Config { proxy_url: env::var("PROXY_URL").unwrap_or("US".to_string()).into(), proxy_user: env::var("PROXY_USER").ok(), proxy_password: env::var("PROXY_PASSWORD").ok(), + proxy_default_port: env::var("PROXY_DEFAULT_PORT") + .ok() + .and_then(|s| s.parse().ok()), proxy_ip_count: env::var("PROXY_IP_COUNT").ok().and_then(|s| s.parse().ok()), }) } @@ -55,6 +59,7 @@ impl Config { proxy_url: None, proxy_user: None, proxy_ip_count: None, + proxy_default_port: None, } } } diff --git a/url_finder/src/http_client.rs b/url_finder/src/http_client.rs index 3d8eb46..e210a7f 100644 --- a/url_finder/src/http_client.rs +++ b/url_finder/src/http_client.rs @@ -10,10 +10,13 @@ const RETRI_TIMEOUT_SEC: u64 = 15; static ATOMIC_PROXY_PORT: AtomicU32 = AtomicU32::new(8001); static ATOMIC_PROXY_LAST_CHANGE: AtomicU64 = AtomicU64::new(0); -fn get_sticky_port_atomic(ip_count: u32) -> u16 { +fn get_sticky_port_atomic(config: &Config) -> u32 { + let proxy_default_port = config.proxy_default_port.unwrap(); + let ip_count = config.proxy_ip_count.unwrap(); + // if no proxy ip count configured, return default port if ip_count == 0 { - return 8000; + return proxy_default_port; } let now = SystemTime::now() @@ -26,7 +29,7 @@ fn get_sticky_port_atomic(ip_count: u32) -> u16 { let expired = now - last > 24 * 3600; // rotate every 24 hours if expired { - let start = 8001; + let start = config.proxy_default_port.unwrap(); let end = start + ip_count - 1; let mut rng = rand::rng(); @@ -35,27 +38,37 @@ fn get_sticky_port_atomic(ip_count: u32) -> u16 { ATOMIC_PROXY_PORT.store(new_port, Ordering::Relaxed); ATOMIC_PROXY_LAST_CHANGE.store(now, Ordering::Relaxed); - return new_port as u16; + return new_port as u32; } - ATOMIC_PROXY_PORT.load(Ordering::Relaxed) as u16 + ATOMIC_PROXY_PORT.load(Ordering::Relaxed) } pub fn build_client(config: &Config) -> Result { let mut builder = Client::builder().timeout(std::time::Duration::from_secs(RETRI_TIMEOUT_SEC)); - if let (Some(proxy_url), Some(proxy_user), Some(proxy_password), Some(proxy_ip_count)) = ( + if let ( + Some(proxy_url), + Some(proxy_user), + Some(proxy_password), + Some(proxy_ip_count), + Some(proxy_default_port), + ) = ( &config.proxy_url, &config.proxy_user, &config.proxy_password, &config.proxy_ip_count, + &config.proxy_default_port, ) { - let ip_count = *proxy_ip_count as u32; + info!( + "Configuring HTTP client with proxy: {} (user: {}, ip_count: {}, default_port: {})", + proxy_url, proxy_user, proxy_ip_count, proxy_default_port + ); - let port = get_sticky_port_atomic(ip_count); + let port = get_sticky_port_atomic(config); let proxy_url_result = format!("{}:{}", proxy_url, port); - info!("Using proxy: {}", proxy_url_result); + info!("Start using proxy: {}", proxy_url_result); let proxy = (Proxy::http(proxy_url_result))?.basic_auth(proxy_user, proxy_password); builder = builder