ohIPPool/proxy.py at master · yitian-reevo/ohIPPool · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
from multiprocessing import Process, Lock
from random import choice
import redis
import time
import requests
import ast
import sys

import webapi
import configure as Configs
import logger as Logger

redis_client = redis.StrictRedis(host=Configs.REDIS_HOST, port=Configs.REDIS_PORT, db=0)

def fn_retriever():
	global redis_clent
	fn_name = sys._getframe().f_code.co_name
	while True:
		Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Routine Retriever Start.")
		cnt = redis_client.zcard(Configs.REDIS_KEY)
		ip_cnt_needed = Configs.POOL_SIZE - redis_client.zcard(Configs.REDIS_KEY)

		if ip_cnt_needed <= 0:
			Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Enough IP(s) in Redis.")
			Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Entering Sleep: {0:d}s.".format(Configs.RETRIEVER_INTERVAL))
			time.sleep(Configs.RETRIEVER_INTERVAL)
			continue

		tid = Configs.DAXIANG_RPOXY_ORDERID
		url = "http://tvp.daxiangdaili.com/ip/?tid={0:s}&num={1:d}&delay=5&category=2&sortby=time&filter=on&format=json&protocol=https".format(tid, ip_cnt_needed)

		try:
			response = requests.get(url)
			content = None

			if response.status_code == requests.codes.ok:
				content = response.text
		except Exception as e:
			print (e)

		res_json = ast.literal_eval(content.strip())

		if res_json:
			for addr in res_json:
				if addr.get('error'):
					continue
				redis_client.zadd(Configs.REDIS_KEY, Configs.MEDIUM, '{0:s}:{1:d}'.format(addr.get('host'), addr.get('port')))

		Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Refill {0:d} IP(s) to Redis.".format(ip_cnt_needed))

def __validation(addr):
	proxies = {
		"http": "http://{0}".format(addr),
		"https": "http://{0}".format(addr)
	}

	header = {}
	header['user-agent'] = choice(Configs.FakeUserAgents)

	try:
		response = requests.get("http://52.206.77.228:5000/myIP", headers=header, proxies=proxies, timeout=5)
	except Exception as e:
		return False
	else:
		if response.status_code == requests.codes.ok:
			#print (response.text)
			return True

def fn_validator():
	global redis_clent
	fn_name = sys._getframe().f_code.co_name
	while True:
		# Test all ips whose score >= POOL_IP_QUALITY - 1
		# False ==> score - 1
		# True  ==> score + 1
		maxscore = redis_client.zrange(Configs.REDIS_KEY, 0, 0, desc=True, withscores=True)
		if not maxscore:
			Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Pool is empty. Entering Sleep: {0:d}s.".format(Configs.VALIDATOR_INTERVAL))
			time.sleep(Configs.VALIDATOR_INTERVAL)
			continue

		maxscore = maxscore[0][1]
		Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Max score in this round: {0:d}.".format(int(maxscore)))
		res = redis_client.zrangebyscore(Configs.REDIS_KEY, Configs.POOL_IP_QUALITY - 1, maxscore)
		Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Start to Validate {0:d} IP(s).".format(len(res)))

		increment = []

		i = 0
		for ip in res:
			n = 1 if __validation(ip.decode('utf-8')) else -1
			increment.append([ip, n])
			Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "[{0:d}]Validated[{1:s}], Result:[{2:d}].".format(i, ip.decode('utf-8'), n))
			i += 1

		for inc in increment:
			redis_client.zincrby(Configs.REDIS_KEY, inc[0], inc[1])

		Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Validation finished. Entering Sleep: {0:d}s.".format(Configs.VALIDATOR_INTERVAL))
		time.sleep(Configs.VALIDATOR_INTERVAL)


def fn_cleaner():
	global redis_clent
	fn_name = sys._getframe().f_code.co_name
	while True:
		# Remove all ips whose score < POOL_IP_QUALITY
		res = redis_client.zremrangebyscore(Configs.REDIS_KEY, -1, Configs.POOL_IP_QUALITY - 1)
		Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Remove {0:d} IP(s) from Redis.".format(res))
		Logger.log(Configs.LOG_LEVEL_FINE, fn_name, "Entering Sleep: {0:d}s.".format(Configs.CLEANER_INTERVAL))
		time.sleep(Configs.CLEANER_INTERVAL)


def fn_webapi():
	webapi.app.run(host=Configs.API_ADDR, port=Configs.API_PORT)

def get_one_ip():
	global redis_clent
	cnt = redis_client.zrange(Configs.REDIS_KEY, 0, 0, desc=True)
	redis_client.zincrby(Configs.REDIS_KEY, cnt[0], -1)

	if cnt:
		return cnt[0].decode('utf-8')
	else:
		return 'ERROR|TRY AGAIN LATER'

def main():
	process_webapi = Process(target=fn_webapi)
	process_webapi.start()
	time.sleep(5)

	process_retriever = Process(target=fn_retriever)
	process_retriever.start()


	process_validator = Process(target=fn_validator)
	process_validator.start()

	process_cleaner = Process(target=fn_cleaner)
	process_cleaner.start()


if __name__ == '__main__':
	main()
	#r = redis.StrictRedis(host=Configs.REDIS_HOST, port=Configs.REDIS_PORT, db=0)
	#print (r.zincrby(Configs.REDIS_KEY, '123.23.3.3', -10))
	#print (r.zadd(Configs.REDIS_KEY, 5, '123.23.3.3'))
	#print (r.zadd(Configs.REDIS_KEY, 5, '123.23.3.3'))
	#print (r.zadd(Configs.REDIS_KEY, 5, '123.23.3.3'))
	#print (r.zadd(Configs.REDIS_KEY, 5, '123.23.3.3'))
	#ret = r.sscan(1)
	#print (ret)