Use your ua first. (#55)

Author: puppylpg

CHANGELOG.md

@@ -119,4 +119,9 @@
 
 ## 3.9.0（2020-12-11）
 * 功能
-    - 每一次请求都随机选用user agent，也许会对封禁有所帮助；
+    - ~~每一次请求都随机~~选用user agent，也许会对封禁有所帮助；
+
+## 3.9.1（2020-12-14）
+* 功能
+    - 配置里支持使用`buff_user_agent`粘贴自己的ua。如果留空，使用随机的ua，且每次爬取都使用该ua，而不是每次请求都随机一个ua。
+

README.md

@@ -87,7 +87,7 @@ steam_cookie = timezoneOffset=28800,0; steamMachineAuth76561198093333055=649A9B5
 1. 工程根目录下运行：`python -m src`
 
 ## 输出结果
-- database：爬的数据；
+~~- database：爬的数据；~~
 - log: 日志；
 - suggestion：分析结果；
 

config/config.ini

@@ -2,6 +2,11 @@
 # 网页登录buff和steam后，把浏览器的cookie贴到这里，才能运行。推荐从Chrome中复制，下面是两条示例cookie
 buff_cookie = _ga=GA1.2.162602080.1551374933; _ntes_nuid=8ce0cf6bdce55512e73f49cb8a49960e; mail_psc_fingerprint=d80ec72871726e9b192181fd1a3633d6; OUTFOX_SEARCH_USER_ID_NCOO=29659292.15961449; Device-Id=33u998YqmNWbhH5GbWUo; vjuids=369cb7d82.170e16a9519.0.3eb2c52902997; vjlast=1584329824.1584329824.30; _ntes_nnid=8ce0cf6bdce55512e73f49cb8a49960e,1584329823520; vinfo_n_f_l_n3=d81bf3a25989eb31.1.4.1561837557589.1576393349946.1585037711031; NTES_CMT_USER_INFO=305053074%7C%E6%9C%89%E6%80%81%E5%BA%A6%E7%BD%91%E5%8F%8B0ibHSi%7Chttp%3A%2F%2Fcms-bucket.nosdn.127.net%2F2018%2F08%2F13%2F078ea9f65d954410b62a52ac773875a1.jpeg%7Cfalse%7CeWQuNzU3YTdkZjAwZWNiNDJlOGJAMTYzLmNvbQ%3D%3D; nts_mail_user=ter@163.com:-1:1; __oc_uuid=ed078220-12cd-11eb-8ff5-199a4d2b4ac4; Locale-Supported=zh-Hans; game=csgo; _gid=GA1.2.648285736.1605190175; _gat_gtag_UA_109989484_1=1; NTES_YD_SESS=SN.CH9UV_zHPlqCiCLvgBOoLTrvc2fBGRyieqBhbqAP1HglxHydKU4DQmq5B7At06lgkSmM_II0j06AJnuMWYnpdtYe8PPxUJMsM4X5yH3jBY3xJdC_d59nM8A1bksgKL51SSXhh3Rbd4SeDy6ZIwse2MUjzElPeLdPKBaMoZafPdtNUF9E67TduT0krt3r6_s46hz3dnGE.y20NruVavQP3kETqQCAqK6iZ3b0Nc6tJw; S_INFO=1605190207|0|3&80##|2051; P_INFO=2051|1605190207|1|netease_buff|00&99|bej&1602387932&netease_buff#bej&null#10#0#0|&0|null|2051; remember_me=U1094050600|T3zeeLJIc6y9kVtTTAGV0mdqvIXDpeX0; session=1-WfP1TH9yGjtZniGRmbfFSezTOMS-ZeYguhJFzDIT5Fem2046524528; csrf_token=ImFjMWE4YTc4MDFkMTAyZjYyYWZhZWVhYzllZGFlNTJiZjc1NWE1MDEi.Eo7TwA.f39WinRhrzJgSTG4as2EjhD6za0
 steam_cookie = ActListPageSize=100; steamMachineAuth76561198251761676=B89D7B0897180E54C9F2E93F8AAFA4583CAADE7D; timezoneOffset=28800,0; _ga=GA1.2.1902489943.1551205764; steamMachineAuth76561198874249759=E46DCE6095514E3D489CAF1E7CBC3F9F8CD3ACC6; browserid=1066728544083117486; recentlyVisitedAppHubs=271590%2C80%2C730; Steam_Language=english; steamCountry=US%7C4705a9aaf22f908f9e4452081abd865a; sessionid=56b51232f9f3936a0ebbf88d; _gid=GA1.2.1847664544.1605190173; steamLoginSecure=76561198251761676%7C%7CE4B6E3BBDD5AF069692D8C8A56755ECBB34ECC68; steamRememberLogin=76561198251761676%7C%7Ca5e43585d1cd13db87c3d856d7676178; webTradeEligibility=%7B%22allowed%22%3A1%2C%22allowed_at_time%22%3A0%2C%22steamguard_required_days%22%3A15%2C%22new_device_cooldown_days%22%3A7%2C%22time_checked%22%3A1605190194%7D
+
+# 获取登陆buff时的User-Agent，和获取cookie的方式相同。如果不使用自己的user-agent，留空即可（不建议），程序会随机一个user-agent。
+# 为了减弱被封禁的风险，强烈建议把自己的user-agent贴在此处。例如：Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36
+buff_user_agent =
+
 # 提供一个代理来访问Steam社区市场。如果不需要代理即可直接访问市场，直接留空即可
 proxy = socks5://127.0.0.1:10808
 

src/config/definitions.py

@@ -25,6 +25,7 @@
 
 # cookie and api
 BUFF_COOKIE = config['BASIC']['buff_cookie']
+USER_AGENT = config['BASIC']['buff_user_agent']
 STEAM_COOKIE = config['BASIC']['steam_cookie']
 
 # proxy

src/util/requester.py

@@ -5,7 +5,7 @@
 import random
 from requests import Timeout
 
-from src.config.definitions import PROXY, BUFF_COOKIE, STEAM_COOKIE, RETRY_TIMES
+from src.config.definitions import PROXY, BUFF_COOKIE, USER_AGENT, STEAM_COOKIE, RETRY_TIMES
 from src.util import timer
 from src.util.logger import log
 from src.util.cache import fetch, store, exist
@@ -26,22 +26,33 @@
     k, v = line.split('=', 1)
     steam_cookies[k] = v
 
+# get user-agent database
 csv = pd.read_csv('config/reference/ua.csv')
 ua = csv.ua
 
 
+# get user-agent
+def get_ua():
+    if USER_AGENT:
+        return USER_AGENT
+    else:
+        return get_random_ua()
+
+
 def get_random_ua():
     return ua[random.randint(0, ua.size)]
 
 
 def get_headers():
-    specific_ua = get_random_ua()
-    log.info('Random ua: {}'.format(specific_ua))
+    target_ua = get_ua()
+    log.info('use User-Agent: {}'.format(target_ua))
     return {
-        'User-Agent': specific_ua
+        'User-Agent': target_ua
     }
 
 
+headers = get_headers()
+
 proxies = {}
 if PROXY:
     proxies["http"] = PROXY
@@ -59,8 +70,8 @@ def get_json_dict_raw(url, cookies, proxy=False, times=1):
     timer.sleep_awhile()
     try:
         if proxy and proxies != {}:
-            return requests.get(url, headers=get_headers(), cookies=cookies, timeout=5, proxies=proxies).text
-        return requests.get(url, headers=get_headers(), cookies=cookies, timeout=5).text
+            return requests.get(url, headers=headers, cookies=cookies, timeout=5, proxies=proxies).text
+        return requests.get(url, headers=headers, cookies=cookies, timeout=5).text
     except Timeout:
         log.warn("Timeout for {}. Try again.".format(url))
     except Exception as e: