diff --git a/app/helper/site_helper.py b/app/helper/site_helper.py index a11e6bf1..f2788687 100644 --- a/app/helper/site_helper.py +++ b/app/helper/site_helper.py @@ -2,10 +2,11 @@ from datetime import datetime import os import re +import json from lxml import etree -from app.utils import SystemUtils +from app.utils import SystemUtils, JsonUtils from config import RMT_SUBEXT @@ -18,6 +19,13 @@ def is_logged_in(cls, html_text): :param html_text: :return: """ + if JsonUtils.is_valid_json(html_text): + json_data = json.loads(html_text) + if 'message' in json_data and json_data['message'] == 'SUCCESS': + return True + else: + return False + html = etree.HTML(html_text) if not html: return False diff --git a/app/sites/site_userinfo.py b/app/sites/site_userinfo.py index 9d815887..e2137c70 100644 --- a/app/sites/site_userinfo.py +++ b/app/sites/site_userinfo.py @@ -79,56 +79,70 @@ def build(self, url, site_id, site_name, html_text = chrome.get_html() else: proxies = Config().get_proxies() if proxy else None - res = RequestUtils(cookies=site_cookie, - session=session, - headers=ua, - proxies=proxies - ).get_res(url=url) + if 'm-team' in url: + profile_url = url + '/api/member/profile' + res = RequestUtils(cookies=site_cookie, + session=session, + headers=ua, + proxies=proxies + ).post_res(url=profile_url, data={}) + else: + res = RequestUtils(cookies=site_cookie, + session=session, + headers=ua, + proxies=proxies + ).get_res(url=url) if res and res.status_code == 200: if "charset=utf-8" in res.text or "charset=UTF-8" in res.text: res.encoding = "UTF-8" else: res.encoding = res.apparent_encoding html_text = res.text - # 第一次登录反爬 - if html_text.find("title") == -1: - i = html_text.find("window.location") - if i == -1: + # 单独处理m-team + if 'm-team' in url: + json_data = json.loads(html_text) + if 'message' in json_data and json_data['message'] != "SUCCESS": return None - tmp_url = url + html_text[i:html_text.find(";")] \ - .replace("\"", "").replace("+", "").replace(" ", "").replace("window.location=", "") - res = RequestUtils(cookies=site_cookie, - session=session, - headers=ua, - proxies=proxies - ).get_res(url=tmp_url) - if res and res.status_code == 200: - if "charset=utf-8" in res.text or "charset=UTF-8" in res.text: - res.encoding = "UTF-8" - else: - res.encoding = res.apparent_encoding - html_text = res.text - if not html_text: + else: + # 第一次登录反爬 + if html_text.find("title") == -1: + i = html_text.find("window.location") + if i == -1: return None - else: - log.error("【Sites】站点 %s 被反爬限制：%s, 状态码：%s" % (site_name, url, res.status_code)) - return None - - # 兼容假首页情况，假首页通常没有 0 else 0 - except Exception as err: - ExceptionUtils.exception_traceback(err) + "free": False, + "2xfree": False, + "hr": False, + "peer_count": 0 + } + if 'm-team' in torrent_url: + split_url = urlsplit(torrent_url) + base_url = f"{split_url.scheme}://{split_url.netloc}" + detail_url = f"{base_url}/api/torrent/detail" + res = re.findall(r'\d+', torrent_url) + param = res[0] + json_text = self.__get_site_page_html(url=detail_url, + cookie=cookie, + ua=ua, + proxy=proxy, + param=param) + json_data = json.loads(json_text) + if json_data['message'] != "SUCCESS": + return ret_attr + discount = json_data['data']['status']['discount'] + seeders = json_data['data']['status']['seeders'] + if discount == 'FREE': + ret_attr["free"] = True + ret_attr['peer_count'] = int(seeders) + + else: + if not torrent_url: + return ret_attr + xpath_strs = self.get_grap_conf(torrent_url) + if not xpath_strs: + return ret_attr + html_text = self.__get_site_page_html(url=torrent_url, + cookie=cookie, + ua=ua, + render=xpath_strs.get('RENDER'), + proxy=proxy) + if not html_text: + return ret_attr + try: + html = etree.HTML(html_text) + # 检测2XFREE + for xpath_str in xpath_strs.get("2XFREE"): + if html.xpath(xpath_str): + ret_attr["free"] = True + ret_attr["2xfree"] = True + # 检测FREE + for xpath_str in xpath_strs.get("FREE"): + if html.xpath(xpath_str): + ret_attr["free"] = True + # 检测HR + for xpath_str in xpath_strs.get("HR"): + if html.xpath(xpath_str): + ret_attr["hr"] = True + # 检测PEER_COUNT当前做种人数 + for xpath_str in xpath_strs.get("PEER_COUNT"): + peer_count_dom = html.xpath(xpath_str) + if peer_count_dom: + peer_count_str = ''.join(peer_count_dom[0].itertext()) + peer_count_digit_str = "" + for m in peer_count_str: + if m.isdigit(): + peer_count_digit_str = peer_count_digit_str + m + if m == " ": + break + ret_attr["peer_count"] = int(peer_count_digit_str) if len(peer_count_digit_str) > 0 else 0 + except Exception as err: + ExceptionUtils.exception_traceback(err) # 随机休眼后再返回 time.sleep(round(random.uniform(1, 5), 1)) return ret_attr @staticmethod @lru_cache(maxsize=128) - def __get_site_page_html(url, cookie, ua, render=False, proxy=False): + def __get_site_page_html(url, cookie, ua, render=False, proxy=False, param=None): chrome = ChromeHelper(headless=True) if render and chrome.get_status(): # 开渲染 @@ -175,6 +199,23 @@ def __get_site_page_html(url, cookie, ua, render=False, proxy=False): # 等待页面加载完成 time.sleep(10) return chrome.get_html() + elif 'm-team' in url: + param = {'id': param} + headers = {} + headers.update({ + "User-Agent": f"{ua}" + }) + headers.update({ + "contentType": 'application/json;charset=UTF-8' + }) + res = RequestUtils( + cookies=cookie, + headers=headers, + proxies=Config().get_proxies() if proxy else None + ).post_res(url=url, data=param) + if res and res.status_code == 200: + res.encoding = res.apparent_encoding + return res.text else: res = RequestUtils( cookies=cookie, diff --git a/app/sites/sites.py b/app/sites/sites.py index 86993a47..1a70de96 100644 --- a/app/sites/sites.py +++ b/app/sites/sites.py @@ -288,10 +288,18 @@ def test_connection(self, site_id): else: # 计时 start_time = datetime.now() - res = RequestUtils(cookies=site_cookie, - headers=ua, - proxies=Config().get_proxies() if site_info.get("proxy") else None - ).get_res(url=site_url) + # m-team处理 + if 'm-team' in site_url: + url = site_url + '/api/member/profile' + res = RequestUtils(cookies=site_cookie, + headers=ua, + proxies=Config().get_proxies() if site_info.get("proxy") else None + ).post_res(url=url, data={}) + else: + res = RequestUtils(cookies=site_cookie, + headers=ua, + proxies=Config().get_proxies() if site_info.get("proxy") else None + ).get_res(url=site_url) seconds = int((datetime.now() - start_time).microseconds / 1000) if res and res.status_code == 200: if not SiteHelper.is_logged_in(res.text): diff --git a/app/sites/siteuserinfo/mteam.py b/app/sites/siteuserinfo/mteam.py new file mode 100644 index 00000000..348a1245 --- /dev/null +++ b/app/sites/siteuserinfo/mteam.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +import json +import base64 + +from urllib.parse import urljoin + +from app.sites.siteuserinfo._base import _ISiteUserInfo, SITE_BASE_ORDER +from app.utils import RequestUtils +from app.utils.types import SiteSchema +from config import Config + + +class MteamUserInfo(_ISiteUserInfo): + schema = SiteSchema.Mteam + order = SITE_BASE_ORDER + 15 + + @classmethod + def match(cls, html_text): + return "authorities" in html_text + + def parse(self): + """ + 解析站点信息 + :return: + """ + self._parse_favicon(self._index_html) + if not self._parse_logged_in(self._index_html): + return + + self._parse_site_page(self._index_html) + self._parse_user_base_info(self._index_html) + self._pase_unread_msgs() + self._parse_user_traffic_info(self._index_html) + self._parse_user_detail_info(self._index_html) + + self._parse_seeding_pages() + self.seeding_info = json.dumps(self.seeding_info) + + + def _parse_favicon(self, html_text): + """ + 解析站点favicon,返回base64 fav图标 + :param html_text: + :return: + """ + self._favicon_url = urljoin(self._base_url, '/favicon.ico') + + res = RequestUtils(cookies=self._site_cookie, session=self._session, timeout=60, headers=self._ua).get_res( + url=self._favicon_url) + if res: + self.site_favicon = base64.b64encode(res.content).decode() + + def _parse_user_base_info(self, html_text): + json_data = json.loads(html_text) + + user_profile = self._get_page_content(self._base_url + '/api/member/profile', params={}) + user_profile = json.loads(user_profile) + if user_profile['message'] == 'SUCCESS' and json_data['data'] is not None: + userid = user_profile['data']['id'] + self.username = json_data['data']['username'] + self.bonus = json_data['data']['memberCount']['bonus'] + self._torrent_seeding_page = '/api/member/getUserTorrentList' + self._torrent_seeding_params = {"userid": userid, "type": "SEEDING", "pageNumber": 1, "pageSize": 25} + + def _parse_site_page(self, html_text): + pass + + def _parse_user_detail_info(self, html_text): + """ + 解析用户额外信息，加入时间，等级 + :param html_text: + :return: + """ + role_dict = { + '1': '小卒/User', + '2': '捕頭/Power User', + '3': '知縣/Elite User', + '4': '通判/Crazy User', + '5': '知州/Insane User', + '6': '府丞/Veteran User', + '7': '府尹/Extreme User', + '8': '總督/Ultimate User', + '9': '大臣/mTorrent Master' + } + json_data = json.loads(html_text) + if json_data['data'] is not None: + # 用户等级 + role = json_data['data']['role'] + self.user_level = role_dict.get(role, '其他') + + # 加入日期 + self.join_at = json_data['data']['createdDate'] + + def _parse_user_traffic_info(self, html_text): + json_data = json.loads(html_text) + if json_data['data'] is not None: + self.upload = int(json_data['data']['memberCount']['uploaded']) + + self.download = int(json_data['data']['memberCount']['downloaded']) + + self.ratio = json_data['data']['memberCount']['shareRate'] + + def _parse_user_torrent_seeding_info(self, html_text, multi_page=False): + json_data = json.loads(html_text) + + page_seeding = 0 + page_seeding_size = 0 + page_seeding_info = [] + next_page = None + + if json_data['data'] is not None: + page_seeding = len(json_data['data']['data']) + for data in json_data['data']['data']: + size = int(data['torrent']['size']) + seeders = data['torrent']['status']['seeders'] + + page_seeding_size += size + page_seeding_info.append([seeders, size]) + + self.seeding += page_seeding + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) + + page_num = int(json_data['data']['pageNumber']) + total_pages = int(json_data['data']['totalPages']) + + next_page = page_num + 1 + self._torrent_seeding_params['pageNumber'] = next_page + if next_page > total_pages: + return None + return next_page + + def _get_page_content(self, url, params=None, headers=None): + """ + :param url: 网页地址 + :param params: post参数 + :param headers: 额外的请求头 + :return: + """ + req_headers = None + proxies = Config().get_proxies() if self._proxy else None + if self._ua or headers or self._addition_headers: + req_headers = {} + if headers: + req_headers.update(headers) + + if isinstance(self._ua, str): + req_headers.update({ + "User-Agent": f"{self._ua}" + }) + else: + req_headers.update(self._ua) + + if self._addition_headers: + req_headers.update(self._addition_headers) + + params = json.dumps(params, separators=(',', ':')) + res = RequestUtils(cookies=self._site_cookie, + session=self._session, + proxies=proxies, + headers=req_headers).post_res(url=url, data=params) + if res is not None and res.status_code in (200, 500, 403): + if "charset=utf-8" in res.text or "charset=UTF-8" in res.text: + res.encoding = "UTF-8" + else: + res.encoding = res.apparent_encoding + return res.text + + return "" + + def _parse_seeding_pages(self): + if self._torrent_seeding_page: + # 第一页 + next_page = self._parse_user_torrent_seeding_info( + self._get_page_content(urljoin(self._base_url, self._torrent_seeding_page), + self._torrent_seeding_params, + self._torrent_seeding_headers)) + + # 其他页处理 + while next_page: + next_page = self._parse_user_torrent_seeding_info( + self._get_page_content(urljoin(self._base_url, self._torrent_seeding_page), + self._torrent_seeding_params, + self._torrent_seeding_headers), + multi_page=True) + + def _parse_message_unread_links(self, html_text, msg_links): + return None + + def _parse_message_content(self, html_text): + return None, None, None diff --git a/app/utils/json_utils.py b/app/utils/json_utils.py index ccfe2e50..fa14b3c6 100644 --- a/app/utils/json_utils.py +++ b/app/utils/json_utils.py @@ -22,3 +22,14 @@ def _try(o): return str(o) return json.loads(json.dumps(obj, default=lambda o: _try(o))) + + @staticmethod + def is_valid_json(text): + """ + 判断是否是有效的json格式字符串 + """ + try: + json.loads(text) + return True + except json.JSONDecodeError: + return False diff --git a/app/utils/number_utils.py b/app/utils/number_utils.py index 7c31653a..ca69f439 100644 --- a/app/utils/number_utils.py +++ b/app/utils/number_utils.py @@ -19,3 +19,32 @@ def get_size_gb(size): if not size: return 0.0 return float(size) / 1024 / 1024 / 1024 + + @staticmethod + def format_byte_repr(byte_num): + """ + size转换 + :param byte_num: 单位Byte + :return: + """ + KB = 1024 + MB = KB * KB + GB = MB * KB + TB = GB * KB + try: + if isinstance(byte_num, str): + byte_num = int(byte_num) + if byte_num > TB: + result = '%s TB' % round(byte_num / TB, 2) + elif byte_num > GB: + result = '%s GB' % round(byte_num / GB, 2) + elif byte_num > MB: + result = '%s MB' % round(byte_num / MB, 2) + elif byte_num > KB: + result = '%s KB' % round(byte_num / KB, 2) + else: + result = '%s B' % byte_num + return result + except Exception as e: + print(e.args) + return byte_num diff --git a/app/utils/types.py b/app/utils/types.py index 5897b1d8..bb4ce1f2 100644 --- a/app/utils/types.py +++ b/app/utils/types.py @@ -95,6 +95,7 @@ class SiteSchema(Enum): TorrentLeech = "TorrentLeech" FileList = "FileList" TNode = "TNode" + Mteam = "M-Team" # 可监听事件 diff --git a/config/sites.dat b/config/sites.dat index 4b96d627..60c17680 100644 Binary files a/config/sites.dat and b/config/sites.dat differ diff --git a/version.py b/version.py index c9d14f53..1fbb60b1 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -APP_VERSION = 'v3.2.2' +APP_VERSION = 'v3.2.3'