v2.0.3: 一些bug fix和代码优化 (#51)

Author: hect0x7

.github/workflows/download.yml

@@ -2,19 +2,23 @@ name: 下载JM本子
 
 on:
   push:
-    branches: [ 'workflow', 'workflow_local' ]
+    branches-ignore:
+      - 'master' # master专门用于发布pip和repo介绍
     paths:
-      - '.github/workflows/*.yml' # 工作流定义
-      - 'usage/**/*.py' # 工作流脚本
-      - 'assets/config/*.yml' # option配置文件
-      - 'src/**/*.py' # 源码
-
-  pull_request:
-    branches: [ 'workflow', 'workflow_local' ]
+      - '.github/workflows/download.yml' # 工作流定义
+      - 'usage/workflow_download.py' # 下载脚本
+      - 'assets/config/option_workflow_download.yml' # 配置文件
 
 jobs:
   crawler:
     runs-on: ubuntu-latest
+    env:
+      JM_USERNAME: ${{ secrets.JM_USERNAME }}
+      JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
+      JM_DOWNLOAD_DIR: /home/runner/work/jmcomic/download/
+      ZIP_NAME: '本子.tar.gz'
+      UPLOAD_NAME: '下载完成的本子'
+
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.11
@@ -29,36 +33,27 @@ jobs:
           pip install jmcomic -i https://pypi.org/project --upgrade
 
       - name: 安装依赖项（local）
-        if: ${{ github.ref == 'refs/heads/workflow_local' }}
-        run: |
-          python -m pip install --upgrade pip
-          pip install commonX -i https://pypi.org/project --upgrade
-          pip install -e ./
-
-      - name: 安装依赖项（pull_request）
-        if: ${{ github.event_name == 'pull_request' }}
+        if: ${{ github.ref != 'refs/heads/workflow' }}
         run: |
           python -m pip install --upgrade pip
           pip install commonX -i https://pypi.org/project --upgrade
           pip install -e ./
 
       - name: 运行下载脚本
-        env:
-          JM_USERNAME: ${{ secrets.JM_USERNAME }}
-          JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
         run: |
           cd ./usage/
           python workflow_download.py
 
-      - name: 压缩下载的漫画
+      - name: 压缩文件
         run: |
-          cd /home/runner/work/jmcomic/download/
-          tar -zcvf ../下载完成的本子.tar.gz ./
+          cd $JM_DOWNLOAD_DIR
+          tar -zcvf ../$ZIP_NAME ./
+          mv ../$ZIP_NAME .
 
       - name: 上传结果
         uses: actions/upload-artifact@v3
         with:
-          name: 下载完成的本子
-          path: /home/runner/work/jmcomic/下载完成的本子.tar.gz
+          name: ${{ env.UPLOAD_NAME }}
+          path: ${{ env.JM_DOWNLOAD_DIR }}/${{ env.ZIP_NAME }}
           if-no-files-found: warn
           retention-days: 90

assets/config/option_workflow_download.yml

@@ -4,4 +4,10 @@ version: '2.0'
 
 dir_rule:
   base_dir: /home/runner/work/jmcomic/download/
-  rule: Bd_Aauthor_Atitle_Pindex
+  rule: Bd_Aauthor_Atitle_Pindex
+
+client:
+  postman:
+    meta_data:
+      headers:
+        referer: https://18comic.vip/

src/jmcomic/__init__.py

@@ -2,6 +2,6 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option
 
-__version__ = '2.0.1'
+__version__ = '2.0.3'
 
 from .api import *

src/jmcomic/jm_client_impl.py

@@ -52,7 +52,7 @@ def request_with_retry(self,
             jm_debug(
                 f'请求重试',
                 ', '.join([
-                    f'次数: [{retry_count + 1}/{self.retry_times}]',
+                    f'次数: [{retry_count}/{self.retry_times}]',
                     f'域名: [{domain} ({domain_index}/{len(self.domain_list)})]',
                     f'路径: [{url}]',
                     f'参数: [{kwargs if "login" not in url else "#login_form#"}]'
@@ -194,56 +194,82 @@ def get_jm_html(self, url, require_200=True, **kwargs):
 
         if require_200 is True and resp.status_code != 200:
             write_text('./resp.html', resp.text)
-            self.check_special_http_code(resp.status_code, url)
-            raise AssertionError(f"请求失败，"
-                                 f"响应状态码为{resp.status_code}，"
-                                 f"URL=[{resp.url}]，"
-                                 + (f"响应文本=[{resp.text}]" if len(resp.text) < 50 else
-                                    f'响应文本过长(len={len(resp.text)})，不打印')
-                                 )
+            self.check_special_http_code(resp)
+            self.raise_request_error(resp)
+
         # 检查请求是否成功
-        self.require_resp_success_else_raise(resp)
+        self.require_resp_success_else_raise(resp, url)
 
         return resp
 
+    @classmethod
+    def raise_request_error(cls, resp, msg: Optional[str] = None):
+        """
+        请求如果失败，统一由该方法抛出异常
+        """
+        if msg is None:
+            msg = f"请求失败，" \
+                  f"响应状态码为{resp.status_code}，" \
+                  f"URL=[{resp.url}]，" \
+                  + (f"响应文本=[{resp.text}]" if len(resp.text) < 200 else
+                     f'响应文本过长(len={len(resp.text)})，不打印'
+                     )
+        raise AssertionError(msg)
+
     def get_jm_image(self, img_url) -> JmImageResp:
         return JmImageResp(self.get(img_url))
 
     @classmethod
-    def require_resp_success_else_raise(cls, resp):
-        # 1. 是否 album_missing
-        resp_url = resp.url
-        if resp_url.endswith('/error/album_missing'):
-            raise AssertionError(f'请求的本子不存在！({resp_url})\n'
-                                 '原因可能为:\n'
-                                 '1. id有误，检查你的本子/章节id\n'
-                                 '2. 该漫画只对登录用户可见，请配置你的cookies\n')
-
-        # 2. 是否是错误html页
-        cls.check_error_html(resp.text.strip(), resp_url)
+    def require_resp_success_else_raise(cls, resp, req_url):
+        # 1. 检查是否 album_missing
+        error_album_missing = '/error/album_missing'
+        if resp.url.endswith(error_album_missing) and not req_url.endswith(error_album_missing):
+            cls.raise_request_error(
+                resp,
+                f'请求的本子不存在！({req_url})\n'
+                '原因可能为:\n'
+                '1. id有误，检查你的本子/章节id\n'
+                '2. 该漫画只对登录用户可见，请配置你的cookies\n'
+            )
+
+        # 2. 是否是特殊的内容
+        cls.check_special_text(resp)
 
     @classmethod
-    def check_error_html(cls, html: str, url=None):
-        html = html.strip()
-        error_msg = JmModuleConfig.JM_ERROR_RESPONSE_HTML.get(html, None)
-        if error_msg is None:
+    def check_special_text(cls, resp):
+        html = resp.text
+        url = resp.url
+
+        if len(html) > 500:
             return
 
-        write_text('./resp.html', html)
-        raise AssertionError(f'{error_msg}'
-                             + (f': {url}' if url is not None else ''))
+        for content, reason in JmModuleConfig.JM_ERROR_RESPONSE_TEXT.items():
+            if content not in html:
+                continue
+
+            write_text('./resp.html', html)
+            cls.raise_request_error(
+                resp,
+                f'{reason}'
+                + (f': {url}' if url is not None else '')
+            )
 
     @classmethod
-    def check_special_http_code(cls, code, url=None):
+    def check_special_http_code(cls, resp):
+        code = resp.status_code
+        url = resp.url
+
         error_msg = JmModuleConfig.JM_ERROR_STATUS_CODE.get(int(code), None)
         if error_msg is None:
             return
 
-        raise AssertionError(f"请求失败，"
-                             f"响应状态码为{code}，"
-                             f'原因为: [{error_msg}], '
-                             + (f'URL=[{url}]' if url is not None else '')
-                             )
+        cls.raise_request_error(
+            resp,
+            f"请求失败，"
+            f"响应状态码为{code}，"
+            f'原因为: [{error_msg}], '
+            + (f'URL=[{url}]' if url is not None else '')
+        )
 
 
 class JmApiClient(AbstractJmClient):

src/jmcomic/jm_client_interface.py

@@ -208,5 +208,6 @@ class JmcomicClient(
     JmImageClient,
     JmDetailClient,
     JmUserClient,
+    Postman,
 ):
     pass

src/jmcomic/jm_config.py

@@ -6,19 +6,20 @@ def default_jm_debug(topic: str, msg: str):
 class JmModuleConfig:
     # 网站相关
     PROT = "https://"
-    _DOMAIN = None
+    DOMAIN = None
     JM_REDIRECT_URL = f'{PROT}jm365.xyz/3YeBdF'  # 永久網域，怕走失的小伙伴收藏起来
     JM_PUB_URL = f'{PROT}jmcomic1.bet'
     JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}'  # index 从1开始
     JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
 
     # 访问JM可能会遇到的异常网页
-    JM_ERROR_RESPONSE_HTML = {
+    JM_ERROR_RESPONSE_TEXT = {
         "Could not connect to mysql! Please check your database settings!": "禁漫服务器内部报错",
         "Restricted Access!": "禁漫拒绝你所在ip地区的访问，你可以选择: 换域名/换代理",
     }
 
     JM_ERROR_STATUS_CODE = {
+        403: 'ip地区禁止访问/爬虫被识别',
         520: '520: Web server is returning an unknown error (禁漫服务器内部报错)',
         524: '524: The origin web server timed out responding to this request. (禁漫服务器处理超时)',
     }
@@ -47,11 +48,11 @@ def domain(cls, postman=None):
         并且设置把 domain 设置为禁漫模块的默认域名。
         这样一来，配置文件也不用配置域名了，一切都在运行时动态获取。
         """
-        if cls._DOMAIN is None:
+        if cls.DOMAIN is None:
             from .jm_toolkit import JmcomicText
-            cls._DOMAIN = JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
+            cls.DOMAIN = JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
 
-        return cls._DOMAIN  # jmcomic默认域名
+        return cls.DOMAIN  # jmcomic默认域名
 
     @classmethod
     def headers(cls, authority=None):
@@ -60,12 +61,13 @@ def headers(cls, authority=None):
             'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
                       'application/signed-exchange;v=b3;q=0.7',
             'accept-language': 'zh-CN,zh;q=0.9',
+            'referer': 'https://18comic.vip',
             'sec-ch-ua': '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"',
             'sec-ch-ua-mobile': '?0',
             'sec-ch-ua-platform': '"Windows"',
             'sec-fetch-dest': 'document',
             'sec-fetch-mode': 'navigate',
-            'sec-fetch-site': 'none',
+            'sec-fetch-site': 'same-origin',
             'sec-fetch-user': '?1',
             'upgrade-insecure-requests': '1',
             'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 '

src/jmcomic/jm_option.py

@@ -312,20 +312,28 @@ def to_file(self, filepath=None):
         'api': JmApiClient,
     }
 
-    def build_jm_client(self) -> JmcomicClient:
+    def build_jm_client(self, **kwargs) -> JmcomicClient:
         if self.cache_jm_client is not True:
-            return self.new_jm_client()
+            return self.new_jm_client(**kwargs)
 
         client = self.jm_client_cache
         if client is None:
-            client = self.new_jm_client()
+            client = self.new_jm_client(**kwargs)
             self.jm_client_cache = client
 
         return client
 
-    def new_jm_client(self) -> JmcomicClient:
+    def new_jm_client(self, **kwargs) -> JmcomicClient:
+        postman_conf: dict = self.client.postman.src_dict
+
+        # support overwrite meta_data
+        if len(kwargs) != 0:
+            meta_data = postman_conf.get('meta_data', {})
+            meta_data.update(kwargs)
+            postman_conf['meta_data'] = meta_data
+
         # postman
-        postman = Postmans.create(data=self.client.postman)
+        postman = Postmans.create(data=postman_conf)
 
         # domain_list
         domain_list = self.client.domain