v2.3.17: 支持从浏览器获取cookies自动登录禁漫【插件】; 重构Client缓存机制、支持配置缓存级别 (#161)

Author: hect0x7

README.md

@@ -54,7 +54,7 @@ $ jmcomic 422866
 - **可扩展性强**
 
   - **支持Plugin插件，可以方便地扩展功能，以及使用别人的插件**
-    - 目前内置支持的插件有：`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` `日志主题过滤插件`
+  - 目前内置支持的插件有：`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` `日志主题过滤插件` `自动使用浏览器cookies插件`
   - 支持自定义本子/章节/图片下载前后的回调函数
   - 支持自定义debug/logging
   - 支持自定义类：`Downloader（负责调度）` `Option（负责配置）` `Client（负责请求）` `实体类`等

assets/docs/sources/index.md

@@ -25,8 +25,7 @@ Python API for JMComic（禁漫天堂）
 - Highly extensible:
 
   - Supports Plugin plugins for easy functionality extension and use of other plugins.
-    - Currently built-in
-      plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin` `debug logging topic filter plugin`.
+  - Currently built-in plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin` `debug logging topic filter plugin` `auto set browser cookies plugin`.
   - Supports custom callback functions before and after downloading album/chapter/images.
   - Supports custom debug logging.
   - Supports custom core

assets/docs/sources/option_file_syntax.md

@@ -120,6 +120,11 @@ plugins:
         proxy_client_key: cl_proxy_future # 代理类的client_key
         whitelist: [ api, ] # 白名单，当client.impl匹配白名单时才代理
 
+    - plugin: auto_set_browser_cookies # 自动获取浏览器cookies，详见插件类
+      kwargs:
+        browser: chrome
+        domain: 18comic.vip
+
   after_album:
     - plugin: zip # 压缩文件插件
       kwargs:

setup.py

@@ -27,7 +27,7 @@
     package_dir={"": "src"},
     python_requires=">=3.7",
     install_requires=[
-        'commonX>=0.5.7',
+        'commonX>=0.6.2',
         'curl_cffi',
         'PyYAML',
         'Pillow',

src/jmcomic/__init__.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.3.16'
+__version__ = '2.3.17'
 
 from .api import *
 from .jm_plugin import *

src/jmcomic/jm_client_impl.py

@@ -25,6 +25,8 @@ def __init__(self,
             fallback_domain_list.insert(0, domain)
 
         self.domain_list = fallback_domain_list
+        self.CLIENT_CACHE = None
+        self.enable_cache()
         self.after_init()
 
     def after_init(self):
@@ -111,31 +113,59 @@ def debug_topic_request(self):
     def before_retry(self, e, kwargs, retry_count, url):
         jm_debug('req.error', str(e))
 
-    def enable_cache(self, debug=False):
-        if self.is_cache_enabled():
-            return
+    def enable_cache(self):
+        # noinspection PyDefaultArgument,PyShadowingBuiltins
+        def make_key(args, kwds, typed,
+                     kwd_mark=(object(),),
+                     fasttypes={int, str},
+                     tuple=tuple, type=type, len=len):
+            key = args
+            if kwds:
+                key += kwd_mark
+                for item in kwds.items():
+                    key += item
+            if typed:
+                key += tuple(type(v) for v in args)
+                if kwds:
+                    key += tuple(type(v) for v in kwds.values())
+            elif len(key) == 1 and type(key[0]) in fasttypes:
+                return key[0]
+            return hash(key)
 
         def wrap_func_with_cache(func_name, cache_field_name):
             if hasattr(self, cache_field_name):
                 return
 
-            if sys.version_info > (3, 9):
-                import functools
-                cache = functools.cache
-            else:
-                from functools import lru_cache
-                cache = lru_cache()
-
             func = getattr(self, func_name)
-            setattr(self, func_name, cache(func))
+
+            def cache_wrapper(*args, **kwargs):
+                cache = self.CLIENT_CACHE
+
+                # Equivalent to not enable cache
+                if cache is None:
+                    return func(*args, **kwargs)
+
+                key = make_key(args, kwargs, False)
+                sentinel = object()  # unique object used to signal cache misses
+
+                result = cache.get(key, sentinel)
+                if result is not sentinel:
+                    return result
+
+                result = func(*args, **kwargs)
+                cache[key] = result
+                return result
+
+            setattr(self, func_name, cache_wrapper)
 
         for func_name in self.func_to_cache:
             wrap_func_with_cache(func_name, f'__{func_name}.cache.dict__')
 
-        setattr(self, '__enable_cache__', True)
+    def set_cache_dict(self, cache_dict: Optional[Dict]):
+        self.CLIENT_CACHE = cache_dict
 
-    def is_cache_enabled(self) -> bool:
-        return getattr(self, '__enable_cache__', False)
+    def get_cache_dict(self):
+        return self.CLIENT_CACHE
 
     def get_domain_list(self):
         return self.domain_list
@@ -635,7 +665,7 @@ class FutureClientProxy(JmcomicClient):
     client_key = 'cl_proxy_future'
     proxy_methods = ['album_comment', 'enable_cache', 'get_domain_list',
                      'get_html_domain', 'get_html_domain_all', 'get_jm_image',
-                     'is_cache_enabled', 'set_domain_list', ]
+                     'set_cache_dict', 'get_cache_dict', 'set_domain_list', ]
 
     class FutureWrapper:
         def __init__(self, future):

src/jmcomic/jm_client_interface.py

@@ -160,10 +160,10 @@ def get_photo_detail(self,
     def of_api_url(self, api_path, domain):
         raise NotImplementedError
 
-    def enable_cache(self, debug=False):
+    def set_cache_dict(self, cache_dict: Optional[Dict]):
         raise NotImplementedError
 
-    def is_cache_enabled(self) -> bool:
+    def get_cache_dict(self) -> Optional[Dict]:
         raise NotImplementedError
 
     def check_photo(self, photo: JmPhotoDetail):

src/jmcomic/jm_config.py

@@ -335,7 +335,7 @@ def new_postman(cls, session=False, **kwargs):
             },
         },
         'client': {
-            'cache': None,
+            'cache': None,  # see CacheRegistry
             'domain': [],
             'postman': {
                 'type': 'cffi',

src/jmcomic/jm_option.py

@@ -1,6 +1,58 @@
 from .jm_client_impl import *
 
 
+class CacheRegistry:
+    REGISTRY = {}
+
+    @classmethod
+    def level_option(cls, option, _client):
+        registry = cls.REGISTRY
+        registry.setdefault(option, {})
+        return registry[option]
+
+    @classmethod
+    def level_client(cls, _option, client):
+        registry = cls.REGISTRY
+        registry.setdefault(client, {})
+        return registry[client]
+
+    @classmethod
+    def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient, cache: Union[None, bool, str, Callable]):
+        """
+        cache parameter
+
+        if None: no cache
+
+        if bool:
+          true: level_option
+
+          false: no cache
+
+        if str:
+          (invoke corresponding Cache class method)
+
+        :param option: JmOption
+        :param client: JmcomicClient
+        :param cache: config dsl
+        """
+        if cache is None:
+            return
+
+        elif isinstance(cache, bool):
+            if cache is False:
+                return
+            else:
+                cache = cls.level_option
+
+        elif isinstance(cache, str):
+            func = getattr(cls, cache, None)
+            assert func is not None, f'未实现的cache配置名: {cache}'
+            cache = func
+
+        cache: Callable
+        client.set_cache_dict(cache(option, client))
+
+
 class DirRule:
     rule_sample = [
         # 根目录 / Album-id / Photo-序号 /
@@ -312,12 +364,17 @@ def build_jm_client(self, **kwargs):
         return self.new_jm_client(**kwargs)
 
     def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
+        """
+        创建新的Client（客户端），不同Client之间的元数据不共享
+        """
+        from copy import deepcopy
+
         # 所有需要用到的 self.client 配置项如下
-        postman_conf: dict = self.client.postman.src_dict  # postman dsl 配置
-        meta_data: dict = postman_conf['meta_data']  # 请求元信息
+        postman_conf: dict = deepcopy(self.client.postman.src_dict)  # postman dsl 配置
+        meta_data: dict = postman_conf['meta_data']  # 元数据
         impl: str = impl or self.client.impl  # client_key
         retry_times: int = self.client.retry_times  # 重试次数
-        cache: str = cache or self.client.cache  # 启用缓存
+        cache: str = cache if cache is not None else self.client.cache  # 启用缓存
 
         # domain
         def decide_domain():
@@ -357,11 +414,19 @@ def decide_domain():
         )
 
         # enable cache
-        if cache is True:
-            client.enable_cache()
+        CacheRegistry.enable_client_cache_on_condition(self, client, cache)
 
         return client
 
+    def update_cookies(self, cookies: dict):
+        metadata: dict = self.client.postman.meta_data.src_dict
+        orig_cookies: Optional[Dict] = metadata.get('cookies', None)
+        if orig_cookies is None:
+            metadata['cookies'] = cookies
+        else:
+            orig_cookies.update(cookies)
+            metadata['cookies'] = orig_cookies
+
     # noinspection PyMethodMayBeStatic
     def decide_client_domain(self, client_key: str) -> List[str]:
         is_client_type = lambda ctype: self.client_key_is_given_type(client_key, ctype)

src/jmcomic/jm_plugin.py

@@ -49,6 +49,12 @@ def require_true(self, case: Any, msg: str):
 
         raise PluginValidationException(self, msg)
 
+    def warning_lib_not_install(self, lib='psutil'):
+        msg = (f'插件`{self.plugin_key}`依赖库: {lib}，请先安装{lib}再使用。'
+               f'安装命令: [pip install {lib}]')
+        import warnings
+        warnings.warn(msg)
+
 
 class JmLoginPlugin(JmOptionPlugin):
     """
@@ -111,12 +117,7 @@ def monitor_resource_usage(
         try:
             import psutil
         except ImportError:
-            msg = (f'插件`{self.plugin_key}`依赖psutil库，请先安装psutil再使用。'
-                   f'安装命令: [pip install psutil]')
-            import warnings
-            warnings.warn(msg)
-            # import sys
-            # print(msg, file=sys.stderr)
+            self.warning_lib_not_install('psutil')
             return
 
         from time import sleep
@@ -464,3 +465,55 @@ def new_jm_debug(topic, msg):
             old_jm_debug(topic, msg)
 
         JmModuleConfig.debug_executor = new_jm_debug
+
+
+class AutoSetBrowserCookiesPlugin(JmOptionPlugin):
+    plugin_key = 'auto_set_browser_cookies'
+
+    accepted_cookies_keys = str_to_set('''
+    yuo1
+    remember_id
+    remember
+    ''')
+
+    def invoke(self,
+               browser: str,
+               domain: str,
+               ) -> None:
+        """
+        坑点预警：由于禁漫需要校验同一设备，使用该插件需要配置自己浏览器的headers，例如
+
+        ```yml
+        client:
+          postman:
+            meta_data:
+              headers: {
+               # 浏览器headers
+              }
+
+        # 插件配置如下：
+        plugins:
+          after_init:
+            - plugin: auto_set_browser_cookies
+              kwargs:
+                browser: chrome
+                domain: 18comic.vip
+        ```
+
+        :param browser: chrome/edge/...
+        :param domain: 18comic.vip/...
+        :return: cookies
+        """
+        cookies, e = get_browser_cookies(browser, domain, safe=True)
+
+        if cookies is None:
+            if isinstance(e, ImportError):
+                self.warning_lib_not_install('browser_cookie3')
+            else:
+                self.debug('获取浏览器cookies失败，请关闭浏览器重试')
+            return
+
+        self.option.update_cookies(
+            {k: v for k, v in cookies.items() if k in self.accepted_cookies_keys}
+        )
+        self.debug('获取浏览器cookies成功')

tests/test_jmcomic/__init__.py

@@ -46,11 +46,7 @@ def tearDown(self) -> None:
     @classmethod
     def setUpClass(cls):
         # 设置 JmOption，JmcomicClient
-        try:
-            option = create_option_by_env('JM_OPTION_PATH_TEST')
-        except JmcomicException:
-            option = create_option('./assets/option/option_test.yml')
-
+        option = cls.new_option()
         cls.option = option
         cls.client = option.build_jm_client()
 
@@ -61,6 +57,13 @@ def setUpClass(cls):
             return
         cost_time_dict[cls.__name__] = ts()
 
+    @classmethod
+    def new_option(cls):
+        try:
+            return create_option_by_env('JM_OPTION_PATH_TEST')
+        except JmcomicException:
+            return create_option('./assets/option/option_test.yml')
+
     @classmethod
     def tearDownClass(cls) -> None:
         if skip_time_cost_debug: