v2.5.35: 优化部分下载失败时异常处理机制，更新文档 (#412)

Author: hect0x7

assets/docs/sources/tutorial/0_common_usage.md

@@ -82,25 +82,38 @@ from jmcomic import *
 # 客户端
 client = JmOption.default().new_jm_client()
 
-# 捕获jmcomic可能出现的异常
+# 捕获获取本子/章节详情时可能出现的异常
 try:
     # 请求本子实体类
     album: JmAlbumDetail = client.get_album_detail('427413')
 except MissingAlbumPhotoException as e:
     print(f'id={e.error_jmid}的本子不存在')
-    
+
 except JsonResolveFailException as e:
     print(f'解析json失败')
     # 响应对象
     resp = e.resp
     print(f'resp.text: {resp.text}, resp.status_code: {resp.status_code}')
-    
+
 except RequestRetryAllFailException as e:
     print(f'请求失败，重试次数耗尽')
-    
+
 except JmcomicException as e:
     # 捕获所有异常，用作兜底
     print(f'jmcomic遇到异常: {e}')
+
+# 多线程下载时，可能出现非当前线程下载失败，抛出异常，
+# 而JmDownloader有对应字段记录了这些线程发生的异常
+# 使用check_exception=True参数可以使downloader主动检查是否存在下载异常
+# 如果有，则当前线程会主动上抛一个PartialDownloadFailedException异常
+# 该参数主要用于主动检查部分下载失败的情况，
+# 因为非当前线程抛出的异常（比如下载章节的线程和下载图片的线程），这些线程如果抛出异常，
+# 当前线程是感知不到的，try-catch下载方法download_album不能捕获到其他线程发生的异常。
+try:
+    album, downloader = download_album(123, check_exception=True)
+except PartialDownloadFailedException as e:
+    downloader: JmDownloader = e.downloader
+    print(f'下载出现部分失败, 下载失败的章节: {downloader.download_failed_photo}, 下载失败的图片: {downloader.download_failed_image}')
 ```
 
 

src/jmcomic/__init__.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.5.34'
+__version__ = '2.5.35'
 
 from .api import *
 from .jm_plugin import *

src/jmcomic/api.py

@@ -48,6 +48,7 @@ def download_album(jm_album_id,
                    option=None,
                    downloader=None,
                    callback=None,
+                   check_exception=True,
                    ) -> Union[__DOWNLOAD_API_RET, Set[__DOWNLOAD_API_RET]]:
     """
     下载一个本子（album），包含其所有的章节（photo）
@@ -58,6 +59,7 @@ def download_album(jm_album_id,
     :param option: 下载选项
     :param downloader: 下载器类
     :param callback: 返回值回调函数，可以拿到 album 和 downloader
+    :param check_exception: 是否检查异常, 如果为True，会检查downloader是否有下载异常，并上抛PartialDownloadFailedException
     :return: 对于的本子实体类，下载器（如果是上述的批量情况，返回值为download_batch的返回值）
     """
 
@@ -69,14 +71,17 @@ def download_album(jm_album_id,
 
         if callback is not None:
             callback(album, dler)
-
+        if check_exception:
+            dler.raise_if_has_exception()
         return album, dler
 
 
 def download_photo(jm_photo_id,
                    option=None,
                    downloader=None,
-                   callback=None):
+                   callback=None,
+                   check_exception=True,
+                   ):
     """
     下载一个章节（photo），参数同 download_album
     """
@@ -88,7 +93,8 @@ def download_photo(jm_photo_id,
 
         if callback is not None:
             callback(photo, dler)
-
+        if check_exception:
+            dler.raise_if_has_exception()
         return photo, dler
 
 

src/jmcomic/jm_client_impl.py

@@ -566,7 +566,7 @@ def check_special_text(cls, resp):
 
             cls.raise_request_error(
                 resp,
-                f'{reason}'
+                f'{reason}({content})'
                 + (f': {url}' if url is not None else '')
             )
 

src/jmcomic/jm_downloader.py

@@ -1,6 +1,31 @@
 from .jm_option import *
 
 
+def catch_exception(func):
+    from functools import wraps
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        self: JmDownloader
+        try:
+            return func(self, *args, **kwargs)
+        except Exception as e:
+            detail: JmBaseEntity = args[0]
+            if detail.is_image():
+                detail: JmImageDetail
+                jm_log('image.failed', f'图片下载失败: [{detail.download_url}], 异常: [{e}]')
+                self.download_failed_image.append((detail, e))
+
+            elif detail.is_photo():
+                detail: JmPhotoDetail
+                jm_log('photo.failed', f'章节下载失败: [{detail.id}], 异常: [{e}]')
+                self.download_failed_photo.append((detail, e))
+
+            raise e
+
+    return wrapper
+
+
 # noinspection PyMethodMayBeStatic
 class DownloadCallback:
 
@@ -50,48 +75,50 @@ class JmDownloader(DownloadCallback):
 
     def __init__(self, option: JmOption) -> None:
         self.option = option
+        self.client = option.build_jm_client()
         # 下载成功的记录dict
         self.download_success_dict: Dict[JmAlbumDetail, Dict[JmPhotoDetail, List[Tuple[str, JmImageDetail]]]] = {}
         # 下载失败的记录list
-        self.download_failed_list: List[Tuple[JmImageDetail, BaseException]] = []
+        self.download_failed_image: List[Tuple[JmImageDetail, BaseException]] = []
+        self.download_failed_photo: List[Tuple[JmPhotoDetail, BaseException]] = []
 
     def download_album(self, album_id):
-        client = self.client_for_album(album_id)
-        album = client.get_album_detail(album_id)
-        self.download_by_album_detail(album, client)
+        album = self.client.get_album_detail(album_id)
+        self.download_by_album_detail(album)
         return album
 
-    def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
+    def download_by_album_detail(self, album: JmAlbumDetail):
         self.before_album(album)
         if album.skip:
             return
-        self.execute_by_condition(
+        self.execute_on_condition(
             iter_objs=album,
-            apply=lambda photo: self.download_by_photo_detail(photo, client),
+            apply=self.download_by_photo_detail,
             count_batch=self.option.decide_photo_batch_count(album)
         )
         self.after_album(album)
 
     def download_photo(self, photo_id):
-        client = self.client_for_photo(photo_id)
-        photo = client.get_photo_detail(photo_id)
-        self.download_by_photo_detail(photo, client)
+        photo = self.client.get_photo_detail(photo_id)
+        self.download_by_photo_detail(photo)
         return photo
 
-    def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
-        client.check_photo(photo)
+    @catch_exception
+    def download_by_photo_detail(self, photo: JmPhotoDetail):
+        self.client.check_photo(photo)
 
         self.before_photo(photo)
         if photo.skip:
             return
-        self.execute_by_condition(
+        self.execute_on_condition(
             iter_objs=photo,
-            apply=lambda image: self.download_by_image_detail(image, client),
+            apply=self.download_by_image_detail,
             count_batch=self.option.decide_image_batch_count(photo)
         )
         self.after_photo(photo)
 
-    def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
+    @catch_exception
+    def download_by_image_detail(self, image: JmImageDetail):
         img_save_path = self.option.decide_image_filepath(image)
 
         image.save_path = img_save_path
@@ -110,22 +137,15 @@ def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
         if use_cache is True and image.exists:
             return
 
-        try:
-            client.download_by_image_detail(
-                image,
-                img_save_path,
-                decode_image=decode_image,
-            )
-        except BaseException as e:
-            jm_log('image.failed', f'图片下载失败: [{image.download_url}], 异常: {e}')
-            # 保存失败记录
-            self.download_failed_list.append((image, e))
-            raise
+        self.client.download_by_image_detail(
+            image,
+            img_save_path,
+            decode_image=decode_image,
+        )
 
         self.after_image(image, img_save_path)
 
-    # noinspection PyMethodMayBeStatic
-    def execute_by_condition(self,
+    def execute_on_condition(self,
                              iter_objs: DetailEntity,
                              apply: Callable,
                              count_batch: int,
@@ -166,20 +186,6 @@ def do_filter(self, detail: DetailEntity):
         """
         return detail
 
-    # noinspection PyUnusedLocal
-    def client_for_album(self, jm_album_id) -> JmcomicClient:
-        """
-        默认情况下，所有的JmDownloader共用一个JmcomicClient
-        """
-        return self.option.build_jm_client()
-
-    # noinspection PyUnusedLocal
-    def client_for_photo(self, jm_photo_id) -> JmcomicClient:
-        """
-        默认情况下，所有的JmDownloader共用一个JmcomicClient
-        """
-        return self.option.build_jm_client()
-
     @property
     def all_success(self) -> bool:
         """
@@ -189,7 +195,7 @@ def all_success(self) -> bool:
 
         注意！如果使用了filter机制，例如通过filter只下载3张图片，那么all_success也会为False
         """
-        if len(self.download_failed_list) != 0:
+        if self.has_download_failures:
             return False
 
         for album, photo_dict in self.download_success_dict.items():
@@ -202,6 +208,10 @@ def all_success(self) -> bool:
 
         return True
 
+    @property
+    def has_download_failures(self):
+        return len(self.download_failed_image) != 0 or len(self.download_failed_photo) != 0
+
     # 下面是回调方法
 
     def before_album(self, album: JmAlbumDetail):
@@ -259,6 +269,23 @@ def after_image(self, image: JmImageDetail, img_save_path):
             downloader=self,
         )
 
+    def raise_if_has_exception(self):
+        if not self.has_download_failures:
+            return
+        msg_ls = ['部分下载失败', '', '']
+
+        if len(self.download_failed_photo) != 0:
+            msg_ls[1] = f'共{len(self.download_failed_photo)}个章节下载失败: {self.download_failed_photo}'
+
+        if len(self.download_failed_image) != 0:
+            msg_ls[2] = f'共{len(self.download_failed_image)}个图片下载失败: {self.download_failed_image}'
+
+        ExceptionTool.raises(
+            '\n'.join(msg_ls),
+            {'downloader': self},
+            PartialDownloadFailedException,
+        )
+
     # 下面是对with语法的支持
 
     def __enter__(self):
@@ -283,7 +310,7 @@ class DoNotDownloadImage(JmDownloader):
     不会下载任何图片的Downloader，用作测试
     """
 
-    def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
+    def download_by_image_detail(self, image: JmImageDetail):
         # ensure make dir
         self.option.decide_image_filepath(image)
 
@@ -297,12 +324,13 @@ class JustDownloadSpecificCountImage(JmDownloader):
     count_lock = Lock()
     count = 0
 
-    def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
+    @catch_exception
+    def download_by_image_detail(self, image: JmImageDetail):
         # ensure make dir
         self.option.decide_image_filepath(image)
 
         if self.try_countdown():
-            return super().download_by_image_detail(image, client)
+            return super().download_by_image_detail(image)
 
     def try_countdown(self):
         if self.count < 0:

src/jmcomic/jm_entity.py

@@ -125,10 +125,9 @@ def idoname(self):
         return f'[{self.id}] {self.oname}'
 
     def __str__(self):
-        return f'{self.__class__.__name__}' \
-               '{' \
-               f'{self.id}: {self.title}' \
-               '}'
+        return f'''{self.__class__.__name__}({self.__alias__()}-{self.id}: "{self.title}")'''
+
+    __repr__ = __str__
 
     @classmethod
     def __alias__(cls):
@@ -258,6 +257,11 @@ def tag(self) -> str:
     def is_image(cls):
         return True
 
+    def __str__(self):
+        return f'''{self.__class__.__name__}(image-[{self.download_url}])'''
+
+    __repr__ = __str__
+
 
 class JmPhotoDetail(DetailEntity, Downloadable):
 

src/jmcomic/jm_exception.py

@@ -15,6 +15,7 @@ def from_context(self, key):
     def __str__(self):
         return self.msg
 
+
 class ResponseUnexpectedException(JmcomicException):
     description = '响应不符合预期异常'
 
@@ -44,7 +45,6 @@ def pattern(self):
 
 class JsonResolveFailException(ResponseUnexpectedException):
     description = 'Json解析异常'
-    pass
 
 
 class MissingAlbumPhotoException(ResponseUnexpectedException):
@@ -57,9 +57,15 @@ def error_jmid(self) -> str:
 
 class RequestRetryAllFailException(JmcomicException):
     description = '请求重试全部失败异常'
-    pass
 
 
+class PartialDownloadFailedException(JmcomicException):
+    description = '部分章节或图片下载失败异常'
+
+    @property
+    def downloader(self):
+        return self.from_context(ExceptionTool.CONTEXT_KEY_DOWNLOADER)
+
 class ExceptionTool:
     """
     抛异常的工具
@@ -71,6 +77,7 @@ class ExceptionTool:
     CONTEXT_KEY_HTML = 'html'
     CONTEXT_KEY_RE_PATTERN = 'pattern'
     CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
+    CONTEXT_KEY_DOWNLOADER = 'downloader'
 
     @classmethod
     def raises(cls,