优化番剧添加重复的问题

backend/dev.sh

@@ -25,4 +25,4 @@ if [ ! -f "$VERSION_FILE" ]; then
 	echo "VERSION='DEV_VERSION'" >>"$VERSION_FILE"
 fi
 
-../venv/bin/uvicorn main:app --reload --port 7892
+../venv/bin/uvicorn main:app --reload --port 7892 --host 0.0.0.0

backend/src/module/checker/checker.py

@@ -5,6 +5,7 @@
 
 from module.conf import VERSION, settings
 from module.downloader import DownloadClient
+from module.rss import RSSEngine
 from module.models import Config
 from module.update import version_check
 
@@ -83,3 +84,12 @@ def check_img_cache() -> bool:
         else:
             img_path.mkdir()
             return False
+
+    @staticmethod
+    def check_torrent_table() -> bool:
+        with RSSEngine() as engine:
+            table_info = engine.exec("PRAGMA table_info(torrent)").all()
+            for item in table_info:
+                if(item[1] == 'save_path'):
+                    return False
+        return True

backend/src/module/core/program.py

@@ -2,7 +2,7 @@
 
 from module.conf import VERSION, settings
 from module.models import ResponseModel
-from module.update import data_migration, first_run, from_30_to_31, start_up, cache_image
+from module.update import data_migration, torrent_table_migration, first_run, from_30_to_31, start_up, cache_image
 
 from .sub_thread import RenameThread, RSSThread
 
@@ -46,6 +46,9 @@ def startup(self):
             # Update database
             from_30_to_31()
             logger.info("[Core] Database updated.")
+        if self.update_torrent_table:
+            torrent_table_migration()
+            logger.info("[Core] Torrent table updated.")
         if not self.img_cache:
             logger.info("[Core] No image cache exists, create image cache.")
             cache_image()

backend/src/module/core/status.py

@@ -58,3 +58,7 @@ def database(self):
     @property
     def img_cache(self):
         return self.check_img_cache()
+
+    @property
+    def update_torrent_table(self):
+        return self.check_torrent_table()

backend/src/module/database/bangumi.py

@@ -96,6 +96,16 @@ def search_id(self, _id: int) -> Optional[Bangumi]:
         else:
             logger.debug(f"[Database] Find bangumi id: {_id}.")
             return self.session.exec(statement).first()
+
+    def search_official_title(self, _official_title: str) -> Optional[Bangumi]:
+        statement = select(Bangumi).where(Bangumi.official_title == _official_title)
+        bangumi = self.session.exec(statement).first()
+        if bangumi is None:
+            logger.debug(f"[Database] Cannot find bangumi title: {_official_title}.")
+            return None
+        else:
+            logger.debug(f"[Database] Find bangumi title: {_official_title}.")
+            return self.session.exec(statement).first()
 
     def match_poster(self, bangumi_name: str) -> str:
         # Use like to match

backend/src/module/database/combine.py

@@ -24,6 +24,10 @@ def create_table(self):
     def drop_table(self):
         SQLModel.metadata.drop_all(self.engine)
 
+    def migrate_torrent(self):
+        self.exec("ALTER TABLE torrent ADD COLUMN save_path varchar(255)")
+        self.exec("ALTER TABLE torrent ADD COLUMN finished int")
+
     def migrate(self):
         # Run migration online
         bangumi_data = self.bangumi.search_all()

backend/src/module/database/torrent.py

@@ -44,6 +44,9 @@ def search(self, _id: int) -> Torrent:
     def search_all(self) -> list[Torrent]:
         return self.session.exec(select(Torrent)).all()
 
+    def search_finished(self, _save_path = "") -> list[Torrent]:
+        return self.session.exec(select(Torrent).where(Torrent.save_path == _save_path)).first()
+
     def search_rss(self, rss_id: int) -> list[Torrent]:
         return self.session.exec(select(Torrent).where(Torrent.rss_id == rss_id)).all()
 

backend/src/module/downloader/download_client.py

@@ -135,17 +135,20 @@ def add_torrent(self, torrent: Torrent | list, bangumi: Bangumi) -> bool:
                 else:
                     torrent_file = req.get_content(torrent.url)
                     torrent_url = None
-        if self.client.add_torrents(
-            torrent_urls=torrent_url,
-            torrent_files=torrent_file,
-            save_path=bangumi.save_path,
-            category="Bangumi",
-        ):
-            logger.debug(f"[Downloader] Add torrent: {bangumi.official_title}")
-            return True
-        else:
-            logger.debug(f"[Downloader] Torrent added before: {bangumi.official_title}")
-            return False
+        try:
+            if self.client.add_torrents(
+                torrent_urls=torrent_url,
+                torrent_files=torrent_file,
+                save_path=bangumi.save_path,
+                category="Bangumi",
+            ):
+                logger.debug(f"[Downloader] Add torrent: {bangumi.official_title}")
+                return True
+            else:
+                logger.debug(f"[Downloader] Torrent added before: {bangumi.official_title}")
+                return False
+        except Exception as e:
+            logger.debug(f"[Downloader] error: {str(e)}")
 
     def move_torrent(self, hashes, location):
         self.client.move_torrent(hashes=hashes, new_location=location)

backend/src/module/manager/collector.py

@@ -18,16 +18,26 @@ def collect_season(self, bangumi: Bangumi, link: str = None):
                 torrents = st.search_season(bangumi)
             else:
                 torrents = st.get_torrents(link, bangumi.filter.replace(",", "|"))
-            if self.add_torrent(torrents, bangumi):
+            # 去除重复的种子
+            new_torrents = []
+            save_path_list = []
+            for item in torrents:
+                if(item.save_path in save_path_list):
+                    continue
+                torrents_info = engine.torrent.search_finished(item.save_path)
+                if(not torrents_info):
+                    new_torrents.append(item)
+                    save_path_list.append(item.save_path)
+            if self.add_torrent(new_torrents, bangumi):
                 logger.info(
                     f"Collections of {bangumi.official_title} Season {bangumi.season} completed."
                 )
-                for torrent in torrents:
+                for torrent in new_torrents:
                     torrent.downloaded = True
                 bangumi.eps_collect = True
                 if engine.bangumi.update(bangumi):
                     engine.bangumi.add(bangumi)
-                engine.torrent.add_all(torrents)
+                engine.torrent.add_all(new_torrents)
                 return ResponseModel(
                     status=True,
                     status_code=200,

backend/src/module/manager/renamer.py

@@ -143,32 +143,35 @@ def rename(self) -> list[Notification]:
         torrents_info = self.get_torrent_info()
         renamed_info: list[Notification] = []
         for info in torrents_info:
-            media_list, subtitle_list = self.check_files(info)
-            bangumi_name, season = self._path_to_bangumi(info.save_path)
-            kwargs = {
-                "torrent_name": info.name,
-                "bangumi_name": bangumi_name,
-                "method": rename_method,
-                "season": season,
-                "_hash": info.hash,
-            }
-            # Rename single media file
-            if len(media_list) == 1:
-                notify_info = self.rename_file(media_path=media_list[0], **kwargs)
-                if notify_info:
-                    renamed_info.append(notify_info)
-                # Rename subtitle file
-                if len(subtitle_list) > 0:
-                    self.rename_subtitles(subtitle_list=subtitle_list, **kwargs)
-            # Rename collection
-            elif len(media_list) > 1:
-                logger.info("[Renamer] Start rename collection")
-                self.rename_collection(media_list=media_list, **kwargs)
-                if len(subtitle_list) > 0:
-                    self.rename_subtitles(subtitle_list=subtitle_list, **kwargs)
-                self.set_category(info.hash, "BangumiCollection")
-            else:
-                logger.warning(f"[Renamer] {info.name} has no media file")
+            try:
+                media_list, subtitle_list = self.check_files(info)
+                bangumi_name, season = self._path_to_bangumi(info.save_path)
+                kwargs = {
+                    "torrent_name": info.name,
+                    "bangumi_name": bangumi_name,
+                    "method": rename_method,
+                    "season": season,
+                    "_hash": info.hash,
+                }
+                # Rename single media file
+                if len(media_list) == 1:
+                    notify_info = self.rename_file(media_path=media_list[0], **kwargs)
+                    if notify_info:
+                        renamed_info.append(notify_info)
+                    # Rename subtitle file
+                    if len(subtitle_list) > 0:
+                        self.rename_subtitles(subtitle_list=subtitle_list, **kwargs)
+                # Rename collection
+                elif len(media_list) > 1:
+                    logger.info("[Renamer] Start rename collection")
+                    self.rename_collection(media_list=media_list, **kwargs)
+                    if len(subtitle_list) > 0:
+                        self.rename_subtitles(subtitle_list=subtitle_list, **kwargs)
+                    self.set_category(info.hash, "BangumiCollection")
+                else:
+                    logger.warning(f"[Renamer] {info.name} has no media file")
+            except Exception as e:
+                logger.error(f"[Renamer] error: {str(e)}")
         logger.debug("[Renamer] Rename process finished.")
         return renamed_info
 

backend/src/module/manager/torrent.py

@@ -1,9 +1,11 @@
 import logging
 
+from module.conf import settings
 from module.database import Database
 from module.downloader import DownloadClient
-from module.models import Bangumi, BangumiUpdate, ResponseModel
+from module.models import Torrent, Bangumi, BangumiUpdate, ResponseModel
 from module.parser import TitleParser
+from module.manager.renamer import Renamer
 
 logger = logging.getLogger(__name__)
 
@@ -174,6 +176,25 @@ def search_one(self, _id: int | str):
             return data
 
 
+    def refine_torrent(self, bangumi: Bangumi, torrent: Torrent):
+        rename_method = settings.bangumi_manage.rename_method
+        # 查找视频是否包含关键字
+        keywords = ['内嵌','外挂','封装','MKV','内挂']
+        suffix = 'mp4'
+        for item1 in keywords:
+            if(item1 in torrent.name):
+                suffix = 'mkv'
+        # 解析视频
+        ep = TitleParser().torrent_parser(
+            torrent_path=f"{torrent.name}.{suffix}"
+        )
+        if(ep):
+            ep_name = Renamer().gen_path(ep, bangumi.official_title, method=rename_method)
+            torrent.bangumi_id = bangumi.id
+            torrent.save_path = f'{bangumi.save_path}/{ep_name}'
+            return torrent
+        return False
+
 if __name__ == "__main__":
     with TorrentManager() as manager:
         manager.refresh_poster()

backend/src/module/models/torrent.py

@@ -12,6 +12,8 @@ class Torrent(SQLModel, table=True):
     url: str = Field("https://example.com/torrent", alias="url")
     homepage: Optional[str] = Field(None, alias="homepage")
     downloaded: bool = Field(False, alias="downloaded")
+    save_path: Optional[str] = Field(None, alias="save_path")
+    finished: bool = Field(False, alias="finished")
 
 
 class TorrentUpdate(SQLModel):

backend/src/module/rss/analyser.py

@@ -5,7 +5,7 @@
 from module.models import Bangumi, ResponseModel, RSSItem, Torrent
 from module.network import RequestContent
 from module.parser import TitleParser
-
+from module.downloader.path import TorrentPath
 from .engine import RSSEngine
 
 logger = logging.getLogger(__name__)
@@ -66,13 +66,21 @@ def torrent_to_data(self, torrent: Torrent, rss: RSSItem) -> Bangumi:
     def rss_to_data(
         self, rss: RSSItem, engine: RSSEngine, full_parse: bool = True
     ) -> list[Bangumi]:
-        rss_torrents = self.get_rss_torrents(rss.url, full_parse)
-        torrents_to_add = engine.bangumi.match_list(rss_torrents, rss.url)
-        if not torrents_to_add:
-            logger.debug("[RSS] No new title has been found.")
-            return []
+        all_rss_torrents = self.get_rss_torrents(rss.url, full_parse)
+        new_data = []
+        title_list = []
+        for torrent in all_rss_torrents:
+            bangumi = self.torrent_to_data(torrent, rss)
+            if(not bangumi):
+                continue
+            if(f'{bangumi.official_title}{bangumi.season}' in title_list):
+                continue
+            title_list.append(f'{bangumi.official_title}{bangumi.season}')
+            q_bangumi = engine.bangumi.search_official_title(bangumi.official_title)
+            if(not q_bangumi):
+                bangumi.save_path = TorrentPath()._gen_save_path(bangumi)
+                new_data.append(bangumi)
         # New List
-        new_data = self.torrents_to_data(torrents_to_add, rss, full_parse)
         if new_data:
             # Add to database
             engine.bangumi.add_all(new_data)

backend/src/module/rss/engine.py

@@ -6,6 +6,7 @@
 from module.downloader import DownloadClient
 from module.models import Bangumi, ResponseModel, RSSItem, Torrent
 from module.network import RequestContent
+from module.manager.torrent import TorrentManager
 
 logger = logging.getLogger(__name__)
 
@@ -97,6 +98,8 @@ def delete_list(self, rss_id_list: list[int]):
     def pull_rss(self, rss_item: RSSItem) -> list[Torrent]:
         torrents = self._get_torrents(rss_item)
         new_torrents = self.torrent.check_new(torrents)
+        # 检查相同视频是否已经呗下载过
+        #for torrent in new_torrents
         return new_torrents
 
     def match_torrent(self, torrent: Torrent) -> Optional[Bangumi]:
@@ -120,14 +123,29 @@ def refresh_rss(self, client: DownloadClient, rss_id: Optional[int] = None):
         # From RSS Items, get all torrents
         logger.debug(f"[Engine] Get {len(rss_items)} RSS items")
         for rss_item in rss_items:
-            new_torrents = self.pull_rss(rss_item)
+            pulled_torrents = self.pull_rss(rss_item)
             # Get all enabled bangumi data
-            for torrent in new_torrents:
+            save_path_list = []
+            new_torrents = []
+            for torrent in pulled_torrents:
                 matched_data = self.match_torrent(torrent)
                 if matched_data:
-                    if client.add_torrent(torrent, matched_data):
-                        logger.debug(f"[Engine] Add torrent {torrent.name} to client")
-                    torrent.downloaded = True
+                    torrent = TorrentManager().refine_torrent(matched_data, torrent)
+                    if(torrent):
+                        torrents_info = self.torrent.search_finished(torrent.save_path)
+                        if(not torrents_info):
+                            if(torrent.save_path in save_path_list):
+                                logger.debug(f"[Engine] Ignore torrent {torrent.name}")
+                                continue
+                            save_path_list.append(torrent.save_path)
+                            new_torrents.append(torrent)
+                            if client.add_torrent(torrent, matched_data):
+                                logger.debug(f"[Engine] Add torrent {torrent.name} to client")
+                                torrent.downloaded = True
+                            else:
+                                logger.warning(f"[Engine] Torrent {torrent.name} has already been added")
+                        else:
+                            logger.debug(f"[Engine] Ignore torrent {torrent.name}")
             # Add all torrents to database
             self.torrent.add_all(new_torrents)
 

backend/src/module/searcher/searcher.py

@@ -3,6 +3,7 @@
 
 from module.models import Bangumi, RSSItem, Torrent
 from module.network import RequestContent
+from module.manager.torrent import TorrentManager
 from module.rss import RSSAnalyser
 
 from .provider import search_url
@@ -19,7 +20,7 @@
 BangumiJSON: TypeAlias = str
 
 
-class SearchTorrent(RequestContent, RSSAnalyser):
+class SearchTorrent(RequestContent, RSSAnalyser, TorrentManager):
     def search_torrents(self, rss_item: RSSItem) -> list[Torrent]:
         return self.get_torrents(rss_item.url)
         # torrents = self.get_torrents(rss_item.url)
@@ -48,8 +49,22 @@ def special_url(data: Bangumi, site: str) -> RSSItem:
         keywords = [getattr(data, key) for key in SEARCH_KEY if getattr(data, key)]
         url = search_url(site, keywords)
         return url
+
+    def build_search_url(self, data: Bangumi, site: str) -> RSSItem:
+        keywords = []
+        keywords.append(data.official_title)
+        if(int(data.season) != 1):
+            keywords.append(data.season_raw)
+        url = search_url(site, keywords)
+        return url
 
     def search_season(self, data: Bangumi, site: str = "mikan") -> list[Torrent]:
-        rss_item = self.special_url(data, site)
+        rss_item = self.build_search_url(data, site)
         torrents = self.search_torrents(rss_item)
-        return [torrent for torrent in torrents if data.title_raw in torrent.name]
+        new_torrents = []
+        for item1 in torrents:
+            if(str(data.season_raw) in item1.name or data.season_raw == None):
+                res = self.refine_torrent(data, item1)
+                if(res):
+                    new_torrents.append(res)
+        return new_torrents