support for novelsave_sources v0.2.0

Author: m-haisham

novelsave/cli/helpers/novel.py

@@ -43,7 +43,7 @@ def retrieve_novel_info(source_gateway: BaseSourceGateway, url: str, browser: st
     logger.info(f"Retrieving novel information ({url=})...")
     try:
         output = source_gateway.novel_by_url(url)
-    except requests.ConnectionError as e:
+    except requests.ConnectionError:
         raise NSError(f"Connection terminated unexpectedly; Make sure you are connected to the internet.")
 
     return output
@@ -61,13 +61,14 @@ def create_novel(
     this includes chapter list and metadata.
     """
     source_gateway = get_source_gateway(url)
-    novel_dto, chapter_dtos, metadata_dtos = retrieve_novel_info(source_gateway, url, browser)
+    novel_dto = retrieve_novel_info(source_gateway, url, browser)
 
     novel = novel_service.insert_novel(novel_dto)
-    novel_service.insert_chapters(novel, chapter_dtos)
-    novel_service.insert_metadata(novel, metadata_dtos)
+    novel_service.insert_chapters(novel, novel_dto.volumes)
+    novel_service.insert_metadata(novel, novel_dto.metadata)
 
-    logger.info(f"Added new novel (id={novel.id}, title='{novel.title}', chapters={len(chapter_dtos)}').")
+    chapters = [c for v in novel_dto.volumes for c in v.chapters]
+    logger.info(f"Added new novel (id={novel.id}, title='{novel.title}', chapters={len(chapters)}').")
 
     data_dir = path_service.novel_data_path(novel)
     if data_dir.exists():
@@ -87,13 +88,14 @@ def update_novel(
     logger.debug(f"Using primary url ({url=})")
 
     source_gateway = get_source_gateway(url)
-    novel_dto, chapter_dtos, metadata_dtos = retrieve_novel_info(source_gateway, url, browser)
+    novel_dto = retrieve_novel_info(source_gateway, url, browser)
 
     novel_service.update_novel(novel, novel_dto)
-    novel_service.update_chapters(novel, chapter_dtos)
-    novel_service.update_metadata(novel, metadata_dtos)
+    novel_service.update_chapters(novel, novel_dto.volumes)
+    novel_service.update_metadata(novel, novel_dto.metadata)
 
-    logger.info(f"Updated novel (id={novel.id}, title='{novel.title}', chapters={len(chapter_dtos)})")
+    chapters = [c for v in novel_dto.volumes for c in v.chapters]
+    logger.info(f"Updated novel (id={novel.id}, title='{novel.title}', chapters={len(chapters)})")
     return novel
 
 
@@ -147,7 +149,7 @@ def download_chapters(
     logger.debug(f"Using primary novel url ({url=}).")
 
     source_gateway = get_source_gateway(url)
-    thread_count = 1 or min(threads, os.cpu_count())
+    thread_count = min(threads, os.cpu_count())
 
     def download(dto: ChapterDTO):
         try:
@@ -238,7 +240,7 @@ def get_novel(
     if not novel:
         quote = "'" if is_url else ''
         logger.error(f"Novel not found ({'url' if is_url else 'id'}={quote}{id_or_url}{quote}).")
-        raise ValueError()
+        raise ValueError("Novel was not found.")
 
     if not silent:
         logger.info(f"Acquired novel from database (id={novel.id}, title='{novel.title}').")

novelsave/core/dtos/__init__.py

@@ -1,3 +1,4 @@
 from .novel_dto import NovelDTO
+from .volume_dto import VolumeDTO
 from .chapter_dto import ChapterDTO
 from .metadata_dto import MetaDataDTO

novelsave/core/dtos/chapter_dto.py

@@ -9,4 +9,3 @@ class ChapterDTO:
     title: str
     url: str
     content: Union[str, List[str]] = None
-    volume: Tuple[int, str] = None

novelsave/core/dtos/novel_dto.py

@@ -1,18 +1,24 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Optional
+from typing import Optional, List
+
+from .metadata_dto import MetaDataDTO
+from .volume_dto import VolumeDTO
 
 
 @dataclass
 class NovelDTO:
     id: Optional[int]
     title: str
     url: str
-    author: str = '<Not specified>'
+    author: str = None
     synopsis: str = None
 
     thumbnail_url: str = None
     thumbnail_path: str = None
 
     lang: str = 'en'
     last_updated: datetime = None
+
+    volumes: List[VolumeDTO] = field(default_factory=lambda: [])
+    metadata: List[MetaDataDTO] = field(default_factory=lambda: [])

novelsave/core/dtos/volume_dto.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from .chapter_dto import ChapterDTO
+
+
+@dataclass
+class VolumeDTO:
+    id: Optional[int]
+    index: int
+    name: str
+
+    chapters: List[ChapterDTO] = field(default_factory=lambda: [])

novelsave/core/services/novel/base_novel_service.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Optional, List, Dict
 
-from novelsave.core.dtos import NovelDTO, ChapterDTO, MetaDataDTO
+from novelsave.core.dtos import NovelDTO, ChapterDTO, MetaDataDTO, VolumeDTO
 from novelsave.core.entities.novel import Novel, Chapter, Volume, MetaData, NovelUrl
 
 
@@ -53,7 +53,7 @@ def insert_novel(self, novel_dto: NovelDTO) -> Novel:
         """insert a new novel into the database"""
 
     @abstractmethod
-    def insert_chapters(self, novel: Novel, chapter_dtos: List[ChapterDTO]):
+    def insert_chapters(self, novel: Novel, volume_dtos: List[VolumeDTO]):
         """insert new chapters into the database as well as their volumes"""
 
     @abstractmethod
@@ -69,7 +69,7 @@ def update_novel(self, novel: Novel, novel_dto: NovelDTO):
         """update existing novel with newer values"""
 
     @abstractmethod
-    def update_chapters(self, novel: Novel, chapter_dtos: List[ChapterDTO]):
+    def update_chapters(self, novel: Novel, volume_dtos: List[VolumeDTO]):
         """update and replace existing novel chapters and their corresponding volumes"""
 
     @abstractmethod

novelsave/core/services/source/base_source_gateway.py

@@ -30,7 +30,7 @@ def login(self, username: str, password: str):
         """login to the source website, making available services or novels which might otherwise be absent"""
 
     @abstractmethod
-    def novel_by_url(self, url: str) -> Tuple[dtos.NovelDTO, List[dtos.ChapterDTO], List[dtos.MetaDataDTO]]:
+    def novel_by_url(self, url: str) -> dtos.NovelDTO:
         """scrape and parse a novel by its url"""
 
     @abstractmethod

novelsave/services/novel/novel_service.py

@@ -5,7 +5,7 @@
 from sqlalchemy import delete, select, update
 from sqlalchemy.orm import Session
 
-from novelsave.core.dtos import NovelDTO, ChapterDTO, MetaDataDTO
+from novelsave.core.dtos import NovelDTO, ChapterDTO, MetaDataDTO, VolumeDTO
 from novelsave.core.entities.novel import Novel, NovelUrl, Chapter, Volume, MetaData
 from novelsave.core.services import BaseNovelService
 from novelsave.services import FileService
@@ -83,10 +83,10 @@ def insert_novel(self, novel_dto: NovelDTO) -> Novel:
     def insert_chapters(
             self,
             novel: Novel,
-            chapter_dtos: List[ChapterDTO],
+            volume_dtos: List[VolumeDTO],
             previous: Dict[str, Chapter] = None
     ):
-        volume_mapped_chapters = self.dto_adapter.volumes_from_chapter_dtos(novel, chapter_dtos)
+        volume_mapped_chapters = self.dto_adapter.volumes_from_dto(novel, volume_dtos)
 
         # add volumes
         self.session.add_all(volume_mapped_chapters.keys())
@@ -121,10 +121,10 @@ def set_thumbnail_asset(self, novel: Novel, r_path: Path):
         novel.thumbnail_path = str(r_path)
         self.session.commit()
 
-    def update_chapters(self, novel: Novel, chapter_dtos: List[ChapterDTO]):
+    def update_chapters(self, novel: Novel, volume_dtos: List[VolumeDTO]):
         volumes = self.session.execute(select(Volume).where(Volume.novel_id == novel.id)).scalars().all()
         chapters = self.get_chapters(novel)
-        volume_mapped_chapters = self.dto_adapter.volumes_from_chapter_dtos(novel, chapter_dtos)
+        volume_mapped_chapters = self.dto_adapter.volumes_from_dto(novel, volume_dtos)
 
         indexed_volumes = {v.index: v for v in volumes}
         volumes_to_add = []
@@ -220,7 +220,6 @@ def update_content(self, chapter_dto: ChapterDTO):
         self.session.commit()
 
     def add_url(self, novel: Novel, url: str):
-        # TODO strip trailing '/'
         if url in [novel_url.url for novel_url in self.get_urls(novel)]:
             raise ValueError(f"Url already exists in novel (id={novel.id}, title='{novel.title}', {url=}).")
 

novelsave/services/source/source_gateway.py

@@ -34,14 +34,10 @@ def search(self, keyword: str):
     def login(self, username: str, password: str):
         self.source.login(username, password)
 
-    def novel_by_url(self, url: str) -> Tuple[dtos.NovelDTO, List[dtos.ChapterDTO], List[dtos.MetaDataDTO]]:
-        novel, chapters, metadata = self.source.novel(url)
+    def novel_by_url(self, url: str) -> dtos.NovelDTO:
+        novel = self.source.novel(url)
 
-        internal_novel = self.source_adapter.novel_to_internal(novel)
-        internal_chapters = [self.source_adapter.chapter_to_internal(c) for c in chapters]
-        internal_metadata = [self.source_adapter.metadata_to_internal(m) for m in metadata]
-
-        return internal_novel, internal_chapters, internal_metadata
+        return self.source_adapter.novel_to_internal(novel)
 
     def update_chapter_content(self, chapter: dtos.ChapterDTO) -> dtos.ChapterDTO:
         source_chapter = self.source_adapter.chapter_to_external(chapter)

novelsave/utils/adapters/dto_adapter.py

@@ -1,7 +1,8 @@
 import json
+from pathlib import Path
 from typing import Tuple, List, Dict
 
-from novelsave.core.dtos import NovelDTO, ChapterDTO, MetaDataDTO
+from novelsave.core.dtos import NovelDTO, ChapterDTO, MetaDataDTO, VolumeDTO
 from novelsave.core.entities.novel import Novel, NovelUrl, Chapter, Volume, MetaData
 
 
@@ -18,8 +19,8 @@ def novel_from_dto(self, novel_dto: NovelDTO) -> Tuple[Novel, NovelUrl]:
 
         url = NovelUrl(
             # ensure trailing forward slash, it is preferred that a uri that isn't
-            # a file end in a slash to signify an endpoint
-            url=novel_dto.url.rstrip('/') + '/',
+            # a file end in a slash to signify a path
+            url=novel_dto.url.rstrip('/') + ('/' if Path(novel_dto.url).suffix else ''),
         )
 
         return novel, url
@@ -33,23 +34,16 @@ def update_novel_from_dto(self, novel: Novel, novel_dto: NovelDTO) -> Novel:
 
         return novel
 
-    def volumes_from_chapter_dtos(self, novel: Novel, chapter_dtos: List[ChapterDTO]) -> Dict[Volume, List[ChapterDTO]]:
-        volumes = {}
-        for dto in chapter_dtos:
-            if dto.volume is None:
-                try:
-                    volumes[-1][1].append(dto)
-                except KeyError:
-                    volumes[-1] = (Volume(index=-1, name='_default', novel_id=novel.id), [dto])
-            elif len(dto.volume) < 2:
-                continue
-            else:
-                try:
-                    volumes[dto.volume[0]][1].append(dto)
-                except KeyError:
-                    volumes[dto.volume[0]] = (Volume(index=dto.volume[0], name=dto.volume[1], novel_id=novel.id), [dto])
-
-        return {volume: chapters for volume_index, (volume, chapters) in volumes.items()}
+    def volumes_from_dto(self, novel: Novel, volume_dtos: List[VolumeDTO]) -> Dict[Volume, List[ChapterDTO]]:
+        return {
+            Volume(
+                id=dto.id,
+                index=dto.index,
+                name=dto.name,
+                novel_id=novel.id,
+            ): dto.chapters
+            for dto in volume_dtos
+        }
 
     def chapter_from_dto(self, volume: Volume, chapter_dto: ChapterDTO) -> Chapter:
         return Chapter(

novelsave/utils/adapters/source_adapter.py

@@ -11,10 +11,16 @@ def novel_to_internal(self, novel: sm.Novel) -> dtos.NovelDTO:
         arguments = {
             key: value
             for key, value in vars(novel).items()
-            if key in {'title', 'url', 'author', 'synopsis', 'thumbnail_url', 'lang'}
+            if key in {'title', 'url', 'author', 'thumbnail_url', 'lang'}
         }
 
-        return dtos.NovelDTO(id=None, **arguments)
+        return dtos.NovelDTO(
+            id=None,
+            synopsis='\n'.join(novel.synopsis),
+            volumes=[self.volume_to_internal(c) for c in novel.volumes],
+            metadata=[self.metadata_to_internal(m) for m in novel.metadata],
+            **arguments
+        )
 
     def novel_to_external(self, novel: dtos.NovelDTO) -> sm.Novel:
         """convert novel from source to internal"""
@@ -27,6 +33,16 @@ def novel_to_external(self, novel: dtos.NovelDTO) -> sm.Novel:
 
         return sm.Novel(**arguments)
 
+    def volume_to_internal(self, volume: sm.Volume) -> dtos.VolumeDTO:
+        """convert volume from source to internal"""
+
+        return dtos.VolumeDTO(
+            id=None,
+            index=volume.index,
+            name=volume.name,
+            chapters=[self.chapter_to_internal(c) for c in volume.chapters]
+        )
+
     def chapter_to_internal(self, chapter: sm.Chapter) -> dtos.ChapterDTO:
         """convert chapter from source to internal"""
 
@@ -35,7 +51,6 @@ def chapter_to_internal(self, chapter: sm.Chapter) -> dtos.ChapterDTO:
             title=chapter.title,
             url=chapter.url,
             content=chapter.paragraphs,
-            volume=chapter.volume,
         )
 
     def chapter_to_external(self, chapter: dtos.ChapterDTO) -> sm.Chapter:
@@ -46,7 +61,6 @@ def chapter_to_external(self, chapter: dtos.ChapterDTO) -> sm.Chapter:
             title=chapter.title,
             url=chapter.url,
             paragraphs=chapter.content,
-            volume=chapter.volume,
         )
 
     def metadata_to_internal(self, metadata: sm.Metadata) -> dtos.MetaDataDTO: