Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion ComicSpider/pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,13 @@ def get_media_requests(self, item, info):
raise TypeError(f"{type(spider).__name__}.image_request_meta() must return dict or None")
else:
meta = dict(meta)
requests.append(Request(url,callback=NO_CALLBACK,headers=dict(headers),meta=meta))
request_headers = dict(headers)
extra_headers = meta.get("headers") if isinstance(meta, dict) else None
if extra_headers is not None:
if not isinstance(extra_headers, dict):
raise TypeError(f"{type(spider).__name__}.image_request_meta()['headers'] must return dict when provided")
request_headers.update(dict(extra_headers))
requests.append(Request(url,callback=NO_CALLBACK,headers=request_headers,meta=meta))
return requests

# 图片存储前调用
Expand Down
67 changes: 67 additions & 0 deletions ComicSpider/spiders/dm5.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
import scrapy

from utils.website import Dm5Utils
from .basecomicspider import BaseComicSpider, ComicspiderItem


class Dm5Spider(BaseComicSpider):
name = "dm5"
image_ua = Dm5Utils.image_ua
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {
"ComicSpider.middlewares.RefererMiddleware": 10,
"ComicSpider.middlewares.FakeMiddleware": 30,
}
}

def _build_episode_items(self, ep, page_urls, *, chapter_referer):
book = ep.from_book
uid, u_md5 = ep.id_and_md5()
group_infos = {"title": book.name, "section": ep.name, "uuid": uid, "uuid_md5": u_md5}
ep.pages = len(page_urls)
self.set_task(ep)
if not hasattr(self, "_chapter_referers"):
self._chapter_referers = {}
if not hasattr(self, "_image_request_headers"):
self._image_request_headers = {}
self._chapter_referers[u_md5] = chapter_referer
self._image_request_headers[u_md5] = dict(getattr(ep, "dm5_image_headers", {}) or {})
for page, image_url in enumerate(page_urls, start=1):
item = ComicspiderItem()
item.update(**group_infos)
item["page"] = page
item["image_urls"] = [image_url]
if self.job_context:
self.job_context.total += 1
self.total += 1
yield item

def _yield_episode_items(self, ep, page_urls, *, chapter_referer):
for item in self._build_episode_items(ep, page_urls, chapter_referer=chapter_referer):
yield scrapy.Request(
url=f'https://fakefakefa.com/{item["image_urls"][0]}',
callback=self.process_item,
meta={'item': item, 'referer': chapter_referer},
dont_filter=True,
)
self._emit_process("fin")

def _process_episode(self, ep):
page_urls = list(getattr(ep, "page_urls", None) or [])
chapter_referer = getattr(ep, "chapter_referer", None) or ep.url
if not page_urls or not chapter_referer:
missing = "page_urls" if not page_urls else "chapter_referer"
raise ValueError(f"dm5 episode requires {missing}: {ep!r}")
yield from self._yield_episode_items(ep, page_urls, chapter_referer=chapter_referer)

def image_request_meta(self, *, url, item):
uuid_md5 = item.get("uuid_md5")
referer = getattr(self, "_chapter_referers", {}).get(uuid_md5)
headers = dict(getattr(self, "_image_request_headers", {}).get(uuid_md5) or {})
if referer and "Referer" not in headers and "referer" not in headers:
headers["Referer"] = referer
return {"referer": referer, "headers": headers} if headers or referer else {}

def process_item(self, response):
yield response.meta["item"]
17 changes: 16 additions & 1 deletion GUI/browser_window.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from PySide6.QtCore import Qt, QUrl, QEvent, QSize, Signal, QLoggingCategory
from PySide6.QtGui import QIcon, QKeySequence, QShortcut
from PySide6.QtWebEngineCore import QWebEnginePage, QWebEngineSettings
from qfluentwidgets import InfoBar, InfoBarPosition, FluentIcon as FIF, ToolTipFilter, ToolTipPosition
from qfluentwidgets import InfoBar, InfoBarPosition, FluentIcon as FIF, IndeterminateProgressBar, ToolTipFilter, ToolTipPosition
from qframelesswindow import FramelessMainWindow
from qframelesswindow.webengine import FramelessWebEngineView
from qframelesswindow.utils import startSystemMove
Expand Down Expand Up @@ -193,6 +193,11 @@ def setupUi(self, _window):
self.closeBtn.setIconSize(QSize(20, 20))
self.closeBtn.setIcon(QIcon(':/close.svg'))

self.submitLoadingBar = IndeterminateProgressBar(self.groupBox, start=False)
self.submitLoadingBar.setFixedWidth(120)
self.submitLoadingBar.hide()
self.horizontalLayout_2.insertWidget(self.horizontalLayout_2.count() - 3, self.submitLoadingBar)

self.homeBtn.clicked.connect(self.load_home)
self.backBtn.clicked.connect(self.view.back)
self.forwardBtn.clicked.connect(self.view.forward)
Expand Down Expand Up @@ -425,6 +430,16 @@ def show_task_added_toast(self, title: str):
js_code = f"window.showTaskAddedToast && window.showTaskAddedToast({json.dumps(title)});"
self.page_runtime.run_js(js_code)

def start_submit_loading(self):
self.submitLoadingBar.show()
if not self.submitLoadingBar.isStarted():
self.submitLoadingBar.start()

def stop_submit_loading(self):
if self.submitLoadingBar.isStarted():
self.submitLoadingBar.stop()
self.submitLoadingBar.hide()

def closeEvent(self, event):
self.page_runtime.shutdown()
self.window_mode.shutdown()
Expand Down
30 changes: 19 additions & 11 deletions GUI/gui.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from GUI.manager import (
TaskProgressManager, ClipGUIManager, AggrSearchManager, RVManager,
CGSMidManagerGUI, PreviewMgr, UpdateNotifier, PublishDomainManager,
SelectionFlowManager, DownloadRuntimeManager
SelectionFlowManager, DownloadRuntimeManager, Shares
)
from utils.config.qc import cgs_cfg
from GUI.manager.preprocess import PreprocessManager
Expand Down Expand Up @@ -134,6 +134,7 @@ def generation_bind(self):
self.ags_mgr = AggrSearchManager(self)
self.preview_mgr = PreviewMgr(self)
self.publish_mgr = PublishDomainManager(self)
self.shares = Shares(self)
self.download_state = DownloadStateStore()
self.dl_mgr = DownloadRuntimeManager(self)
self.sel_mgr = SelectionFlowManager(self)
Expand Down Expand Up @@ -273,10 +274,9 @@ def chooseBox_changed_tips(self, index):
self.say(font_color(res.EHentai.GUIDE, cls='theme-highlight'))
case _:
if self.gui_site_runtime is not None:
self.say(
font_color(getattr(self.res, f"{self.gui_site_runtime.name}_desc", ""), cls='theme-highlight'),
ignore_http=True,
)
desc = getattr(self.res, f"{self.gui_site_runtime.name}_desc", None)
if isinstance(desc, str) and desc:
self.say(font_color(desc, cls='theme-highlight'), ignore_http=True)
if index in Spider.mangas():
self.say(font_color(self.res.manga_fav_tip, cls='theme-tip'))

Expand Down Expand Up @@ -346,13 +346,19 @@ def btn_logic_bind(self):
self.aggrBtn.clicked.connect(lambda: self.show_toolWin("ags"))
self.htBtn.clicked.connect(lambda: self.show_toolWin("hitomi"))
self.openPBtn.clicked.connect(lambda: curr_os.open_folder(self.sv_path))
self.shareBtn.clicked.connect(self.shares.upload)
self.shares.changed.connect(self._sync_share_btn_visible)
self._sync_share_btn_visible()
self.domainBtn.clicked.connect(self.do_publish)

_safe_disconnect(self.mpreviewBtn.clicked)
self.mpreviewBtn.clicked.connect(self.show_preview)

self.page_turn_frame()

def _sync_share_btn_visible(self):
self.shareBtn.setVisible(not self.shares.is_empty())

def page_turn_frame(self):
def page_turn(_p):
if not hasattr(self, "preview_mgr"):
Expand Down Expand Up @@ -530,13 +536,12 @@ def start_and_search(self, keyword=None, site_index=None):
if keyword:
self.searchinput.setText(keyword)
kw = self.searchinput.text().strip()
if not kw:
InfoBar.info(
title='', content='先输入搜索词吧', isClosable=True,
position=InfoBarPosition.BOTTOM, duration=2000,
parent=self.textBrowser
)
if kw.startswith("dc:"):
share_id = kw[3:].strip()
self.shares.download(share_id)
return
if not kw:
return InfoBar.info(title='', content='先输入搜索词吧', isClosable=True, position=InfoBarPosition.BOTTOM, duration=2000, parent=self.textBrowser)
site = self.chooseBox.currentIndex()
if site not in SPIDERS or not getattr(self.preview_mgr, "worker", None):
self.refresh_lifecycle_state()
Expand Down Expand Up @@ -683,4 +688,7 @@ def _show_skip_info(self, skip_info: dict):
if tips:
self.say(font_color(f"已跳过:{','.join(tips)}", cls='theme-tip'), ignore_http=True)

def publish_share_books(self, books):
self.preview_mgr.publish_share_books(books)

# ---
3 changes: 3 additions & 0 deletions GUI/mainwindow.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ def preset(self):

def task_init(self):
self.expandBtn = ExpandButton(self)
self.shareBtn = TransparentToolButton(FIF.SHARE, self)
self.clearBtn = TransparentToolButton(FIF.BROOM)
self.repairBtn = TransparentToolButton(QIcon(':/main/repair.svg'), self)
self.repairBtn.setStatusTip("Patch missing page/补漏页")
Expand All @@ -185,10 +186,12 @@ def task_init(self):
self.scroll_area.setWidgetResizable(True)

self.barHLayout.insertWidget(0, self.expandBtn)
self.barHLayout.addWidget(self.shareBtn)
self.barHLayout.addWidget(self.repairBtn)
self.barHLayout.addWidget(self.clearBtn)
self.barVLayout.addWidget(self.scroll_area)

self.shareBtn.setVisible(False)
self.expandBtn.setVisible(False)
self.repairBtn.setVisible(False)
self.clearBtn.setVisible(False)
Expand Down
3 changes: 2 additions & 1 deletion GUI/manager/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,14 @@
from GUI.manager.publish import PublishDomainManager
from GUI.manager.selection import SelectionFlowManager
from GUI.manager.download import DownloadRuntimeManager
from GUI.manager.share import Shares

__all__ = [
'Updater', 'UpdateNotifier', 'TaskConfig',
'RVManager', 'TaskProgressManager','AsyncTaskManager',
'ClipGUIManager', 'AggrSearchManager', 'CGSMidManagerGUI',
'PreviewMgr', 'PublishDomainManager',
'SelectionFlowManager', 'DownloadRuntimeManager'
'SelectionFlowManager', 'DownloadRuntimeManager', 'Shares'
]


Expand Down
4 changes: 4 additions & 0 deletions GUI/manager/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ def resubmit_download(self, task_id: str):
site_index, task_info = self._submitted_task_infos[task_id]
self.submit_download(deepcopy(task_info), site_index=site_index)

def build_share_payload(self, task_id: str) -> dict:
site_index, task_info = self._submitted_task_infos[task_id]
return self.gui.shares.build_share_payload(deepcopy(task_info))

def ensure_work_thread(self) -> WorkThread:
if self.b_thread and self.b_thread.isRunning():
return self.b_thread
Expand Down
12 changes: 12 additions & 0 deletions GUI/manager/preview/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,17 @@ def show_preview(self, *, ensure_handler=None, reload_tf=True, bridge=None):
self._bind_page_interactive(browser)
return browser

def publish_share_books(self, books):
first = books[0]
site_index = next((idx for idx, spider_name in SPIDERS.items() if spider_name == getattr(first, "source", "")), None)
if self.gui.chooseBox.currentIndex() != site_index:
self.gui.chooseBox.setCurrentIndex(site_index)
self.begin_preview_session()
self.gui.flow_stage = GUIFlowStage.SEARCHED
self._current_page = 1
self._target_page = None
self._active.publish(books)

def _legacy_run_js(self, js, session_id):
if session_id != self._session_id:
return False
Expand Down Expand Up @@ -250,6 +261,7 @@ def create_worker(self, gui_site_runtime: GuiSiteRuntime):
self._worker.search_done.connect(self._on_search_done)
ep_handler = self._fix if self.is_fix else self._manga
self._worker.episodes_done.connect(ep_handler.on_episodes_done)
self._worker.episodes_error.connect(ep_handler.on_episodes_error)
self._worker.pages_done.connect(ep_handler.on_pages_done)
self._worker.cover_done.connect(self.gui.task_mgr.on_cover_preload_success)
self._worker.cover_error.connect(self.gui.task_mgr.on_cover_preload_error)
Expand Down
21 changes: 14 additions & 7 deletions GUI/manager/preview/manga.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,12 +348,8 @@ def on_episodes_done(self, generation, session_id, book_key, episodes):
for episode in episodes
if episode.id_and_md5()[1] in downloaded_md5s
}
ep_data = [
{
"idx": ep.idx,
"name": ep.name,
"downloaded": f"ep{book_key}-{ep.idx}" in downloaded_episode_ids,
}
ep_data = [{"idx": ep.idx, "name": ep.name,
"downloaded": f"ep{book_key}-{ep.idx}" in downloaded_episode_ids}
for ep in episodes
]
self.mgr.send_command("manga.episodes.loaded", {"bookKey": str(book_key), "episodes": ep_data}, session_id=session_id)
Expand All @@ -364,7 +360,9 @@ def on_episodes_error(self, generation, session_id, book_key, error):
self._inflight_books.discard((session_id, book_key))
if generation != self.mgr._generation or session_id != self.mgr._session_id:
return
self.mgr.send_command("manga.episodes.error", {"bookKey": str(book_key), "code": "fetch_failed"}, session_id=session_id)
self.mgr.send_command("manga.episodes.error", {"bookKey": str(book_key), "code": "fetch_failed", "message": str(error or "")},
session_id=session_id,
)

# ------------------------------------------------------------------
# Pages fetch
Expand All @@ -377,19 +375,22 @@ def on_pages_done(self, generation, book_key, episodes):
if generation != self.mgr._generation:
if not self._inflight_pages:
self.mgr.send_command("preview.scan.hide", {})
self._stop_submit_loading()
return
book, selected_eps = pending
book.episodes = list(selected_eps)
self.gui.sel_mgr.submit_decision("EP", book)
if not self._inflight_pages:
self.mgr.send_command("preview.scan.hide", {})
self._stop_submit_loading()

def on_pages_error(self, generation, book_key, error):
self._inflight_pages.pop(book_key, None)
if generation == self.mgr._generation:
self.mgr.send_command("manga.episodes.error", {"bookKey": str(book_key), "code": "pages_fetch_failed"})
if not self._inflight_pages:
self.mgr.send_command("preview.scan.hide", {})
self._stop_submit_loading()

# ------------------------------------------------------------------
# Selection / ensure
Expand Down Expand Up @@ -474,4 +475,10 @@ def submit_page_selections(self):
self._submit_payload(self._current_submit_payload())

def _handle_submit_request(self):
self.gui.BrowserWindow.start_submit_loading()
self._submit_payload(self._current_submit_payload())
if not self._inflight_pages:
self._stop_submit_loading()

def _stop_submit_loading(self):
self.gui.BrowserWindow.stop_submit_loading()
Loading
Loading