From dc1615b909ca5ff21cfd5f9a1bf75769da715707 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 10 Jul 2020 17:42:07 -0600 Subject: [PATCH 1/4] Moved download function to Downloader class --- monkey_dl/gui/GUI.py | 113 +-------------------------- monkey_dl/util/downloader.py | 144 ++++++++++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 112 deletions(-) diff --git a/monkey_dl/gui/GUI.py b/monkey_dl/gui/GUI.py index 0ef82cd..e917d89 100644 --- a/monkey_dl/gui/GUI.py +++ b/monkey_dl/gui/GUI.py @@ -1,7 +1,6 @@ import sys import queue import json -import cloudscraper import traceback import monkey_dl import PySimpleGUI as sg @@ -10,15 +9,6 @@ from util.downloader import Downloader from util.Color import printer from util.name_collector import EpisodeNamesCollector -from scrapers.fouranime.fouranime_scraper import FourAnimeScraper -from scrapers.nineanime.nineanime_scraper import NineAnimeScraper -from scrapers.animeultima.animeultima_scraper import AnimeUltimaScraper -from scrapers.animeflix.animeflix_scraper import AnimeFlixScraper -from scrapers.animepahe.animepahe_scraper import AnimePaheScraper -from scrapers.gogoanime.gogoanime_scraper import GoGoAnimeScraper -from scrapers.animefreak.animefreak_scraper import AnimeFreakScraper -from scrapers.animetake.animetake_scraper import AnimeTakeScraper -from scrapers.twist.twist_scraper import TwistScraper sg.theme('Dark Amber') i = 0 @@ -29,107 +19,12 @@ def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, tok resolution="720", is_dub=False): global max_val - session = cloudscraper.create_scraper() - api_key = "" - try: - with open("settings.json", "r") as json_file: - data = json.load(json_file) - api_key = data["api_key"] - except Exception: - api_key = "" + #downloader = Downloader(directory, episodes, threads, gui, is_titles) + downloader = Downloader(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, resolution, is_dub) - if api_key != "" and api_key != "insert_2captcha_api_key": - session = cloudscraper.create_scraper( - recaptcha={ - 'provider': '2captcha', - 'api_key': api_key - } - ) + max_val = downloader.get_episodes() - scraper = None - episodes = [] - - anime_url = anime_url.lower() - - try: - if "9anime" in anime_url: - printer("INFO", "9Anime URL detected...", gui) - scraper = NineAnimeScraper(anime_url, start_epi, end_epi, session, gui, token) - - elif "4anime.to" in anime_url: - printer("INFO", "4Anime URL detected...", gui) - scraper = FourAnimeScraper(anime_url, start_epi, end_epi, session, gui) - - elif "animeultima.to" in anime_url: - printer("INFO", "AnimeUltima URL detected...", gui) - scraper = AnimeUltimaScraper(anime_url, start_epi, end_epi, session, gui, resolution, is_dub) - - elif "animeflix" in anime_url: - printer("INFO", "AnimeFlix URL detected...", gui) - scraper = AnimeFlixScraper(anime_url, start_epi, end_epi, session, gui, resolution, is_dub) - - elif "gogoanime" in anime_url: - printer("INFO", "GoGoAnime URL detected...", gui) - if "gogoanime.pro" in anime_url: - printer("ERROR", "goganime.pro links are not supported yet try gogoanime.io or gogoanime.video", gui) - return - - scraper = GoGoAnimeScraper(anime_url, start_epi, end_epi, session, gui, resolution) - - elif "animefreak" in anime_url: - printer("INFO", "AnimeFreak URL detected...", gui) - scraper = AnimeFreakScraper(anime_url, start_epi, end_epi, session, gui, is_dub) - - elif "twist" in anime_url: - printer("INFO", "Twist URL detected...", gui) - scraper = TwistScraper(anime_url, start_epi, end_epi, session, gui) - - elif "animetake" in anime_url: - printer("INFO", "AnimeTake URL detected...", gui) - scraper = AnimeTakeScraper(anime_url, start_epi, end_epi, session, gui, resolution) - - elif "animepahe.com" in anime_url: - printer("INFO", "AnimePahe URL detected...", gui) - - if api_key == "" or api_key == "insert_2captcha_api_key": - printer("ERROR", "You need 2captcha API key to download from AnimePahe!", gui) - printer("ERROR", "Set 2captcha API key in 'settings.json' file to download from AnimePahe!", gui) - return - - scraper = AnimePaheScraper(anime_url, start_epi, end_epi, session, gui, resolution, is_filler) - - else: - printer("ERROR", "Incorrect URL provided!", gui) - return - - printer("INFO", "Collecting download links...", gui) - episodes = scraper.get_direct_links() - - if episodes is None: - printer("INFO", "Retrying to collect download links...", gui) - sleep(5) - episodes = scraper.get_direct_links() - - if episodes: - if is_titles: - printer("INFO", "Setting episode titles...", gui) - episodes = EpisodeNamesCollector(names_url, start_epi, end_epi, is_filler, - episodes).collect_episode_names() - - else: - printer("ERROR", "Failed to retrieve download links!", gui) - return - - max_val = len(episodes) - # print("is titles", is_titles) - downloader = Downloader(directory, episodes, threads, gui, is_titles) - downloader.download() - - except Exception as ex: - trace = traceback.format_exc() - print(trace) - printer("ERROR", ex, gui) - printer("ERROR", "Something went wrong! Please close and restart Anime Downloader to retry!", gui) + downloader.download() class AnimeGUI: diff --git a/monkey_dl/util/downloader.py b/monkey_dl/util/downloader.py index eb446a5..4dc5738 100644 --- a/monkey_dl/util/downloader.py +++ b/monkey_dl/util/downloader.py @@ -8,6 +8,18 @@ from util import Color from util.hls_downloader import HLSDownloader +from util.Color import printer +from time import sleep +import cloudscraper +from scrapers.fouranime.fouranime_scraper import FourAnimeScraper +from scrapers.nineanime.nineanime_scraper import NineAnimeScraper +from scrapers.animeultima.animeultima_scraper import AnimeUltimaScraper +from scrapers.animeflix.animeflix_scraper import AnimeFlixScraper +from scrapers.animepahe.animepahe_scraper import AnimePaheScraper +from scrapers.gogoanime.gogoanime_scraper import GoGoAnimeScraper +from scrapers.animefreak.animefreak_scraper import AnimeFreakScraper +from scrapers.animetake.animetake_scraper import AnimeTakeScraper +from scrapers.twist.twist_scraper import TwistScraper def clean_file_name(file_name): for c in r'[]/\;,><&*:%=+@#^()|?^': @@ -53,13 +65,25 @@ def wait_completion(self): class Downloader: - def __init__(self, directory, episodes, threads=1, gui=None, is_titles=False): + def __init__(self, anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads=1, directory=".", gui=None, resolution="720", is_dub=False): + self.anime_url = anime_url + self.names_url = names_url + self.start_epi = start_epi + self.end_epi = end_epi + self.is_filler = is_filler + self.is_titles = is_titles + self.token = token + self.resolution = resolution + self.is_dub = is_dub + self.directory = directory self.threads = threads - self.episodes = episodes - self.is_titles = is_titles + #self.episodes = episodes + #self.is_titles = is_titles self.gui = gui + self.episodes = None + def __download_episode(self, episode): if system() == "Windows": episode.title = clean_file_name(episode.title) @@ -97,7 +121,121 @@ def __download_episode(self, episode): Color.printer("ERROR", "Custom HLS Downloader failed to download {epi}".format(epi=episode.episode), self.gui) + def get_episodes(self): + max_val = 100 + + session = cloudscraper.create_scraper() + api_key = "" + try: + with open("settings.json", "r") as json_file: + data = json.load(json_file) + api_key = data["api_key"] + except Exception: + api_key = "" + + if api_key != "" and api_key != "insert_2captcha_api_key": + session = cloudscraper.create_scraper( + recaptcha={ + 'provider': '2captcha', + 'api_key': api_key + } + ) + + scraper = None + self.episodes = [] + + anime_url = self.anime_url.lower() + start_epi = self.start_epi + end_epi = self.end_epi + gui = self.gui + token = self.gui + resolution = self.resolution + is_dub = self.is_dub + is_filler = self.is_filler + + try: + if "9anime" in anime_url: + printer("INFO", "9Anime URL detected...", gui) + scraper = NineAnimeScraper(anime_url, start_epi, end_epi, session, gui, token) + + elif "4anime.to" in anime_url: + printer("INFO", "4Anime URL detected...", gui) + scraper = FourAnimeScraper(anime_url, start_epi, end_epi, session, gui) + + elif "animeultima.to" in anime_url: + printer("INFO", "AnimeUltima URL detected...", gui) + scraper = AnimeUltimaScraper(anime_url, start_epi, end_epi, session, gui, resolution, is_dub) + + elif "animeflix" in anime_url: + printer("INFO", "AnimeFlix URL detected...", gui) + scraper = AnimeFlixScraper(anime_url, start_epi, end_epi, session, gui, resolution, is_dub) + + elif "gogoanime" in anime_url: + printer("INFO", "GoGoAnime URL detected...", gui) + if "gogoanime.pro" in anime_url: + printer("ERROR", "goganime.pro links are not supported yet try gogoanime.io or gogoanime.video", gui) + return + + scraper = GoGoAnimeScraper(anime_url, start_epi, end_epi, session, gui, resolution) + + elif "animefreak" in anime_url: + printer("INFO", "AnimeFreak URL detected...", gui) + scraper = AnimeFreakScraper(anime_url, start_epi, end_epi, session, gui, is_dub) + + elif "twist" in anime_url: + printer("INFO", "Twist URL detected...", gui) + scraper = TwistScraper(anime_url, start_epi, end_epi, session, gui) + + elif "animetake" in anime_url: + printer("INFO", "AnimeTake URL detected...", gui) + scraper = AnimeTakeScraper(anime_url, start_epi, end_epi, session, gui, resolution) + + elif "animepahe.com" in anime_url: + printer("INFO", "AnimePahe URL detected...", gui) + + if api_key == "" or api_key == "insert_2captcha_api_key": + printer("ERROR", "You need 2captcha API key to download from AnimePahe!", gui) + printer("ERROR", "Set 2captcha API key in 'settings.json' file to download from AnimePahe!", gui) + return + + scraper = AnimePaheScraper(anime_url, start_epi, end_epi, session, gui, resolution, is_filler) + + else: + printer("ERROR", "Incorrect URL provided!", gui) + return + + printer("INFO", "Collecting download links...", gui) + self.episodes = scraper.get_direct_links() + + if self.episodes is None: + printer("INFO", "Retrying to collect download links...", gui) + sleep(5) + self.episodes = scraper.get_direct_links() + + if self.episodes: + if self.is_titles: + printer("INFO", "Setting episode titles...", gui) + self.episodes = EpisodeNamesCollector(names_url, start_epi, end_epi, is_filler, + self.episodes).collect_episode_names() + + else: + printer("ERROR", "Failed to retrieve download links!", gui) + return + + max_val = len(self.episodes) + + except Exception as ex: + trace = traceback.format_exc() + print(trace) + printer("ERROR", ex, gui) + printer("ERROR", "Something went wrong! Please close and restart Anime Downloader to retry!", gui) + + return max_val + + def download(self): + if self.episodes is None: + self.get_episodes() try: _create_unverified_https_context = ssl._create_unverified_context From 9cdec98ac348fb5cb34cf2569c791c9b6efc4767 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 10 Jul 2020 18:10:54 -0600 Subject: [PATCH 2/4] Added CLI and --nogui option --- .gitignore | 5 +++- monkey_dl/cli/__init__.py | 0 monkey_dl/monkey-dl.py | 54 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 monkey_dl/cli/__init__.py diff --git a/.gitignore b/.gitignore index f092640..1026f82 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ monkey_dl/build/ monkey_dl/dist/ monkey_dl/settings.json monkey_dl/results.csv -*.spec \ No newline at end of file +*.spec + +*.mp4 +*.mkv diff --git a/monkey_dl/cli/__init__.py b/monkey_dl/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monkey_dl/monkey-dl.py b/monkey_dl/monkey-dl.py index 76b39ec..9132132 100644 --- a/monkey_dl/monkey-dl.py +++ b/monkey_dl/monkey-dl.py @@ -1,5 +1,57 @@ from queue import Queue from gui.GUI import AnimeGUI +import argparse +from util.downloader import Downloader + +def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, + resolution="720", is_dub=False): + + downloader = Downloader(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, resolution, is_dub) + downloader.download() + +def main(): + + parser = argparse.ArgumentParser(description="Anime Downloader Command Line Tool") + argparse.ArgumentParser(description="Help option parcer for Anime Downloader Command Line Tool", add_help=False, + formatter_class=argparse.HelpFormatter) + + parser.add_argument("-u", "--url", required=False, help="9Anime.to URL for the anime to be downloaded", dest="url") + parser.add_argument("-n", "--names", required=False, + help="https://www.animefillerlist.com/ URL to retrieve episode titles", dest="title_url") + parser.add_argument("-d", "--directory", required=False, + help="Download destination. Will use the current directory if not provided", default="", + dest="dir") + parser.add_argument("-s", "--start", required=False, help="Starting episode", default=1, type=int, dest="start") + parser.add_argument("-e", "--end", required=False, help="End episode", default=9999, type=int, dest="end") + parser.add_argument("-c", "--code", required=False, + help="Recaptcha answer token code. Insert this if you don't have 2captcha captcha bypass api_key", + default=None, dest="token") + parser.add_argument("-t", "--threads", required=False, + help="Number of parrallel downloads. Will download sequencially if not provided", default=1, + type=int, dest="threads") + parser.add_argument("-f", "--filler", required=False, help="Whether fillers needed", default=True, type=bool, + dest="isFiller") + parser.add_argument("--nogui", action='store_true', required=False, help="Disable GUI for cli", + dest="nogui") + + args = parser.parse_args() + + titles = False + if args.title_url: + titles = True + + directory = args.dir + if directory != "": + directory = directory.replace("\\", "/") + if not directory.endswith("/"): + directory += "/" + + if args.nogui: + download(args.url, args.title_url, args.start, args.end, args.isFiller, titles, args.token, args.threads, directory, None) + else: + AnimeGUI(Queue()).run() + if __name__ == "__main__": - AnimeGUI(Queue()).run() + #AnimeGUI(Queue()).run() + main() From 4bdc145159dd02d1a5f08f5cceece98d8beb36cc Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 10 Jul 2020 18:32:21 -0600 Subject: [PATCH 3/4] Moved CLI to seperate file --- monkey_dl/cli/CLI.py | 53 ++++++++++++++++++++++++++++++++++++++++++ monkey_dl/gui/GUI.py | 3 --- monkey_dl/monkey-dl.py | 49 +++----------------------------------- 3 files changed, 56 insertions(+), 49 deletions(-) create mode 100644 monkey_dl/cli/CLI.py diff --git a/monkey_dl/cli/CLI.py b/monkey_dl/cli/CLI.py new file mode 100644 index 0000000..7f6028b --- /dev/null +++ b/monkey_dl/cli/CLI.py @@ -0,0 +1,53 @@ +import argparse +from util.downloader import Downloader + +def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, + resolution="720", is_dub=False): + + downloader = Downloader(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, resolution, is_dub) + downloader.download() + +class AnimeCLI: + def __init__(self): + pass + + def run(self): + parser = argparse.ArgumentParser(description="Anime Downloader Command Line Tool") + argparse.ArgumentParser(description="Help option parcer for Anime Downloader Command Line Tool", add_help=False, + formatter_class=argparse.HelpFormatter) + + parser.add_argument("-u", "--url", required=False, help="9Anime.to URL for the anime to be downloaded", dest="url") + parser.add_argument("-n", "--names", required=False, + help="https://www.animefillerlist.com/ URL to retrieve episode titles", dest="title_url") + parser.add_argument("-d", "--directory", required=False, + help="Download destination. Will use the current directory if not provided", default="", + dest="dir") + parser.add_argument("-s", "--start", required=False, help="Starting episode", default=1, type=int, dest="start") + parser.add_argument("-e", "--end", required=False, help="End episode", default=9999, type=int, dest="end") + parser.add_argument("-c", "--code", required=False, + help="Recaptcha answer token code. Insert this if you don't have 2captcha captcha bypass api_key", + default=None, dest="token") + parser.add_argument("-t", "--threads", required=False, + help="Number of parrallel downloads. Will download sequencially if not provided", default=1, + type=int, dest="threads") + parser.add_argument("-f", "--filler", required=False, help="Whether fillers needed", default=True, type=bool, + dest="isFiller") + parser.add_argument("--nogui", action='store_true', required=False, help="Disable GUI for cli", + dest="nogui") + + args = parser.parse_args() + + titles = False + if args.title_url: + titles = True + + directory = args.dir + if directory != "": + directory = directory.replace("\\", "/") + if not directory.endswith("/"): + directory += "/" + + if args.nogui: + download(args.url, args.title_url, args.start, args.end, args.isFiller, titles, args.token, args.threads, directory, None) + + return args.nogui diff --git a/monkey_dl/gui/GUI.py b/monkey_dl/gui/GUI.py index e917d89..e77260d 100644 --- a/monkey_dl/gui/GUI.py +++ b/monkey_dl/gui/GUI.py @@ -19,11 +19,8 @@ def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, tok resolution="720", is_dub=False): global max_val - #downloader = Downloader(directory, episodes, threads, gui, is_titles) downloader = Downloader(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, resolution, is_dub) - max_val = downloader.get_episodes() - downloader.download() diff --git a/monkey_dl/monkey-dl.py b/monkey_dl/monkey-dl.py index 9132132..8ea0198 100644 --- a/monkey_dl/monkey-dl.py +++ b/monkey_dl/monkey-dl.py @@ -1,55 +1,12 @@ from queue import Queue from gui.GUI import AnimeGUI - -import argparse -from util.downloader import Downloader - -def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, - resolution="720", is_dub=False): - - downloader = Downloader(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, resolution, is_dub) - downloader.download() +from cli.CLI import AnimeCLI def main(): - parser = argparse.ArgumentParser(description="Anime Downloader Command Line Tool") - argparse.ArgumentParser(description="Help option parcer for Anime Downloader Command Line Tool", add_help=False, - formatter_class=argparse.HelpFormatter) - - parser.add_argument("-u", "--url", required=False, help="9Anime.to URL for the anime to be downloaded", dest="url") - parser.add_argument("-n", "--names", required=False, - help="https://www.animefillerlist.com/ URL to retrieve episode titles", dest="title_url") - parser.add_argument("-d", "--directory", required=False, - help="Download destination. Will use the current directory if not provided", default="", - dest="dir") - parser.add_argument("-s", "--start", required=False, help="Starting episode", default=1, type=int, dest="start") - parser.add_argument("-e", "--end", required=False, help="End episode", default=9999, type=int, dest="end") - parser.add_argument("-c", "--code", required=False, - help="Recaptcha answer token code. Insert this if you don't have 2captcha captcha bypass api_key", - default=None, dest="token") - parser.add_argument("-t", "--threads", required=False, - help="Number of parrallel downloads. Will download sequencially if not provided", default=1, - type=int, dest="threads") - parser.add_argument("-f", "--filler", required=False, help="Whether fillers needed", default=True, type=bool, - dest="isFiller") - parser.add_argument("--nogui", action='store_true', required=False, help="Disable GUI for cli", - dest="nogui") - - args = parser.parse_args() - - titles = False - if args.title_url: - titles = True - - directory = args.dir - if directory != "": - directory = directory.replace("\\", "/") - if not directory.endswith("/"): - directory += "/" + noGui = AnimeCLI().run() - if args.nogui: - download(args.url, args.title_url, args.start, args.end, args.isFiller, titles, args.token, args.threads, directory, None) - else: + if not noGui: AnimeGUI(Queue()).run() if __name__ == "__main__": From 7411e910a579c14d910adbfcfc18e54b1202dbf9 Mon Sep 17 00:00:00 2001 From: Nicholas Wallace Date: Fri, 10 Jul 2020 18:39:00 -0600 Subject: [PATCH 4/4] Cleaned up function names --- monkey_dl/cli/CLI.py | 6 +++--- monkey_dl/gui/GUI.py | 6 +++--- monkey_dl/monkey-dl.py | 1 - monkey_dl/util/downloader.py | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/monkey_dl/cli/CLI.py b/monkey_dl/cli/CLI.py index 7f6028b..c540775 100644 --- a/monkey_dl/cli/CLI.py +++ b/monkey_dl/cli/CLI.py @@ -1,8 +1,8 @@ import argparse from util.downloader import Downloader -def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, - resolution="720", is_dub=False): +def cli_download_wrapper(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, + resolution="720", is_dub=False): downloader = Downloader(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, resolution, is_dub) downloader.download() @@ -48,6 +48,6 @@ def run(self): directory += "/" if args.nogui: - download(args.url, args.title_url, args.start, args.end, args.isFiller, titles, args.token, args.threads, directory, None) + cli_download_wrapper(args.url, args.title_url, args.start, args.end, args.isFiller, titles, args.token, args.threads, directory, None) return args.nogui diff --git a/monkey_dl/gui/GUI.py b/monkey_dl/gui/GUI.py index e77260d..87c3150 100644 --- a/monkey_dl/gui/GUI.py +++ b/monkey_dl/gui/GUI.py @@ -15,8 +15,8 @@ max_val = 100 -def download(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, - resolution="720", is_dub=False): +def gui_download_wrapper(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, + resolution="720", is_dub=False): global max_val downloader = Downloader(anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, gui, resolution, is_dub) @@ -134,7 +134,7 @@ def run(self): self.window["txt_msg"].update("") self.window.refresh() - thread = Thread(target=download, args=( + thread = Thread(target=gui_download_wrapper, args=( anime_url, names_url, start_epi, end_epi, is_filler, is_titles, token, threads, directory, self, resolution, is_dub), daemon=True) thread.start() diff --git a/monkey_dl/monkey-dl.py b/monkey_dl/monkey-dl.py index 8ea0198..1d6909b 100644 --- a/monkey_dl/monkey-dl.py +++ b/monkey_dl/monkey-dl.py @@ -10,5 +10,4 @@ def main(): AnimeGUI(Queue()).run() if __name__ == "__main__": - #AnimeGUI(Queue()).run() main() diff --git a/monkey_dl/util/downloader.py b/monkey_dl/util/downloader.py index 4dc5738..ab2b4cd 100644 --- a/monkey_dl/util/downloader.py +++ b/monkey_dl/util/downloader.py @@ -27,7 +27,6 @@ def clean_file_name(file_name): return file_name - class Worker(Thread): def __init__(self, tasks, gui=None): Thread.__init__(self)