Merge pull request #814 from rix1337/dev

rix1337 · web-flow · commit bdcc98e5a0eb · 2024-12-30T23:40:32.000+01:00
Upgrade to version 21.0.1 and update captcha provider references.
diff --git a/feedcrawler/providers/config.py b/feedcrawler/providers/config.py
@@ -27,7 +27,6 @@ class CrawlerConfig(object):
             ("port", "int", "9090"),
             ("prefix", "str", ""),
             ("interval", "int", "60"),
-            ("sponsors_helper", "str", ""),
             ("flaresolverr", "str", ""),
             ("english", "bool", "False"),
             ("surround", "bool", ""),
diff --git a/feedcrawler/providers/http_requests/cache_handler.py b/feedcrawler/providers/http_requests/cache_handler.py
@@ -11,7 +11,6 @@
 from feedcrawler.providers.common_functions import site_blocked, site_blocked_with_advanced_methods
 from feedcrawler.providers.http_requests.cloudflare_handlers import flaresolverr_task
 from feedcrawler.providers.http_requests.cloudflare_handlers import get_solver_url
-from feedcrawler.providers.http_requests.cloudflare_handlers import sponsors_helper_task
 from feedcrawler.providers.http_requests.request_handler import request
 from feedcrawler.providers.sqlite_database import FeedDb
 
@@ -53,7 +52,6 @@ def cached_request(url, method='get', params=None, headers=None, redirect_url=Fa
     if dont_cache:
         shared_state.logger.debug("Aufruf ohne HTTP-Cache: " + url)
 
-    sponsors_helper_url = get_solver_url("sponsors_helper")
     flaresolverr_url = get_solver_url("flaresolverr")
 
     if not headers:
@@ -76,29 +74,16 @@ def cached_request(url, method='get', params=None, headers=None, redirect_url=Fa
     proxies = {}
     force_ipv4 = False
 
-    flaresolverr_run = False
-    allow_sponsors_helper_run = False
-    if sponsors_helper_url and not flaresolverr_url:
-        allow_sponsors_helper_run = True
-
     while True:
         try:
             if site_blocked(url):
                 if site_blocked_with_advanced_methods(url):
                     print(f"Der Aufruf von {url} wurde blockiert!")
                     return {'status_code': status_code, 'text': text, 'headers': response_headers, 'url': url}
-                if allow_sponsors_helper_run:  # will only be used when flaresolverr is not available or not working
-                    cookiejar, user_agent, proxy = sponsors_helper_task(sponsors_helper_url, url)
-                    proxies = {"http": proxy, "https": proxy}
-                    headers['User-Agent'] = user_agent
-                    force_ipv4 = False
-                    flaresolverr_run = False
-                elif flaresolverr_url:
+                if flaresolverr_url:
                     cookiejar, user_agent = flaresolverr_task(flaresolverr_url, url)
                     headers['User-Agent'] = user_agent
                     force_ipv4 = True
-                    flaresolverr_run = True
-                    allow_sponsors_helper_run = True
 
             if method == 'post':
                 response = request(url, method="POST", data=params, timeout=60, headers=headers,
@@ -115,16 +100,13 @@ def cached_request(url, method='get', params=None, headers=None, redirect_url=Fa
                     normal_blocked = db_status.retrieve(site + "_normal")
                     if not normal_blocked:
                         db_status.store(site + "_normal", "Blocked")
-                        if sponsors_helper_url or flaresolverr_url:
+                        if flaresolverr_url:
                             print("Versuche es mit Cloudfare-Umgehung erneut...")
                             continue  # try again with any solver
                     else:
                         advanced_blocked = db_status.retrieve(site + "_advanced")
                         if not advanced_blocked:
                             db_status.store(site + "_advanced", "Blocked")
-                if flaresolverr_run and allow_sponsors_helper_run:
-                    print("Lösung mit FlareSolverr gescheitert. Versuche es mit Sponsors Helper...")
-                    continue  # try again with sponsors helper
                 return {'status_code': status_code, 'text': text, 'headers': response_headers, 'url': url}
 
             if redirect_url:
diff --git a/feedcrawler/providers/http_requests/cloudflare_handlers.py b/feedcrawler/providers/http_requests/cloudflare_handlers.py
@@ -142,67 +142,6 @@ def test_challenge_path(url):
         return "/"
 
 
-def sponsors_helper_task(solver_url, url):
-    base_domain = url.split("/")[2]
-    last_solution = unpickle_db("sponsors_helper", base_domain)
-    if last_solution:
-        try:
-            if last_solution["valid_until"] > int(time.time()):
-                cookiejar = cookie_dict_to_cookiejar(last_solution["cookies"])
-                user_agent = last_solution["user_agent"]
-                proxy = get_local_proxy_url(solver_url, last_solution["proxy"])
-                if cookiejar:
-                    shared_state.logger.debug("Bestehende Cloudflare-Cookies werden für " + url + " verwendet.")
-                    return cookiejar, user_agent, proxy
-            else:
-                clean_db("sponsors_helper", base_domain)
-        except:
-            pass
-
-    if test_solver_url("sponsors_helper", solver_url + "/status"):
-        shared_state.logger.debug(f"Versuche Cloudflare auf der Seite {url} mit Sponsors Helper zu umgehen...")
-
-        solver_endpoint = "/cloudflare_cookie/"
-        solver_payload = {
-            'url': "https://" + base_domain + test_challenge_path(url)
-        }
-
-        response = request(
-            solver_url + solver_endpoint,
-            method="POST",
-            json=solver_payload,
-            timeout=180)
-
-        if response.status_code == 200:
-            try:
-                response = loads(response.text)
-                cookies = response["cookies"]
-                if cookies:
-                    cookiejar = cookie_dict_to_cookiejar(cookies)
-                    user_agent = response["user_agent"]
-                    proxy = get_local_proxy_url(solver_url, response["proxy"])
-                    valid_until = int(time.time()) + 1800 - 60
-
-                    pickle_db("sponsors_helper",
-                              base_domain,
-                              {
-                                  "cookies": cookies,
-                                  "user_agent": user_agent,
-                                  "proxy": proxy,
-                                  "valid_until": valid_until
-                              })
-
-                    shared_state.logger.debug(
-                        "Die Erzeugung von Cloudflare-Cookies für " + url + " war mit Sponsors Helper erfolgreich.")
-                    return cookiejar, user_agent, proxy
-            except:
-                pass
-        else:
-            shared_state.logger.debug(
-                "Die Erzeugung von Cloudflare-Cookies für " + url + " ist mit Sponsors Helper fehlgeschlagen.")
-    return False, False
-
-
 def flaresolverr_task(solver_url, url):
     base_domain = url.split("/")[2]
     last_solution = unpickle_db("flaresolverr", base_domain)
diff --git a/feedcrawler/providers/version.py b/feedcrawler/providers/version.py
@@ -8,7 +8,7 @@
 
 
 def get_version():
-    return "21.0.0"
+    return "21.0.1"
 
 
 def create_version_file():
diff --git a/feedcrawler/run.py b/feedcrawler/run.py
@@ -104,6 +104,7 @@ def main():
             else:
                 myjd_config(port, local_address, shared_state)
 
+        shared_state.update("connected", False)
         process_jdownloader = multiprocessing.Process(target=jdownloader_connection,
                                                       args=(shared_state_dict, shared_state_lock))
         process_jdownloader.start()
@@ -129,10 +130,17 @@ def main():
 
         if arguments.delay:
             delay = int(arguments.delay)
-            print(f"Verzögere den ersten Suchlauf um {delay} Sekunden")
-            time.sleep(delay)
+        else:
+            delay = 60
 
         if not os.environ.get('GITHUB_ACTION_PR'):
+            try:
+                while not shared_state.values["connected"]:
+                    print(f"Verbindung zu JDownloader noch nicht hergestellt - verzögere Suchlauf um {delay} Sekunden")
+                    time.sleep(60)
+            except KeyboardInterrupt:
+                sys.exit(1)
+
             process_feed_crawler = multiprocessing.Process(target=feed_crawler,
                                                            args=(shared_state_dict, shared_state_lock,))
             process_feed_crawler.start()
@@ -178,16 +186,21 @@ def jdownloader_connection(shared_state_dict, shared_state_lock):
 
     shared_state.set_device_from_config()
     connection_established = shared_state.get_device() and shared_state.get_device().name
-    if not connection_established:
-        i = 0
-        while i < 10:
-            i += 1
-            print(f'Verbindungsversuch {i} mit My JDownloader gescheitert. Gerätename: "{device}"')
-            time.sleep(60)
-            set_device_from_config()
-            connection_established = shared_state.get_device() and shared_state.get_device().name
-            if connection_established:
-                break
+
+    try:
+        if not connection_established:
+            i = 0
+            while i < 10:
+                i += 1
+                print(f'Verbindungsversuch {i} mit My JDownloader gescheitert.')
+                time.sleep(60)
+                set_device_from_config()
+                connection_established = shared_state.get_device() and shared_state.get_device().name
+                if connection_established:
+                    shared_state.update("connected", True)
+                    break
+    except KeyboardInterrupt:
+        sys.exit(1)
 
     if connection_established:
         print(f'Erfolgreich mit My JDownloader verbunden. Gerätename: "{shared_state.get_device().name}"')
diff --git a/feedcrawler/web_interface/serve/api.py b/feedcrawler/web_interface/serve/api.py
@@ -261,7 +261,7 @@ def redirect_sponsors():
 
     @app.get(prefix + "/redirect_captcha/")
     def redirect_captcha():
-        redirect("https://getcaptchasolution.com/zuoo67f5cq")
+        redirect("https://deathbycaptcha.com/register?refid=6184288242b")
 
     @app.get(prefix + "/redirect_hoster/")
     def redirect_captcha():
@@ -301,7 +301,6 @@ def get_settings():
                         "port": port,
                         "prefix": general_conf.get("prefix"),
                         "interval": to_int(general_conf.get("interval")),
-                        "sponsors_helper": general_conf.get("sponsors_helper"),
                         "flaresolverr": general_conf.get("flaresolverr"),
                         "english": general_conf.get("english"),
                         "surround": general_conf.get("surround"),
@@ -441,7 +440,6 @@ def post_settings():
             if to_int(interval) < 5:
                 interval = '5'
             section.save("interval", interval)
-            section.save("sponsors_helper", to_str(data['general']['sponsors_helper']))
             section.save("flaresolverr", to_str(data['general']['flaresolverr']))
             section.save("english", to_str(data['general']['english']))
             section.save("surround", to_str(data['general']['surround']))
diff --git a/feedcrawler/web_interface/vuejs_frontend/package-lock.json b/feedcrawler/web_interface/vuejs_frontend/package-lock.json
diff --git a/feedcrawler/web_interface/vuejs_frontend/package.json b/feedcrawler/web_interface/vuejs_frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "feedcrawler-web",
-  "version": "21.0.0",
+  "version": "21.0.1",
   "type": "module",
   "scripts": {
     "dev": "vite",
diff --git a/feedcrawler/web_interface/vuejs_frontend/src/components/Help.vue b/feedcrawler/web_interface/vuejs_frontend/src/components/Help.vue
@@ -358,7 +358,7 @@ const waitTimeActive = computed(() => {
                  data-bs-parent="#accordionHelp">
               <div class="accordion-body">
                 <button class="btn btn-outline-success" @click="openCaptchaLink">
-                  Anti-Captcha.com (schnelles Lösen von reCAPTCHA, cutcaptcha, etc. - auch im Browser)
+                  DeathByCaptcha.com (schnelles Lösen von reCAPTCHA, cutcaptcha, etc. - auch im Browser)
                 </button>
               </div>
             </div>
diff --git a/feedcrawler/web_interface/vuejs_frontend/src/components/Settings.vue b/feedcrawler/web_interface/vuejs_frontend/src/components/Settings.vue
@@ -349,27 +349,6 @@ function updatePlexUrl() {
               <div id="collapseSolvers" aria-labelledby="headingSolvers" class="accordion-collapse collapse"
                    data-bs-parent="#accordionSettings">
                 <div class="accordion-body">
-                  <FormKit v-model="store.settings.general.sponsors_helper"
-                           help="Hier die URL des durch FeedCrawler erreichbaren Sponsors Helpers (Port 9700) angeben. Der Sponsors Helper wird für jede Seite genutzt, auf der eine Blockade durch Cloudflare erkannt wurde."
-                           help-class="text-muted"
-                           input-class="form-control bg-light mb-2"
-                           label="Sponsors-Helper-URL"
-                           messages-class="text-danger"
-                           outer-class="mb-4"
-                           placeholder="Bspw. http://192.168.0.1:9700"
-                           type="url"
-                           validation="url"
-                           validation-visibility="live"/>
-                  <div class="mb-4">
-                    <mark>
-                      Die zuverlässigste Möglichkeit, Cloudflare-Blockaden zu umgehen, ist dafür zu bezahlen.<br>
-                      Über den Sponsors Helper ist es möglich, vor dem Suchlauf erkannte Cloudflare-Blockaden zu
-                      umgehen.
-                      Dabei wird pro blockierter Seite ein kurzfristig gültiger Cloudflare-Cookie erzeugt, der dann für
-                      den Suchlauf verwendet wird. Die Kosten pro Seite und Suchlauf liegen im Sub-Cent-Bereich.<br>
-                      Um Kosten zu sparen, kann parallel ein FlareSolverr betrieben werden.
-                    </mark>
-                  </div>
                   <FormKit v-model="store.settings.general.flaresolverr"
                            help="Hier die URL eines durch FeedCrawler erreichbaren FlareSolverrs angeben. FlareSolverr wird für jede Seite genutzt, auf der eine Blockade durch Cloudflare erkannt wurde."
                            help-class="text-muted"

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "feedcrawler-web",`
`3`		`- "version": "21.0.0",`
	`3`	`+ "version": "21.0.1",`
`4`	`4`	`"type": "module",`
`5`	`5`	`"scripts": {`
`6`	`6`	`"dev": "vite",`