Remove Sponsors Helper support for captcha solving

This commit eliminates all references to the Sponsors Helper functionality, including related functions, configurations, and UI elements. The focus now shifts to using FlareSolverr as the sole solution for Cloudflare bypass handling. The cleanup streamlines code maintenance and reduces unnecessary complexity.

Author: rix

feedcrawler/providers/config.py

@@ -27,7 +27,6 @@ class CrawlerConfig(object):
             ("port", "int", "9090"),
             ("prefix", "str", ""),
             ("interval", "int", "60"),
-            ("sponsors_helper", "str", ""),
             ("flaresolverr", "str", ""),
             ("english", "bool", "False"),
             ("surround", "bool", ""),

feedcrawler/providers/http_requests/cache_handler.py

@@ -11,7 +11,6 @@
 from feedcrawler.providers.common_functions import site_blocked, site_blocked_with_advanced_methods
 from feedcrawler.providers.http_requests.cloudflare_handlers import flaresolverr_task
 from feedcrawler.providers.http_requests.cloudflare_handlers import get_solver_url
-from feedcrawler.providers.http_requests.cloudflare_handlers import sponsors_helper_task
 from feedcrawler.providers.http_requests.request_handler import request
 from feedcrawler.providers.sqlite_database import FeedDb
 
@@ -53,7 +52,6 @@ def cached_request(url, method='get', params=None, headers=None, redirect_url=Fa
     if dont_cache:
         shared_state.logger.debug("Aufruf ohne HTTP-Cache: " + url)
 
-    sponsors_helper_url = get_solver_url("sponsors_helper")
     flaresolverr_url = get_solver_url("flaresolverr")
 
     if not headers:
@@ -76,29 +74,16 @@ def cached_request(url, method='get', params=None, headers=None, redirect_url=Fa
     proxies = {}
     force_ipv4 = False
 
-    flaresolverr_run = False
-    allow_sponsors_helper_run = False
-    if sponsors_helper_url and not flaresolverr_url:
-        allow_sponsors_helper_run = True
-
     while True:
         try:
             if site_blocked(url):
                 if site_blocked_with_advanced_methods(url):
                     print(f"Der Aufruf von {url} wurde blockiert!")
                     return {'status_code': status_code, 'text': text, 'headers': response_headers, 'url': url}
-                if allow_sponsors_helper_run:  # will only be used when flaresolverr is not available or not working
-                    cookiejar, user_agent, proxy = sponsors_helper_task(sponsors_helper_url, url)
-                    proxies = {"http": proxy, "https": proxy}
-                    headers['User-Agent'] = user_agent
-                    force_ipv4 = False
-                    flaresolverr_run = False
-                elif flaresolverr_url:
+                if flaresolverr_url:
                     cookiejar, user_agent = flaresolverr_task(flaresolverr_url, url)
                     headers['User-Agent'] = user_agent
                     force_ipv4 = True
-                    flaresolverr_run = True
-                    allow_sponsors_helper_run = True
 
             if method == 'post':
                 response = request(url, method="POST", data=params, timeout=60, headers=headers,
@@ -115,16 +100,13 @@ def cached_request(url, method='get', params=None, headers=None, redirect_url=Fa
                     normal_blocked = db_status.retrieve(site + "_normal")
                     if not normal_blocked:
                         db_status.store(site + "_normal", "Blocked")
-                        if sponsors_helper_url or flaresolverr_url:
+                        if flaresolverr_url:
                             print("Versuche es mit Cloudfare-Umgehung erneut...")
                             continue  # try again with any solver
                     else:
                         advanced_blocked = db_status.retrieve(site + "_advanced")
                         if not advanced_blocked:
                             db_status.store(site + "_advanced", "Blocked")
-                if flaresolverr_run and allow_sponsors_helper_run:
-                    print("Lösung mit FlareSolverr gescheitert. Versuche es mit Sponsors Helper...")
-                    continue  # try again with sponsors helper
                 return {'status_code': status_code, 'text': text, 'headers': response_headers, 'url': url}
 
             if redirect_url:

feedcrawler/providers/http_requests/cloudflare_handlers.py

@@ -142,67 +142,6 @@ def test_challenge_path(url):
         return "/"
 
 
-def sponsors_helper_task(solver_url, url):
-    base_domain = url.split("/")[2]
-    last_solution = unpickle_db("sponsors_helper", base_domain)
-    if last_solution:
-        try:
-            if last_solution["valid_until"] > int(time.time()):
-                cookiejar = cookie_dict_to_cookiejar(last_solution["cookies"])
-                user_agent = last_solution["user_agent"]
-                proxy = get_local_proxy_url(solver_url, last_solution["proxy"])
-                if cookiejar:
-                    shared_state.logger.debug("Bestehende Cloudflare-Cookies werden für " + url + " verwendet.")
-                    return cookiejar, user_agent, proxy
-            else:
-                clean_db("sponsors_helper", base_domain)
-        except:
-            pass
-
-    if test_solver_url("sponsors_helper", solver_url + "/status"):
-        shared_state.logger.debug(f"Versuche Cloudflare auf der Seite {url} mit Sponsors Helper zu umgehen...")
-
-        solver_endpoint = "/cloudflare_cookie/"
-        solver_payload = {
-            'url': "https://" + base_domain + test_challenge_path(url)
-        }
-
-        response = request(
-            solver_url + solver_endpoint,
-            method="POST",
-            json=solver_payload,
-            timeout=180)
-
-        if response.status_code == 200:
-            try:
-                response = loads(response.text)
-                cookies = response["cookies"]
-                if cookies:
-                    cookiejar = cookie_dict_to_cookiejar(cookies)
-                    user_agent = response["user_agent"]
-                    proxy = get_local_proxy_url(solver_url, response["proxy"])
-                    valid_until = int(time.time()) + 1800 - 60
-
-                    pickle_db("sponsors_helper",
-                              base_domain,
-                              {
-                                  "cookies": cookies,
-                                  "user_agent": user_agent,
-                                  "proxy": proxy,
-                                  "valid_until": valid_until
-                              })
-
-                    shared_state.logger.debug(
-                        "Die Erzeugung von Cloudflare-Cookies für " + url + " war mit Sponsors Helper erfolgreich.")
-                    return cookiejar, user_agent, proxy
-            except:
-                pass
-        else:
-            shared_state.logger.debug(
-                "Die Erzeugung von Cloudflare-Cookies für " + url + " ist mit Sponsors Helper fehlgeschlagen.")
-    return False, False
-
-
 def flaresolverr_task(solver_url, url):
     base_domain = url.split("/")[2]
     last_solution = unpickle_db("flaresolverr", base_domain)

feedcrawler/run.py

@@ -133,14 +133,14 @@ def main():
         else:
             delay = 60
 
-        try:
-            while not shared_state.values["connected"]:
-                print(f"Verbindung zu JDownloader noch nicht hergestellt - verzögere Suchlauf um {delay} Sekunden")
-                time.sleep(60)
-        except KeyboardInterrupt:
-            sys.exit(1)
-
         if not os.environ.get('GITHUB_ACTION_PR'):
+            try:
+                while not shared_state.values["connected"]:
+                    print(f"Verbindung zu JDownloader noch nicht hergestellt - verzögere Suchlauf um {delay} Sekunden")
+                    time.sleep(60)
+            except KeyboardInterrupt:
+                sys.exit(1)
+
             process_feed_crawler = multiprocessing.Process(target=feed_crawler,
                                                            args=(shared_state_dict, shared_state_lock,))
             process_feed_crawler.start()

feedcrawler/web_interface/serve/api.py

@@ -301,7 +301,6 @@ def get_settings():
                         "port": port,
                         "prefix": general_conf.get("prefix"),
                         "interval": to_int(general_conf.get("interval")),
-                        "sponsors_helper": general_conf.get("sponsors_helper"),
                         "flaresolverr": general_conf.get("flaresolverr"),
                         "english": general_conf.get("english"),
                         "surround": general_conf.get("surround"),
@@ -441,7 +440,6 @@ def post_settings():
             if to_int(interval) < 5:
                 interval = '5'
             section.save("interval", interval)
-            section.save("sponsors_helper", to_str(data['general']['sponsors_helper']))
             section.save("flaresolverr", to_str(data['general']['flaresolverr']))
             section.save("english", to_str(data['general']['english']))
             section.save("surround", to_str(data['general']['surround']))

feedcrawler/web_interface/vuejs_frontend/src/components/Settings.vue

@@ -349,27 +349,6 @@ function updatePlexUrl() {
               <div id="collapseSolvers" aria-labelledby="headingSolvers" class="accordion-collapse collapse"
                    data-bs-parent="#accordionSettings">
                 <div class="accordion-body">
-                  <FormKit v-model="store.settings.general.sponsors_helper"
-                           help="Hier die URL des durch FeedCrawler erreichbaren Sponsors Helpers (Port 9700) angeben. Der Sponsors Helper wird für jede Seite genutzt, auf der eine Blockade durch Cloudflare erkannt wurde."
-                           help-class="text-muted"
-                           input-class="form-control bg-light mb-2"
-                           label="Sponsors-Helper-URL"
-                           messages-class="text-danger"
-                           outer-class="mb-4"
-                           placeholder="Bspw. http://192.168.0.1:9700"
-                           type="url"
-                           validation="url"
-                           validation-visibility="live"/>
-                  <div class="mb-4">
-                    <mark>
-                      Die zuverlässigste Möglichkeit, Cloudflare-Blockaden zu umgehen, ist dafür zu bezahlen.<br>
-                      Über den Sponsors Helper ist es möglich, vor dem Suchlauf erkannte Cloudflare-Blockaden zu
-                      umgehen.
-                      Dabei wird pro blockierter Seite ein kurzfristig gültiger Cloudflare-Cookie erzeugt, der dann für
-                      den Suchlauf verwendet wird. Die Kosten pro Seite und Suchlauf liegen im Sub-Cent-Bereich.<br>
-                      Um Kosten zu sparen, kann parallel ein FlareSolverr betrieben werden.
-                    </mark>
-                  </div>
                   <FormKit v-model="store.settings.general.flaresolverr"
                            help="Hier die URL eines durch FeedCrawler erreichbaren FlareSolverrs angeben. FlareSolverr wird für jede Seite genutzt, auf der eine Blockade durch Cloudflare erkannt wurde."
                            help-class="text-muted"