Add FileLions extractor and improve eval_solver (#135)

* Add FileLions extractor and support for p.a.c.k.e.d pattern array

* Improve eval_solver URL creation

* Support also hls3 link

* Improve eval_resolver failing when no pattern matches

* Improve URL generation a bit more

* Apply more review suggestions

* Further simplify URL creation and fix error message

* Remove hls3 link again because it's not always working

Author: webstreamr

mediaflow_proxy/extractors/factory.py

@@ -3,6 +3,7 @@
 from mediaflow_proxy.extractors.base import BaseExtractor, ExtractorError
 from mediaflow_proxy.extractors.dlhd import DLHDExtractor
 from mediaflow_proxy.extractors.doodstream import DoodStreamExtractor
+from mediaflow_proxy.extractors.filelions import FileLionsExtractor
 from mediaflow_proxy.extractors.livetv import LiveTVExtractor
 from mediaflow_proxy.extractors.maxstream import MaxstreamExtractor
 from mediaflow_proxy.extractors.mixdrop import MixdropExtractor
@@ -19,6 +20,7 @@ class ExtractorFactory:
 
     _extractors: Dict[str, Type[BaseExtractor]] = {
         "Doodstream": DoodStreamExtractor,
+        "FileLions": FileLionsExtractor,
         "Uqload": UqloadExtractor,
         "Mixdrop": MixdropExtractor,
         "Streamtape": StreamtapeExtractor,

mediaflow_proxy/extractors/fastream.py

@@ -14,9 +14,9 @@ def __init__(self, *args, **kwargs):
 
     async def extract(self, url: str, **kwargs) -> Dict[str, Any]:
         headers  = {'Accept': '*/*', 'Connection': 'keep-alive','Accept-Language': 'en-US,en;q=0.5','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:138.0) Gecko/20100101 Firefox/138.0'}
-        pattern = r'file:"(.*?)"'
+        patterns = [r'file:"(.*?)"']
 
-        final_url = await eval_solver(self, url, headers, pattern)
+        final_url = await eval_solver(self, url, headers, patterns)
 
         self.base_headers["referer"] = f'https://{url.replace("https://","").split("/")[0]}/'
         self.base_headers["origin"] = f'https://{url.replace("https://","").split("/")[0]}'

mediaflow_proxy/extractors/filelions.py

@@ -0,0 +1,25 @@
+from typing import Dict, Any
+
+from mediaflow_proxy.extractors.base import BaseExtractor
+from mediaflow_proxy.utils.packed import eval_solver
+
+class FileLionsExtractor(BaseExtractor):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.mediaflow_endpoint = "hls_manifest_proxy"
+
+    async def extract(self, url: str, **kwargs) -> Dict[str, Any]:
+        headers  = {}
+        patterns = [ # See https://github.com/Gujal00/ResolveURL/blob/master/script.module.resolveurl/lib/resolveurl/plugins/filelions.py
+            r'''sources:\s*\[{file:\s*["'](?P<url>[^"']+)''',
+            r'''["']hls[24]["']:\s*["'](?P<url>[^"']+)'''
+        ]
+
+        final_url = await eval_solver(self, url, headers, patterns)
+
+        self.base_headers["referer"] = url
+        return {
+            "destination_url": final_url,
+            "request_headers": self.base_headers,
+            "mediaflow_endpoint": self.mediaflow_endpoint,
+        }

mediaflow_proxy/extractors/mixdrop.py

@@ -13,9 +13,9 @@ async def extract(self, url: str, **kwargs) -> Dict[str, Any]:
             url = url.replace("club", "ps").split("/2")[0]
 
         headers = {"accept-language": "en-US,en;q=0.5"}
-        pattern = r'MDCore.wurl ?= ?"(.*?)"'
+        patterns = [r'MDCore.wurl ?= ?"(.*?)"']
 
-        final_url = f"https:{await eval_solver(self, url, headers, pattern)}"
+        final_url = await eval_solver(self, url, headers, patterns)
 
         self.base_headers["referer"] = url
         return {

mediaflow_proxy/extractors/supervideo.py

@@ -15,9 +15,9 @@ def __init__(self, *args, **kwargs):
 
     async def extract(self, url: str, **kwargs) -> Dict[str, Any]:
         headers  = {'Accept': '*/*', 'Connection': 'keep-alive', 'User-Agent': 'Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.71 Mobile Safari/537.36', 'user-agent': 'Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.71 Mobile Safari/537.36'}
-        pattern = r'file:"(.*?)"'
+        patterns = [r'file:"(.*?)"']
 
-        final_url = await eval_solver(self, url, headers, pattern)
+        final_url = await eval_solver(self, url, headers, patterns)
 
         self.base_headers["referer"] = url
         return {

mediaflow_proxy/schemas.py

@@ -96,7 +96,7 @@ class MPDSegmentParams(GenericParams):
 
 class ExtractorURLParams(GenericParams):
     host: Literal[
-        "Doodstream", "Mixdrop", "Uqload", "Streamtape", "Supervideo", "VixCloud", "Okru", "Maxstream", "LiveTV", "DLHD", "Fastream"
+        "Doodstream", "FileLions", "Mixdrop", "Uqload", "Streamtape", "Supervideo", "VixCloud", "Okru", "Maxstream", "LiveTV", "DLHD", "Fastream"
     ] = Field(..., description="The host to extract the URL from.")
     destination: str = Field(..., description="The URL of the stream.", alias="d")
     redirect_stream: bool = Field(False, description="Whether to redirect to the stream endpoint automatically.")

mediaflow_proxy/utils/packed.py

@@ -14,6 +14,7 @@
 
 import re
 from bs4 import BeautifulSoup, SoupStrainer
+from urllib.parse import urljoin, urlparse
 import logging
 
 
@@ -142,18 +143,23 @@ class UnpackingError(Exception):
 
 
 
-async def eval_solver(self, url: str, headers, pattern: str) -> str:
+async def eval_solver(self, url: str, headers: dict[str, str] | None, patterns: list[str]) -> str:
     try:
         response = await self._make_request(url, headers=headers)
         soup = BeautifulSoup(response.text, "lxml",parse_only=SoupStrainer("script"))
         script_all = soup.find_all("script")
         for i in script_all:
             if detect(i.text):
                 unpacked_code = unpack(i.text)
-                match = re.search(pattern, unpacked_code)
-                if match:
-                    m3u8_url = match.group(1)
-                    return m3u8_url
+                for pattern in patterns:
+                    match = re.search(pattern, unpacked_code)
+                    if match:
+                        extracted_url = match.group(1)
+                        if not urlparse(extracted_url).scheme:
+                            extracted_url = urljoin(url, extracted_url)
+
+                        return extracted_url
+        raise UnpackingError("No p.a.c.k.e.d JS found or no pattern matched.")
     except Exception as e:
-        logger.error("Eval solver error\n",e)
-        raise Exception("Error in eval_solver")
+        logger.exception("Eval solver error for %s", url)
+        raise UnpackingError("Error in eval_solver") from e