Update compare_mermeroo_mirror.py

Signed-off-by: саня <109840177+kort0881@users.noreply.github.com>

Author: kort0881

compare_mermeroo_mirror.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 import os
 import json
+from urllib.parse import urlparse
 
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 
-from mirror import URLS_BASE, CONFIG_SOURCES_FILE  # импорт из mirror.py (нижний регистр)
+from mirror import URLS_BASE, CONFIG_SOURCES_FILE  # mirror.py в корне репо
 
 MERMEROO_FILE = os.path.join(BASE_PATH, "mermeroo_sources.txt")
 OUT_FILE = os.path.join(BASE_PATH, "mermeroo_only_new_for_mirror.txt")
@@ -33,43 +34,108 @@ def load_config_sources():
 
 
 def extract_repo_key(url: str) -> str:
-    if "raw.githubusercontent.com" not in url:
-        return url
-    parts = url.split("/")
-    try:
-        i = parts.index("raw.githubusercontent.com")
-        owner = parts[i + 1]
-        repo = parts[i + 2]
-        return f"{owner}/{repo}"
-    except Exception:
-        return url
+    if "raw.githubusercontent.com" in url:
+        parts = url.split("/")
+        try:
+            i = parts.index("raw.githubusercontent.com")
+            owner = parts[i + 1]
+            repo = parts[i + 2]
+            return f"{owner}/{repo}"
+        except Exception:
+            return url
+
+    # github.com/.../raw/...
+    if "github.com" in url and "/raw/" in url:
+        parts = url.split("/")
+        try:
+            i = parts.index("github.com")
+            owner = parts[i + 1]
+            repo = parts[i + 2]
+            return f"{owner}/{repo}"
+        except Exception:
+            return url
+
+    return url
+
+
+# ✅ оставляем только «нормальные» источники
+def is_good_source_url(url: str) -> bool:
+    u = urlparse(url)
+
+    if not u.scheme.startswith("http"):
+        return False
+
+    host = (u.netloc or "").lower()
+    path = (u.path or "").lower()
+
+    # Git raw / CDN raw
+    if "raw.githubusercontent.com" in host:
+        return True
+    if "github.com" in host and "/raw/" in path:
+        return True
+    if "gitlab.com" in host and "/-/raw/" in path:
+        return True
+    if "bitbucket.org" in host and "/raw/" in path:
+        return True
+    if "jsdelivr.net" in host and "/gh/" in path:
+        return True
+
+    # всё, что явно «clash/proxies», api subscribe, просто сайты — режем
+    bad_substrings = [
+        "/clash/proxies",
+        "/api/v1/client/subscribe",
+        "/subscribe?",
+        "token=",
+        "/wp-content/",
+        ".html",
+        ".htm",
+        "/free-ss",
+        "/free-ssr",
+        "/v2ray/",
+        "/free/",
+    ]
+    for b in bad_substrings:
+        if b in url.lower():
+            return False
+
+    # текстовые/plain-файлы тоже можно оставить как потенциальные
+    good_exts = (".txt", ".yaml", ".yml", ".json")
+    if any(path.endswith(ext) for ext in good_exts):
+        return True
+
+    return False
 
 
 def main():
     mer_all = load_mermeroo_sources()
     print(f"Всего в mermeroo_sources.txt: {len(mer_all)}")
 
-    # Все источники, которые уже знает mirror.py
+    # 1) фильтруем mermeroo по хорошим URL
+    mer_filtered = [u for u in mer_all if is_good_source_url(u)]
+    print(f"Из них после фильтра по типу URL: {len(mer_filtered)}")
+
+    # 2) все источники, которые уже знает mirror.py
     known_urls = set(URLS_BASE)
     known_urls |= load_config_sources()
 
-    # Новые по точному URL
-    mer_new_urls = [u for u in mer_all if u not in known_urls]
+    # 3) убираем уже известные URL
+    mer_new_urls = [u for u in mer_filtered if u not in known_urls]
 
-    # Новые по репозиторию
+    # 4) новые репозитории-доноры
     known_repos = {extract_repo_key(u) for u in known_urls}
-    mer_repos = {extract_repo_key(u) for u in mer_all}
+    mer_repos = {extract_repo_key(u) for u in mer_filtered}
     mer_new_repos = sorted(r for r in mer_repos if r not in known_repos)
 
-    print(f"Новых URL (нет в Mirror): {len(mer_new_urls)}")
+    print(f"Новых URL (нет в Mirror, после фильтра): {len(mer_new_urls)}")
     print(f"Новых репозиториев-доноров: {len(mer_new_repos)}")
     for r in mer_new_repos:
         print("  ", r)
 
+    # 5) сохраняем только новые «нормальные» URL в файл
     with open(OUT_FILE, "w", encoding="utf-8") as f:
         f.write("\n".join(mer_new_urls))
 
-    print(f"\nСписок новых URL сохранён в {OUT_FILE}")
+    print(f"\nСписок новых отфильтрованных URL сохранён в {OUT_FILE}")
 
 
 if __name__ == "__main__":