Merge pull request #746 from NFDI4BIOIMAGE/url-validity-again

Url validity again

Author: haesleinhuepf

.github/workflows/auto-url-validity-check.yml

@@ -2,7 +2,7 @@ name: URL Validity Check
 
 on:
   schedule:
-    - cron: '0 0 4 9,5,3 *'  # Runs at midnight on the 4th of March and September
+    - cron: '0 0 4 6,12 *'  # Runs at midnight on the 4th of June and December
   workflow_dispatch:  # Allows manual trigger
 
 jobs:
@@ -32,7 +32,7 @@ jobs:
       - name: Extract Failing URLs
         run: |
           echo "Please be so kind to double-check the validity of the following urls:" > failing_urls.md
-          grep -E '❌|⚠️' url_check_results.log >> failing_urls.md || echo "No issues found!" >> failing_urls.md
+          grep -E '❌' url_check_results.log >> failing_urls.md || echo "No issues found!" >> failing_urls.md
 
       - name: Create GitHub Issue
         if: success()

scripts/auto-url-validity-check.py

@@ -5,6 +5,7 @@
 import time
 import datetime
 import random
+import os
 
 # File paths
 yaml_file_path = "resources/nfdi4bioimage.yml"
@@ -15,8 +16,12 @@
 #create filename
 log_file = f'scripts/url_validity_check/{date}_url_check_results.log'
 
+# Ensure log directory exists
+os.makedirs(os.path.dirname(log_file), exist_ok=True)
+
 # Max retries for failed requests
 max_retries = 3
+base_backoff = 2  
 
 def main():
     """
@@ -49,48 +54,47 @@ def extract_urls(file_path):
     return urls
 
 def check_url(url):
+    user_agents = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
+        "Mozilla/5.0 (X11; Linux x86_64)",
+    ]
+    
     headers = {
-        "User-Agent": (
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/91.0.4472.124 Safari/537.36"
-        ),
+        "User-Agent": random.choice(user_agents),
         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.5",
-        "Referer": "https://www.google.com"
+        "Accept-Language": "en-US,en;q=0.9",
+        "Referer": "https://www.google.com/",
+        "DNT": "1",  # Do Not Track
+        "Connection": "keep-alive",
+        "Upgrade-Insecure-Requests": "1",
+        "Cache-Control": "no-cache"
     }
 
     last_error = ""
     for attempt in range(1, max_retries + 1):
         try:
-            response = requests.get(url, headers=headers, timeout=5, allow_redirects=True)
+            response = requests.get(url, headers=headers, timeout=15, allow_redirects=True)
             status = response.status_code
 
             if status == 200:
                 return f"✅ {url} is reachable (Attempt {attempt})"
             elif status == 429:
-                return f"✅ {url} is rate-limited (429), considering it reachable."
+                return f"⚠️ {url} is rate-limited (429), considering it reachable."
             elif status in (404, 410):
                 return f"❌ {url} returned {status} (Attempt {attempt})"
+            elif status in (403, 400):
+                return f"⚠️ {url} returned status {status}. It may be blocking bots. (Attempt {attempt})"
             else:
                 last_error = f"{url} returned status {status} (Attempt {attempt}), final URL: {response.url}"
 
-        except requests.exceptions.SSLError as e:
-            last_error = f"{url} SSL Error: {e}"
-        except requests.exceptions.ConnectionError as e:
-            last_error = f"{url} Connection Error: {e}"
-        except requests.exceptions.Timeout:
-            last_error = f"{url} Timeout"
         except requests.exceptions.RequestException as e:
-            last_error = f"{url} failed: {e}"
+            last_error = f"{url} failed: {repr(e)}"
 
-        time.sleep(random.uniform(1, 3))
+        sleep_time = base_backoff ** attempt + random.uniform(0, 1)
+        time.sleep(sleep_time)
 
-    # Final classification
-    if any(code in last_error for code in ["returned 404", "returned 410"]):
-        return f"❌ {last_error}"
-    else:
-        return f"⚠️ {url} is potentially reachable but failed after {max_retries} attempts. Last error: {last_error}"
+    return f"⚠️ {url} is potentially reachable but failed after {max_retries} attempts. Last error: {last_error}"
 
 def log_results(results):
     """Log results to a file."""