Merge pull request #718 from NFDI4BIOIMAGE/improve-url-validity

Improve url validity script

Author: haesleinhuepf

.github/workflows/auto-add-newsletter-links.yml

@@ -2,7 +2,7 @@ name: Generate Newsletter Links
 
 on:
   schedule:
-    - cron: '0 0 4 * *'  # Runs at midnight (UTC) on the 4st day of every month
+    - cron: '0 0 4 9,5,3 *'  # Runs at midnight on the 4th of March, May and September
     # Runs at 8:00 AM UTC on the first Monday of every 2nd month
     #- cron: "0 8 * 1,3,5,7,9,11 1"
 

.github/workflows/auto-url-validity-check.yml

@@ -2,7 +2,7 @@ name: URL Validity Check
 
 on:
   schedule:
-    - cron: '0 0 4 * *'  # Runs at midnight (UTC) on the 4st day of every month
+    - cron: '0 0 4 9,5,3 *'  # Runs at midnight on the 4th of March and September
   workflow_dispatch:  # Allows manual trigger
 
 jobs:
@@ -32,7 +32,7 @@ jobs:
       - name: Extract Failing URLs
         run: |
           echo "Please be so kind to double-check the validity of the following urls:" > failing_urls.md
-          grep -E '❌' url_check_results.log >> failing_urls.md || echo "No issues found!" >> failing_urls.md
+          grep -E '❌|⚠️' url_check_results.log >> failing_urls.md || echo "No issues found!" >> failing_urls.md
 
       - name: Create GitHub Issue
         if: success()

scripts/auto-url-validity-check.py

@@ -49,34 +49,48 @@ def extract_urls(file_path):
     return urls
 
 def check_url(url):
-    """Check if a URL is reachable with retries and logging."""
     headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/91.0.4472.124 Safari/537.36"
+        ),
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Referer": "https://www.google.com"
     }
 
+    last_error = ""
     for attempt in range(1, max_retries + 1):
         try:
             response = requests.get(url, headers=headers, timeout=5, allow_redirects=True)
-            
-            if response.status_code == 200:
+            status = response.status_code
+
+            if status == 200:
                 return f"✅ {url} is reachable (Attempt {attempt})"
-            elif response.status_code == 429:
+            elif status == 429:
                 return f"✅ {url} is rate-limited (429), considering it reachable."
-            elif response.status_code in [403, 404]:
-                return f"❌ {url} returned status {response.status_code}. It may be blocking bots. (Attempt {attempt})"
+            elif status in (404, 410):
+                return f"❌ {url} returned {status} (Attempt {attempt})"
             else:
-                return f"❌ {url} returned status {response.status_code} (Attempt {attempt})"
-        except requests.exceptions.ConnectionError:
-            return f"❌ {url} is unreachable (Connection Error) (Attempt {attempt})"
+                last_error = f"{url} returned status {status} (Attempt {attempt}), final URL: {response.url}"
+
+        except requests.exceptions.SSLError as e:
+            last_error = f"{url} SSL Error: {e}"
+        except requests.exceptions.ConnectionError as e:
+            last_error = f"{url} Connection Error: {e}"
         except requests.exceptions.Timeout:
-            return f"❌ {url} is unreachable (Timeout) (Attempt {attempt})"
+            last_error = f"{url} Timeout"
         except requests.exceptions.RequestException as e:
-            return f"❌ {url} failed due to {e} (Attempt {attempt})"
-        
-        # Wait before retrying
+            last_error = f"{url} failed: {e}"
+
         time.sleep(random.uniform(1, 3))
 
-    return f"❌ {url} is unreachable after {max_retries} attempts."
+    # Final classification
+    if any(code in last_error for code in ["returned 404", "returned 410"]):
+        return f"❌ {last_error}"
+    else:
+        return f"⚠️ {url} is potentially reachable but failed after {max_retries} attempts. Last error: {last_error}"
 
 def log_results(results):
     """Log results to a file."""