|
22 | 22 | """ |
23 | 23 |
|
24 | 24 | import re |
| 25 | +import time |
| 26 | +import urllib.request |
| 27 | +import urllib.error |
25 | 28 |
|
26 | 29 | VALID_DOMAINS = [ |
27 | 30 | 'Agriculture', # Farming, harvesting, crop monitoring, and precision agriculture |
| 31 | + 'Aviation', # Aircraft systems, avionics, luggage handling, passenger service, and airport operations |
28 | 32 | 'Aerial/Drone', # UAVs, drones, aerial inspection, and survey systems |
29 | 33 | 'Automotive', # Self-driving cars, ADAS, and ground vehicle autonomy |
30 | 34 | 'Components', # Robot parts and peripherals (cameras, LIDAR, RADAR, SONAR, etc) |
@@ -100,3 +104,85 @@ def validate_adopters(adopters): |
100 | 104 | f'ISO 3166-1 alpha-2 code, got "{code}"' |
101 | 105 | ) |
102 | 106 | return errors |
| 107 | + |
| 108 | + |
| 109 | +def validate_adopter_urls(adopters, timeout=10): |
| 110 | + """Check URL availability for adopter entries. Returns list of warning strings. |
| 111 | +
|
| 112 | + Only flags URLs that are genuinely unreachable (connection failures |
| 113 | + or server-down HTTP status codes). Any HTTP response — even an |
| 114 | + error like 403 — proves the host is alive. |
| 115 | + """ |
| 116 | + warnings = [] |
| 117 | + if not isinstance(adopters, list): |
| 118 | + return ["'adopters' must be a list"] |
| 119 | + |
| 120 | + seen_urls = {} # cache results to avoid duplicate requests |
| 121 | + |
| 122 | + for i, entry in enumerate(adopters): |
| 123 | + if not isinstance(entry, dict): |
| 124 | + continue |
| 125 | + prefix = ( |
| 126 | + f'Entry {i + 1} ' |
| 127 | + f'({entry.get("organization", "unknown")}/{entry.get("project", "unknown")})' |
| 128 | + ) |
| 129 | + for url_field in ('organization_url', 'project_url'): |
| 130 | + url = entry.get(url_field) |
| 131 | + if not url: |
| 132 | + continue |
| 133 | + |
| 134 | + # Use cached result if we already checked this URL. |
| 135 | + if url in seen_urls: |
| 136 | + if seen_urls[url] is not None: |
| 137 | + warnings.append(f'{prefix}: "{url_field}" {seen_urls[url]}') |
| 138 | + continue |
| 139 | + |
| 140 | + error_msg = _check_url(url, timeout) |
| 141 | + seen_urls[url] = error_msg |
| 142 | + if error_msg: |
| 143 | + warnings.append(f'{prefix}: "{url_field}" {error_msg}') |
| 144 | + return warnings |
| 145 | + |
| 146 | + |
| 147 | +def _check_url(url, timeout, retries=3, backoff_factor=2): |
| 148 | + """Return an error message string if the URL is unreachable, else None. |
| 149 | +
|
| 150 | + Only flags URLs that are genuinely dead — connection failures and |
| 151 | + HTTP status codes that explicitly indicate the server or origin is |
| 152 | + down. Any other HTTP response (even 403, 429, etc.) proves the host |
| 153 | + is alive, so those are treated as reachable. SSL/TLS errors also |
| 154 | + indicate the host responded at the TCP level, so those are not flagged. |
| 155 | +
|
| 156 | + Retries transient failures (timeouts, network errors, 5xx codes) up to |
| 157 | + ``retries`` times with exponential backoff to avoid false alarms from |
| 158 | + flaky network conditions. |
| 159 | + """ |
| 160 | + # HTTP codes that mean the server/origin is down or unreachable. |
| 161 | + _UNREACHABLE_CODES = (502, 503, 504, 521, 522, 523, 525, 530) |
| 162 | + |
| 163 | + last_error = None |
| 164 | + for attempt in range(retries): |
| 165 | + if attempt > 0: |
| 166 | + time.sleep(backoff_factor ** attempt) |
| 167 | + try: |
| 168 | + req = urllib.request.Request(url, method='HEAD') |
| 169 | + with urllib.request.urlopen(req, timeout=timeout) as resp: |
| 170 | + if resp.status in _UNREACHABLE_CODES: |
| 171 | + last_error = f'URL returned HTTP {resp.status}: {url}' |
| 172 | + continue |
| 173 | + return None |
| 174 | + except urllib.error.HTTPError as e: |
| 175 | + if e.code in _UNREACHABLE_CODES: |
| 176 | + last_error = f'URL returned HTTP {e.code}: {url}' |
| 177 | + continue |
| 178 | + return None # any other HTTP response means the server is alive |
| 179 | + except (urllib.error.URLError, OSError) as e: |
| 180 | + # SSL/TLS errors mean the TCP connection succeeded — host is alive. |
| 181 | + err_str = str(e) |
| 182 | + if 'SSL' in err_str or 'CERTIFICATE' in err_str: |
| 183 | + return None |
| 184 | + last_error = f'URL is not reachable: {url} ({e})' |
| 185 | + continue |
| 186 | + except ValueError as e: |
| 187 | + return f'invalid URL: {url} ({e})' |
| 188 | + return last_error |
0 commit comments