Skip to content

Commit 305ea03

Browse files
committed
simplify
1 parent 85e84b3 commit 305ea03

1 file changed

Lines changed: 25 additions & 43 deletions

File tree

src/liana/resource/get_metalinks.py

Lines changed: 25 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -34,50 +34,32 @@ def _download_metalinksdb(verbose=True):
3434
else:
3535
return db_path
3636

37-
# Download the database with retry logic
38-
import time
39-
max_retries = 3
40-
retry_delay = 2 # seconds
41-
42-
for attempt in range(max_retries):
43-
_logg(f"Downloading database (attempt {attempt + 1}/{max_retries})...", verbose=verbose)
44-
try:
45-
# Figshare requires browser-like headers to bypass WAF
46-
headers = {
47-
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
48-
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
49-
'Accept-Language': 'en-US,en;q=0.5',
50-
'Accept-Encoding': 'gzip, deflate',
51-
'Connection': 'keep-alive',
52-
'Upgrade-Insecure-Requests': '1'
53-
}
54-
response = requests.get(METALINKS_URL, headers=headers, stream=True, allow_redirects=True, timeout=60)
55-
response.raise_for_status() # Raise an error for bad status codes
56-
57-
with open(db_path, 'wb') as f:
58-
for chunk in response.iter_content(chunk_size=8192):
59-
f.write(chunk)
60-
61-
# Validate the downloaded file
62-
file_size = os.path.getsize(db_path)
63-
if file_size == 0:
64-
os.remove(db_path)
65-
raise RuntimeError("Downloaded file is empty. Please check the URL and try again.")
66-
67-
_logg(f"Database downloaded and saved to {db_path} ({file_size} bytes).", verbose=verbose)
68-
return db_path
37+
# Download the database
38+
_logg("Downloading database...", verbose=verbose)
39+
try:
40+
# Figshare requires a browser-like User-Agent to bypass WAF
41+
headers = {
42+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
43+
}
44+
response = requests.get(METALINKS_URL, headers=headers, stream=True, allow_redirects=True)
45+
response.raise_for_status() # Raise an error for bad status codes
46+
47+
with open(db_path, 'wb') as f:
48+
for chunk in response.iter_content(chunk_size=8192):
49+
f.write(chunk)
50+
51+
# Validate the downloaded file
52+
file_size = os.path.getsize(db_path)
53+
if file_size == 0:
54+
os.remove(db_path)
55+
raise RuntimeError("Downloaded file is empty. Please check the URL and try again.")
6956

70-
except (requests.exceptions.RequestException, OSError, RuntimeError) as e:
71-
# Clean up failed download
72-
if os.path.exists(db_path):
73-
os.remove(db_path)
74-
75-
if attempt < max_retries - 1:
76-
_logg(f"Download failed: {e}. Retrying in {retry_delay} seconds...", verbose=verbose)
77-
time.sleep(retry_delay)
78-
retry_delay *= 2 # Exponential backoff
79-
else:
80-
raise RuntimeError(f"Failed to download database after {max_retries} attempts: {e}") from e
57+
_logg(f"Database downloaded and saved to {db_path} ({file_size} bytes).", verbose=verbose)
58+
except (requests.exceptions.RequestException, OSError, RuntimeError) as e:
59+
# Clean up failed download
60+
if os.path.exists(db_path):
61+
os.remove(db_path)
62+
raise RuntimeError(f"Failed to download database: {e}") from e
8163

8264
return db_path
8365

0 commit comments

Comments
 (0)