Skip to content

Commit d564470

Browse files
NebraskaCoderclaude
andcommitted
feat: add retry logic with exponential backoff for URL timeout checks
When URLs timeout during the download check, the script now retries up to 10 times with exponential backoff (1s, 2s, 4s, 8s... capped at 60s). This helps avoid false failures from transient network issues while not retrying actual 404s or other real errors. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <[email protected]>
1 parent 8a3065e commit d564470

File tree

1 file changed

+76
-5
lines changed

1 file changed

+76
-5
lines changed

scripts/check-download-urls.js

Lines changed: 76 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ function extractUrls(obj, path = "") {
4040
return urls;
4141
}
4242

43-
// Check if URL is accessible
44-
function checkUrl(urlInfo) {
43+
// Check if URL is accessible (single attempt)
44+
function checkUrlOnce(urlInfo) {
4545
return new Promise((resolve) => {
4646
const url = new URL(urlInfo.url);
4747
const protocol = url.protocol === "https:" ? https : http;
@@ -76,6 +76,63 @@ function checkUrl(urlInfo) {
7676
});
7777
}
7878

79+
// Check URL with retry logic for timeouts only
80+
async function checkUrl(urlInfo) {
81+
return checkUrlOnce(urlInfo);
82+
}
83+
84+
// Retry timed out URLs with exponential backoff
85+
async function retryTimeouts(timedOutResults) {
86+
if (timedOutResults.length === 0) return [];
87+
88+
const maxRetries = 10;
89+
const baseDelay = 1000; // Start with 1 second
90+
let remaining = [...timedOutResults];
91+
const successful = [];
92+
93+
for (
94+
let attempt = 1;
95+
attempt <= maxRetries && remaining.length > 0;
96+
attempt++
97+
) {
98+
// Exponential backoff: 1s, 2s, 4s, 8s, 16s, 32s, 60s (capped), 60s, 60s, 60s
99+
const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), 60000);
100+
101+
console.log(
102+
`\n${colors.yellow}Retry attempt ${attempt}/${maxRetries} for ${remaining.length} timed out URL(s) (waiting ${delay / 1000}s)...${colors.reset}`
103+
);
104+
105+
await new Promise((resolve) => setTimeout(resolve, delay));
106+
107+
const retryResults = await Promise.all(
108+
remaining.map((r) => checkUrlOnce({ url: r.url, path: r.path }))
109+
);
110+
111+
const stillTimedOut = [];
112+
for (const result of retryResults) {
113+
if (result.status === "TIMEOUT") {
114+
stillTimedOut.push(result);
115+
} else {
116+
successful.push(result);
117+
console.log(
118+
` ${colors.green}${colors.reset} ${result.url} - ${result.status === "OK" ? "OK" : result.status}`
119+
);
120+
}
121+
}
122+
123+
remaining = stillTimedOut;
124+
125+
if (remaining.length === 0) {
126+
console.log(
127+
`${colors.green}All timed out URLs recovered after ${attempt} retry attempt(s)!${colors.reset}`
128+
);
129+
}
130+
}
131+
132+
// Return both successful retries and still-failed ones
133+
return [...successful, ...remaining];
134+
}
135+
79136
// Progress bar
80137
function updateProgress(current, total) {
81138
const percentage = Math.round((current / total) * 100);
@@ -121,16 +178,30 @@ async function main() {
121178
// Clear progress bar
122179
process.stdout.write("\r" + " ".repeat(80) + "\r");
123180

181+
// Separate timeouts from other failures for retry logic
182+
const timedOut = results.filter((r) => r.status === "TIMEOUT");
183+
const otherResults = results.filter((r) => r.status !== "TIMEOUT");
184+
185+
// Retry timed out URLs with exponential backoff
186+
let finalResults = [...otherResults];
187+
if (timedOut.length > 0) {
188+
console.log(
189+
`\n${colors.yellow}${timedOut.length} URL(s) timed out. Retrying with exponential backoff...${colors.reset}`
190+
);
191+
const retryResults = await retryTimeouts(timedOut);
192+
finalResults = [...finalResults, ...retryResults];
193+
}
194+
124195
// Report results
125-
const failed = results.filter((r) => r.status !== "OK");
126-
const successful = results.filter((r) => r.status === "OK");
196+
const failed = finalResults.filter((r) => r.status !== "OK");
197+
const successful = finalResults.filter((r) => r.status === "OK");
127198

128199
console.log("\n" + colors.blue + "=" + "=".repeat(50) + colors.reset);
129200
console.log(`${colors.blue}URL Check Results${colors.reset}`);
130201
console.log(colors.blue + "=" + "=".repeat(50) + colors.reset + "\n");
131202

132203
console.log(
133-
`Total URLs checked: ${colors.yellow}${results.length}${colors.reset}`
204+
`Total URLs checked: ${colors.yellow}${finalResults.length}${colors.reset}`
134205
);
135206
console.log(`Successful: ${colors.green}${successful.length}${colors.reset}`);
136207
console.log(`Failed: ${colors.red}${failed.length}${colors.reset}\n`);

0 commit comments

Comments
 (0)