improve retries

Author: leordev

metrics-collector/README.md

@@ -17,3 +17,13 @@ See the collected metrics at:
 
 - `pr_metrics.json` or `pr_metrics.csv`
 - TODO: `usage_metrics.json`...
+
+## metrics service
+
+```sh
+docker compose build
+docker compose up
+
+# test multiple concurrent requests
+hey -n 1000 -c 50 -m POST -H "Content-Type: application/json" -d '{"metricName":"test_concurrent_metric","value":1,"labels":{"test":"concurrency"},"operation":"increment"}' http://localhost:3001/api/v1/metrics
+```

metrics-collector/src/npm-metrics.ts

@@ -1,7 +1,7 @@
 import * as fs from "fs";
 import * as path from "path";
 import { createObjectCsvWriter } from "csv-writer";
-import { readJsonFile, writeJsonFile } from "./utils";
+import { fetchWithRetry, readJsonFile, writeJsonFile } from "./utils";
 import { postMetric } from "./post-metric";
 
 // Define the npm packages to collect metrics for
@@ -52,7 +52,7 @@ export const collectNpmMetrics = async (metricDate: string) => {
   }
 
   console.info("NPM metrics collected successfully");
-}
+};
 
 // Save collected total downloads and last 30d downloads to a local file
 export async function saveNpmMetrics() {
@@ -125,25 +125,22 @@ async function getNpmDownloadCount(
   onlyLastMonth?: boolean,
   dateRange?: { begin: string; end: string }
 ): Promise<{ downloads: number; start: string }> {
-  try {
-    let url = `https://api.npmjs.org/downloads/point`;
-    if (onlyLastMonth) {
-      url += `/last-month/${packageName}`;
-    } else if (dateRange) {
-      url += `/${dateRange.begin}:${dateRange.end}/${packageName}`;
-    } else {
-      url += `/1970-01-01:2100-01-01/${packageName}`;
-    }
+  let url = `https://api.npmjs.org/downloads/point`;
+  if (onlyLastMonth) {
+    url += `/last-month/${packageName}`;
+  } else if (dateRange) {
+    url += `/${dateRange.begin}:${dateRange.end}/${packageName}`;
+  } else {
+    url += `/1970-01-01:2100-01-01/${packageName}`;
+  }
 
-    const response = await fetch(url);
-    if (!response.ok) {
-      throw new Error(`HTTP error! Status: ${response.status}`);
-    }
+  try {
+    const response = await fetchWithRetry(url);
     const data = await response.json();
     return data;
   } catch (error) {
     console.error(
-      `Error fetching download count for package ${packageName}:`,
+      `Error fetching download count for package ${packageName} from ${url}:`,
       error
     );
     throw error;

metrics-collector/src/utils.ts

@@ -20,3 +20,57 @@ export const getYesterdayDate = () => {
   yesterday.setDate(yesterday.getDate() - 1);
   return yesterday.toISOString().split("T")[0];
 };
+
+interface FetchWithRetryOptions {
+  maxRetries?: number;
+  retryDelay?: number;
+  timeout?: number;
+}
+
+export async function fetchWithRetry(
+  url: string,
+  options: RequestInit & FetchWithRetryOptions = {}
+): Promise<Response> {
+  const {
+    maxRetries = 3,
+    retryDelay = 1000,
+    timeout = 10000,
+    ...fetchOptions
+  } = options;
+
+  let retries = 0;
+  while (retries < maxRetries) {
+    try {
+      const controller = new AbortController();
+      const id = setTimeout(() => controller.abort(), timeout);
+
+      const response = await fetch(url, {
+        ...fetchOptions,
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        const errorData = await response.text();
+        console.error(
+          `HTTP error! Status: ${response.status}, Response: ${errorData}`
+        );
+        throw new Error(
+          `HTTP error! Status: ${response.status}, Response: ${errorData}`
+        );
+      }
+
+      clearTimeout(id);
+
+      return response;
+    } catch (error) {
+      console.error(`Attempt ${retries + 1} failed:`, error);
+      retries++;
+      if (retries >= maxRetries) {
+        throw error;
+      }
+      const exponentialDelay = retryDelay * Math.pow(2, retries - 1);
+      await new Promise((resolve) => setTimeout(resolve, exponentialDelay));
+    }
+  }
+  throw new Error("Max retries reached");
+}