completes sonatype

Author: leordev

metrics-collector/package.json

@@ -18,6 +18,7 @@
     "@octokit/types": "^13.5.0",
     "@types/node": "^20.12.12",
     "csv-writer": "^1.6.0",
+    "date-fns": "^3.6.0",
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
     "faker": "^6.6.6",

metrics-collector/pnpm-lock.yaml

@@ -136,12 +136,6 @@ async function initialLoad(
     `Initial load from ${initialLoadFromDate} to ${initialLoadToDate} with date ${date}`
   );
 
-  // if (monthlyInterval) {
-  //   // Change the date to the first day of the month
-  //   date.setDate(0);
-  // }
-  // console.info(`Date after setting to first day of the month: ${date}`);
-
   while (date <= initialLoadToDate) {
     const dateStr = date.toISOString().split("T")[0];
     console.log(`\n\n>>> Collecting metric ${metricName} for date: ${dateStr}`);

metrics-collector/src/index.ts

@@ -1,6 +1,7 @@
 import * as fs from "fs";
 import * as path from "path";
 import { createObjectCsvWriter } from "csv-writer";
+import { parse, format, subMonths } from "date-fns";
 import { fetchWithRetry, readJsonFile, writeJsonFile } from "./utils";
 import { postMetric } from "./post-metric";
 
@@ -27,24 +28,28 @@ export async function collectSonatypeMetrics(metricDate: string) {
       continue; // TODO: add parameterized filter
     }
 
+    const reportPeriod = getLastMonthPeriod(metricDate);
+    const reportPeriodWithoutHyphen = reportPeriod.replace("-", "");
+
     const rawDownloads = await getArtifactStats(
       projectId,
       groupId,
       artifact,
       "raw",
-      metricDate
+      reportPeriodWithoutHyphen
     );
     const uniqueIPs = await getArtifactStats(
       projectId,
       groupId,
       artifact,
       "ip",
-      metricDate
+      reportPeriodWithoutHyphen
     );
 
     await postSonatypeMavenMetrics({
       artifact,
       metricDate: new Date(metricDate),
+      reportPeriod,
       rawDownloads: rawDownloads.total,
       uniqueIPs: uniqueIPs.total,
     });
@@ -56,12 +61,14 @@ export async function collectSonatypeMetrics(metricDate: string) {
 async function postSonatypeMavenMetrics(metric: {
   artifact: string;
   metricDate: Date;
+  reportPeriod: string;
   rawDownloads: number;
   uniqueIPs: number;
 }) {
   console.info("posting sonatype metric", { metric });
   const labels = {
     artifact: metric.artifact,
+    reportPeriod: metric.reportPeriod,
   };
 
   await postMetric(
@@ -195,11 +202,9 @@ async function getArtifactStats(
   groupId: string,
   artifactId: string,
   type: string,
-  fromDate?: string
+  reportPeriod?: string
 ): Promise<{ total: number }> {
-  const from = fromDate
-    ? convertDateToLastYearMonth(fromDate)
-    : getLastMonthDate();
+  const from = reportPeriod ?? getLastMonthDate();
   console.info(
     `Fetching ${type} stats for artifact ${artifactId} from ${from}...`
   );
@@ -275,12 +280,8 @@ function getLastMonthDate() {
   return `${lastMonthYear}${String(lastMonth).padStart(2, "0")}`;
 }
 
-// function to convert YYYY-MM-DD to YYYYMM
-function convertDateToLastYearMonth(date: string) {
-  console.info(`Converting date ${date} to last year month`);
-  const lastMonth = new Date(date);
-  // reduce 1 month, JS will automatically adjust the year if needed
-  lastMonth.setMonth(lastMonth.getMonth() - 1);
-  const [year, month] = lastMonth.toISOString().split("T")[0].split("-");
-  return `${year}${month}`;
+function getLastMonthPeriod(date: string): string {
+  const parsedDate = parse(date, "yyyy-MM-dd", new Date());
+  const previousMonth = subMonths(parsedDate, 1);
+  return format(previousMonth, "yyyy-MM");
 }

metrics-collector/src/sonatype-metrics.ts

@@ -1,5 +1,5 @@
 import * as fs from "fs";
-import * as path from "path";
+import { subDays, format } from "date-fns";
 
 // Read JSON data from the file
 export function readJsonFile(filePath: string): any {
@@ -16,9 +16,8 @@ export function writeJsonFile(filePath: string, data: any): void {
 }
 
 export const getYesterdayDate = () => {
-  const yesterday = new Date();
-  yesterday.setDate(yesterday.getDate() - 1);
-  return yesterday.toISOString().split("T")[0];
+  const yesterday = subDays(new Date(), 1);
+  return format(yesterday, "yyyy-MM-dd");
 };
 
 interface FetchWithRetryOptions {
@@ -32,7 +31,7 @@ export async function fetchWithRetry(
   options: RequestInit & FetchWithRetryOptions = {}
 ): Promise<Response> {
   const {
-    maxRetries = 3,
+    maxRetries = 9,
     retryDelay = 1000,
     timeout = 10000,
     ...fetchOptions