Skip to content
This repository was archived by the owner on Nov 18, 2024. It is now read-only.

Commit dcce792

Browse files
leordevleordev
committed
update service and scripts
1 parent 1f8c546 commit dcce792

File tree

5 files changed

+165
-47
lines changed

5 files changed

+165
-47
lines changed

‎metrics-collector/src/gh-contributions-to-csv.js

+5-6
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,11 @@ function extractData(data) {
2020

2121
// Collect all months from all projects
2222
projects.forEach(project => {
23-
data[project].forEach(entry => {
24-
const { data } = entry;
25-
Object.values(data).forEach(item => {
26-
Object.keys(item).forEach(month => {
27-
allMonths.add(month);
28-
});
23+
const lastEntry = data[project][data[project].length - 1];
24+
const { data: lastEntryData } = lastEntry;
25+
Object.values(lastEntryData).forEach(item => {
26+
Object.keys(item).forEach(month => {
27+
allMonths.add(month);
2928
});
3029
});
3130
});

‎metrics-collector/src/gh-metrics.ts

+3
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,18 @@ import { readJsonFile, writeJsonFile } from "./utils";
77

88
const orgName = "TBD54566975";
99
const repos = [
10+
"tbdex",
1011
"tbdex-js",
1112
"tbdex-kt",
1213
"tbdex-swift",
1314
"tbdex-rs",
15+
"web5-spec",
1416
"web5-js",
1517
"web5-kt",
1618
"web5-swift",
1719
"web5-rs",
1820
"dwn-sdk-js",
21+
"dwn-server",
1922
];
2023

2124
const KNOWN_PAST_MEMBERS = ["amika-sq"];

‎metrics-collector/src/index.ts

+49-17
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@ import yargs from "yargs";
55
import { hideBin } from "yargs/helpers";
66

77
import { collectGhMetrics } from "./gh-metrics";
8-
import { collectNpmMetrics, saveNpmMetrics } from "./npm-metrics";
8+
import { collectNpmMetrics } from "./npm-metrics";
99
import {
1010
collectSonatypeMetrics,
1111
saveSonatypeMetrics,
1212
} from "./sonatype-metrics";
13-
import { getYesterdayDate } from "./utils";
13+
import { getYesterdayDate, readJsonFile } from "./utils";
14+
import { readFile, writeFile } from "fs/promises";
15+
import { existsSync } from "fs";
1416

1517
const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true";
1618

@@ -71,7 +73,12 @@ async function main() {
7173
if (collectNpm) {
7274
console.info(`\n\n============\n\n>>> Collecting metrics for NPM...`);
7375
if (initialLoadFromDate) {
74-
await initialLoad(initialLoadFromDate, metricDate, collectNpmMetrics);
76+
await initialLoad(
77+
"npm-metrics",
78+
initialLoadFromDate,
79+
metricDate,
80+
collectNpmMetrics
81+
);
7582
} else {
7683
await collectNpmMetrics(metricDateStr);
7784
}
@@ -84,6 +91,7 @@ async function main() {
8491
);
8592
if (initialLoadFromDate) {
8693
await initialLoad(
94+
"sonatype-metrics",
8795
initialLoadFromDate,
8896
metricDate,
8997
collectSonatypeMetrics,
@@ -102,45 +110,69 @@ async function main() {
102110

103111
const localCollection = !collectGh && !collectNpm && !collectSonatype;
104112
if (localCollection) {
105-
console.info(
106-
`\n\n============\n\n>>> Collecting local metrics...`
107-
);
113+
console.info(`\n\n============\n\n>>> Collecting local metrics...`);
108114
// await saveNpmMetrics();
109115
// await saveSonatypeMetrics();
110116
await collectGhMetrics(true);
117+
// await saveNpmMetrics();
118+
await saveSonatypeMetrics();
119+
// await collectGhMetrics(true);
111120
}
112121
}
113122

114123
async function initialLoad(
124+
metricName: string,
115125
initialLoadFromDate: Date,
116126
initialLoadToDate: Date,
117127
collectMetrics: (metricDate: string) => Promise<void>,
118-
monthlyInterval = false
128+
monthlyInterval = false,
129+
skipLastSavedState = false
119130
) {
120-
let date = initialLoadFromDate;
131+
const lastSavedState =
132+
!skipLastSavedState && (await getLastSavedState(metricName));
133+
const date = lastSavedState || initialLoadFromDate;
134+
121135
if (monthlyInterval) {
122136
// Change the date to the first day of the month
123137
date.setDate(0);
124138
}
125139

126140
while (date <= initialLoadToDate) {
127141
const dateStr = date.toISOString().split("T")[0];
128-
console.log(`\n\n>>> Collecting metrics for date: ${dateStr}`);
142+
console.log(`\n\n>>> Collecting metric ${metricName} for date: ${dateStr}`);
129143
await collectMetrics(dateStr);
130-
131144
if (monthlyInterval) {
132145
// Move to the next month (JS will handle year change automatically)
133146
date.setMonth(date.getMonth() + 1);
134147
} else {
135148
date.setDate(date.getDate() + 1);
136149
}
150+
await saveLastSavedState(metricName, date);
137151
}
138152
}
139153

140-
main().then(() => {
141-
console.log("Data collection completed successfully");
142-
process.exit(0);
143-
}).catch((error) => {
144-
console.error("Data collection failed", error);
145-
process.exit(1);
146-
});
154+
export const getLastSavedState = async (metricName: string) => {
155+
const stateDir = process.env.LAST_SAVED_STATE_PATH || "./";
156+
const filePath = `${stateDir}/last-saved-state-${metricName}`;
157+
if (!existsSync(filePath)) {
158+
return undefined;
159+
}
160+
const lastSavedState = await readFile(filePath);
161+
return new Date(lastSavedState.toString("utf8"));
162+
};
163+
164+
export const saveLastSavedState = (metricName: string, date: Date) => {
165+
const stateDir = process.env.LAST_SAVED_STATE_PATH || "./";
166+
const filePath = `${stateDir}/last-saved-state-${metricName}`;
167+
return writeFile(filePath, date.toISOString());
168+
};
169+
170+
main()
171+
.then(() => {
172+
console.log("Data collection completed successfully");
173+
process.exit(0);
174+
})
175+
.catch((error) => {
176+
console.error("Data collection failed", error);
177+
process.exit(1);
178+
});

‎metrics-collector/src/service-app.ts

+74-16
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,24 @@ const initDb = async () => {
3535
value DOUBLE PRECISION NOT NULL,
3636
labels JSONB,
3737
metric_timestamp TIMESTAMPTZ NOT NULL DEFAULT now(),
38-
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
38+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
39+
updated_at TIMESTAMPTZ
3940
);
4041
4142
CREATE INDEX IF NOT EXISTS idx_metric_name ON metrics(metric_name);
4243
CREATE INDEX IF NOT EXISTS idx_metric_timestamp ON metrics(metric_timestamp);
4344
CREATE INDEX IF NOT EXISTS idx_metric_labels ON metrics USING GIN(labels);
45+
46+
DO $$
47+
BEGIN
48+
IF NOT EXISTS (
49+
SELECT 1 FROM pg_constraint
50+
WHERE conname = 'metrics_unique_constraint'
51+
) THEN
52+
ALTER TABLE metrics ADD CONSTRAINT metrics_unique_constraint
53+
UNIQUE (metric_name, labels, metric_timestamp);
54+
END IF;
55+
END $$;
4456
`;
4557
await pool.query(createTableQuery);
4658
console.log("Database initialized");
@@ -84,12 +96,20 @@ const seedDb = async () => {
8496
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
8597
);
8698

87-
const insertQuery = `
88-
INSERT INTO metrics (metric_name, value, labels, metric_timestamp) VALUES ($1, $2, $3, $4)
99+
const upsertQuery = `
100+
INSERT INTO metrics (metric_name, value, labels, metric_timestamp)
101+
VALUES ($1, $2, $3, $4)
102+
ON CONFLICT (metric_name, labels, metric_timestamp)
103+
DO UPDATE SET
104+
value = EXCLUDED.value,
105+
updated_at = CASE
106+
WHEN metrics.value = EXCLUDED.value THEN metrics.updated_at
107+
ELSE now()
108+
END
89109
`;
90110

91111
for (const val of values) {
92-
await pool.query(insertQuery, [
112+
await pool.query(upsertQuery, [
93113
val.metricName,
94114
val.value,
95115
val.labels,
@@ -106,29 +126,67 @@ if (!process.env.DB_SKIP_INIT) {
106126

107127
// Collect Metrics Endpoint
108128
app.post("/api/v1/metrics", async (req: any, res: any) => {
109-
const { metricName, value, labels, timestamp } = req.body;
129+
const { metricName, value, labels, timestamp, operation = "set" } = req.body;
110130
if (!metricName || value == undefined) {
111131
return res
112132
.status(400)
113133
.send({ error: "Missing required fields: metricName and value" });
114134
}
115135

116136
try {
117-
await pool.query(
118-
"INSERT INTO metrics (metric_name, value, labels, metric_timestamp) VALUES ($1, $2, $3, $4)",
119-
[
137+
let query;
138+
139+
switch (operation) {
140+
case "inc":
141+
query = `
142+
INSERT INTO metrics (metric_name, value, labels, metric_timestamp)
143+
VALUES ($1, $2, $3, $4)
144+
ON CONFLICT (metric_name, labels, metric_timestamp)
145+
DO UPDATE SET
146+
value = metrics.value + EXCLUDED.value,
147+
updated_at = CASE
148+
WHEN metrics.value = metrics.value + EXCLUDED.value THEN metrics.updated_at
149+
ELSE now()
150+
END
151+
`;
152+
break;
153+
case "set":
154+
query = `
155+
INSERT INTO metrics (metric_name, value, labels, metric_timestamp)
156+
VALUES ($1, $2, $3, $4)
157+
ON CONFLICT (metric_name, labels, metric_timestamp)
158+
DO UPDATE SET
159+
value = EXCLUDED.value,
160+
updated_at = CASE
161+
WHEN metrics.value = EXCLUDED.value THEN metrics.updated_at
162+
ELSE now()
163+
END
164+
`;
165+
break;
166+
default:
167+
throw new Error(`Unsupported operation: ${operation}`);
168+
}
169+
170+
const params = [
171+
metricName,
172+
value,
173+
labels || undefined,
174+
timestamp || new Date().toISOString(),
175+
];
176+
177+
await pool.query(query, params);
178+
179+
console.info(
180+
`>>> Metric ${operation}: ${JSON.stringify({
120181
metricName,
121182
value,
122-
labels || undefined,
123-
timestamp || new Date().toISOString(),
124-
]
125-
);
126-
console.info(
127-
">>> Metric collected: ",
128-
JSON.stringify({ metricName, value, labels, timestamp })
183+
labels,
184+
timestamp,
185+
})}`
129186
);
130-
res.status(201).send({ message: "Metric collected" });
187+
res.status(201).send();
131188
} catch (err) {
189+
console.error("Failed to collect metric", err);
132190
res.status(500).send({ error: "Failed to collect metric" });
133191
}
134192
});

‎metrics-collector/src/sonatype-metrics.ts

+34-8
Original file line numberDiff line numberDiff line change
@@ -72,17 +72,32 @@ export async function saveSonatypeMetrics() {
7272
const metrics = [];
7373

7474
for (const artifact of artifacts) {
75-
const [rawDownloads, uniqueIPs] = await Promise.all([
76-
getArtifactStats(projectId, groupId, artifact, "raw"),
77-
getArtifactStats(projectId, groupId, artifact, "ip"),
78-
]);
75+
if (!["tbdex", "web5"].find((a) => artifact.includes(a))) {
76+
continue;
77+
}
7978

80-
metrics.push({
79+
const rawDownloads = await getArtifactStats(
80+
projectId,
81+
groupId,
82+
artifact,
83+
"raw"
84+
);
85+
const uniqueIPs = await getArtifactStats(
86+
projectId,
87+
groupId,
88+
artifact,
89+
"ip"
90+
);
91+
92+
const artifactMetrics = {
8193
artifact,
8294
timestamp,
8395
rawDownloads: rawDownloads.total,
8496
uniqueIPs: uniqueIPs.total,
85-
});
97+
};
98+
console.info({ [artifact]: artifactMetrics });
99+
metrics.push(artifactMetrics);
100+
await new Promise((resolve) => setTimeout(resolve, 5000)); // prevent rate limit
86101
}
87102

88103
console.info("Sonatype metrics collected successfully", { metrics });
@@ -180,8 +195,19 @@ async function getArtifactStats(
180195
}
181196
);
182197

183-
const data = await response.json();
184-
return data.data;
198+
const responseText = await response.text();
199+
try {
200+
const data = JSON.parse(responseText);
201+
return data.data;
202+
} catch (error) {
203+
console.error(
204+
"Failed to parse response as JSON:",
205+
response.status,
206+
response.statusText,
207+
responseText
208+
);
209+
throw new Error("Unable to parse response as JSON");
210+
}
185211
} catch (error) {
186212
console.error(
187213
`Error fetching ${type} stats for artifact ${artifactId}:`,

0 commit comments

Comments
 (0)