Skip to content

Commit 193e377

Browse files
grstgrst
authored
Add report section to download JSON report (#123)
Add a downloads page to the stats website to obtain a JSON with metrics relevant for pipeline risk assessment. Frankly, this is mostly written by Claude, but * I checked the JSON and it is the same as the one generated by the Python script I wrote during the Barcelona hackathon (see #96) * The SQL queries are based on the ones I defined during the hackathon (also see #96).
2 parents 18c3994 + be4dafc commit 193e377

6 files changed

Lines changed: 223 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ static/data
1010
.evidence/meta
1111
*.pyc
1212
__pycache__
13+
regulatory

‎components/DownloadStatsJson.svelte‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
<script>
2+
/** @type {Array<Object>} */
3+
export let pipelineStats = [];
4+
/** @type {Array<Object>} */
5+
export let issueStats = [];
6+
/** @type {Array<Object>} */
7+
export let contributorStats = [];
8+
9+
/**
10+
* Build the stats JSON structure matching the regulatory report format.
11+
* Output is keyed by pipeline name with pipeline_stats, issue_stats,
12+
* and contributor_stats sub-objects.
13+
*/
14+
function buildStatsJson() {
15+
// Index issue stats by pipeline name
16+
const issuesByPipeline = {};
17+
if (issueStats) {
18+
for (const row of issueStats) {
19+
issuesByPipeline[row.pipeline_name] = {
20+
pipeline_name: row.pipeline_name,
21+
issue_count: row.issue_count ?? null,
22+
closed_issue_count: row.closed_issue_count ?? null,
23+
median_seconds_to_issue_closed: row.median_seconds_to_issue_closed ?? null,
24+
pr_count: row.pr_count ?? null,
25+
closed_pr_count: row.closed_pr_count ?? null,
26+
median_seconds_to_pr_closed: row.median_seconds_to_pr_closed ?? null,
27+
};
28+
}
29+
}
30+
31+
// Index contributor stats by pipeline name
32+
const contribByPipeline = {};
33+
if (contributorStats) {
34+
for (const row of contributorStats) {
35+
contribByPipeline[row.pipeline_name] = {
36+
pipeline_name: row.pipeline_name,
37+
number_of_contributors: row.number_of_contributors,
38+
};
39+
}
40+
}
41+
42+
// Build the final structure keyed by pipeline name
43+
const result = {};
44+
if (pipelineStats) {
45+
for (const row of pipelineStats) {
46+
const name = row.name;
47+
result[name] = {
48+
pipeline_stats: {
49+
name: row.name,
50+
description: row.description,
51+
stargazers_count: row.stargazers_count,
52+
forks_count: row.forks_count,
53+
open_issues_count: row.open_issues_count,
54+
archived: row.archived,
55+
last_release_date: row.last_release_date ?? null,
56+
category: row.category,
57+
},
58+
issue_stats: issuesByPipeline[name] ?? null,
59+
contributor_stats: contribByPipeline[name] ?? null,
60+
};
61+
}
62+
}
63+
64+
return result;
65+
}
66+
67+
function downloadJson() {
68+
const data = buildStatsJson();
69+
const blob = new Blob([JSON.stringify(data, null, 2)], {
70+
type: "application/json",
71+
});
72+
const url = URL.createObjectURL(blob);
73+
const a = document.createElement("a");
74+
a.href = url;
75+
a.download = "nf-core-stats.json";
76+
document.body.appendChild(a);
77+
a.click();
78+
document.body.removeChild(a);
79+
URL.revokeObjectURL(url);
80+
}
81+
82+
$: statsData = buildStatsJson();
83+
$: pipelineCount = Object.keys(statsData).length;
84+
</script>
85+
86+
<div class="download-section">
87+
<button class="download-btn" on:click={downloadJson}>
88+
Download nf-core-stats.json
89+
</button>
90+
<span class="pipeline-count">{pipelineCount} pipelines included</span>
91+
</div>
92+
93+
<style>
94+
.download-section {
95+
display: flex;
96+
align-items: center;
97+
gap: 1rem;
98+
margin: 1.5rem 0;
99+
}
100+
101+
.download-btn {
102+
background-color: var(--primary-color, #22ae63);
103+
color: white;
104+
border: none;
105+
padding: 0.75rem 1.5rem;
106+
border-radius: 0.5rem;
107+
font-size: 1rem;
108+
font-weight: 600;
109+
cursor: pointer;
110+
transition: background-color 0.2s;
111+
}
112+
113+
.download-btn:hover {
114+
background-color: var(--primary-color-dark, #1a9655);
115+
}
116+
117+
.pipeline-count {
118+
color: var(--grey-600, #666);
119+
font-size: 0.9rem;
120+
}
121+
</style>

‎pages/regulatory.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
---
2+
title: Regulatory Stats
3+
sidebar_position: 10
4+
---
5+
6+
Pipeline statistics for regulatory and compliance reporting. This data is generated at build time from the nf-core statistics database and can be downloaded as a JSON export.
7+
8+
## Download
9+
10+
Download a JSON export of all pipeline statistics, including issue metrics and contributor data, for use in regulatory reports.
11+
12+
```sql pipeline_stats
13+
SELECT * FROM nfcore_db.regulatory_pipeline_stats
14+
```
15+
16+
```sql issue_stats
17+
SELECT * FROM nfcore_db.regulatory_issue_stats
18+
```
19+
20+
```sql contributor_stats
21+
SELECT * FROM nfcore_db.regulatory_contributor_stats
22+
```
23+
24+
<DownloadStatsJson
25+
pipelineStats={pipeline_stats}
26+
issueStats={issue_stats}
27+
contributorStats={contributor_stats}
28+
/>
29+
30+
The exported JSON contains the following data per pipeline:
31+
32+
- **Pipeline stats** — name, description, stars, forks, open issues, archive status, last release date
33+
- **Issue stats** — total/closed issue and PR counts, median time to close
34+
- **Contributor stats** — number of unique contributors

‎sources/nfcore_db/regulatory_contributor_stats.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
USE nf_core_stats_bot;
2+
3+
SELECT
4+
pipeline_name,
5+
COUNT(DISTINCT author) AS number_of_contributors
6+
FROM github.contributor_stats
7+
GROUP BY pipeline_name
8+
ORDER BY pipeline_name;

‎sources/nfcore_db/regulatory_issue_stats.sql‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
USE nf_core_stats_bot;
2+
3+
SELECT
4+
COALESCE(i.pipeline_name, c.pipeline_name, p.pipeline_name, pc.pipeline_name) AS pipeline_name,
5+
i.issue_count,
6+
c.closed_issue_count,
7+
c.median_seconds_to_issue_closed,
8+
p.pr_count,
9+
pc.closed_pr_count,
10+
pc.median_seconds_to_pr_closed
11+
FROM (
12+
SELECT
13+
pipeline_name,
14+
COUNT(issue_number) AS issue_count
15+
FROM github.issue_stats
16+
WHERE issue_type = 'issue'
17+
GROUP BY pipeline_name
18+
) AS i
19+
FULL JOIN (
20+
SELECT
21+
pipeline_name,
22+
COUNT(issue_number) AS closed_issue_count,
23+
MEDIAN(closed_wait_seconds) AS median_seconds_to_issue_closed
24+
FROM github.issue_stats
25+
WHERE issue_type = 'issue' AND state = 'closed'
26+
GROUP BY pipeline_name
27+
) AS c ON i.pipeline_name = c.pipeline_name
28+
FULL JOIN (
29+
SELECT
30+
pipeline_name,
31+
COUNT(issue_number) AS pr_count
32+
FROM github.issue_stats
33+
WHERE issue_type = 'pr'
34+
GROUP BY pipeline_name
35+
) AS p ON COALESCE(i.pipeline_name, c.pipeline_name) = p.pipeline_name
36+
FULL JOIN (
37+
SELECT
38+
pipeline_name,
39+
COUNT(issue_number) AS closed_pr_count,
40+
MEDIAN(closed_wait_seconds) AS median_seconds_to_pr_closed
41+
FROM github.issue_stats
42+
WHERE issue_type = 'pr' AND state = 'closed'
43+
GROUP BY pipeline_name
44+
) AS pc ON COALESCE(i.pipeline_name, c.pipeline_name, p.pipeline_name) = pc.pipeline_name
45+
ORDER BY pipeline_name;

‎sources/nfcore_db/regulatory_pipeline_stats.sql‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
USE nf_core_stats_bot;
2+
3+
SELECT
4+
name,
5+
description,
6+
stargazers_count,
7+
forks_count,
8+
open_issues_count,
9+
archived,
10+
last_release_date,
11+
category
12+
FROM github.nfcore_pipelines
13+
WHERE category = 'pipeline'
14+
ORDER BY name;

0 commit comments

Comments
 (0)