Skip to content

Commit 520a9ff

Browse files
tom-bywildtom-bywild
authored
feat: CRAN download statistics (#194)
* feat: add downloads * fix: ref manual usage * chore: bump version
1 parent 6990d90 commit 520a9ff

12 files changed

+324
-60
lines changed

web/app/data/package-insight-service.server.ts renamed to web/app/data/package-insight.service.server.ts

Lines changed: 42 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,16 @@ import { format, sub } from "date-fns";
22
import {
33
CranDownloadsResponse,
44
CranResponse,
5+
CranTopDownloadedPackagesRes,
6+
CranTrendingPackagesRes,
57
PackageDownloadTrend,
68
} from "./types";
9+
import { TopDownloadedPackagesRange } from "./package-insight.shape";
10+
import { slog } from "../modules/observability.server";
711

812
export class PackageInsightService {
13+
private static readonly CRAN_LOGS_URL = "https://cranlogs.r-pkg.org";
14+
915
/**
1016
* Get the downloads for a package in the last n days, starting
1117
* from today. The result is an array of objects with the number
@@ -80,6 +86,24 @@ export class PackageInsightService {
8086
return downloads;
8187
}
8288

89+
static async getTopDownloadedPackages(
90+
period: TopDownloadedPackagesRange,
91+
count: number,
92+
) {
93+
return this.fetchFromCRAN<CranTopDownloadedPackagesRes>(
94+
`/top/${period}/${count.toString()}`,
95+
);
96+
}
97+
98+
static async getTrendingPackages() {
99+
// Only for last week.
100+
const data = await this.fetchFromCRAN<CranTrendingPackagesRes>("/trending");
101+
return data.map((item) => ({
102+
...item,
103+
increase: `${new Number(item.increase).toFixed(0)}%`,
104+
}));
105+
}
106+
83107
/*
84108
* Private.
85109
*/
@@ -95,13 +119,26 @@ export class PackageInsightService {
95119
// the last day according to its point of reference (likely UTC).
96120
if (days === 1 && !from) {
97121
return this
98-
.fetchFromCRAN<CranDownloadsResponse>`/downloads/total/last-day/${name}`;
122+
.fetchLogsFromCRAN<CranDownloadsResponse>`/downloads/total/last-day/${name}`;
99123
}
100124

101125
const validFrom = from || new Date();
102126
const past = sub(validFrom, { days });
103127
return this
104-
.fetchFromCRAN<CranDownloadsResponse>`/downloads/total/${past}:${validFrom}/${name}`;
128+
.fetchLogsFromCRAN<CranDownloadsResponse>`/downloads/total/${past}:${validFrom}/${name}`;
129+
}
130+
131+
private static async fetchFromCRAN<R extends CranResponse = CranResponse>(
132+
url: string,
133+
): Promise<R> {
134+
return fetch(this.CRAN_LOGS_URL + url, {
135+
headers: { "Content-Type": "application/json" },
136+
})
137+
.then((response) => response.json())
138+
.catch((error) => {
139+
slog.error("Failed to fetch CRAN statistics", error);
140+
return undefined;
141+
});
105142
}
106143

107144
/**
@@ -112,18 +149,12 @@ export class PackageInsightService {
112149
* @param params
113150
* @returns
114151
*/
115-
private static async fetchFromCRAN<R extends CranResponse = CranResponse>(
152+
private static async fetchLogsFromCRAN<R extends CranResponse = CranResponse>(
116153
template: TemplateStringsArray,
117154
...params: (string | Date)[]
118155
): Promise<R> {
119156
const url = this.getCRANLogsUrl(template, ...params);
120-
return fetch(url, {
121-
headers: {
122-
"Content-Type": "application/json",
123-
},
124-
})
125-
.then((response) => response.json())
126-
.catch(() => undefined);
157+
return this.fetchFromCRAN<R>(url);
127158
}
128159

129160
/**
@@ -152,7 +183,7 @@ export class PackageInsightService {
152183
return format(part, "yyyy-MM-dd");
153184
});
154185

155-
return "https://cranlogs.r-pkg.org" + stringified.join("");
186+
return stringified.join("");
156187
}
157188

158189
private static format1kDelimiter(total: number) {

‎web/app/data/package-insight.shape.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { z } from "zod";
2+
3+
export const topDownloadedPackagesRangeSchema = z.union([
4+
z.literal("last-day"),
5+
z.literal("last-week"),
6+
z.literal("last-month"),
7+
]);
8+
9+
export type TopDownloadedPackagesRange = z.infer<
10+
typeof topDownloadedPackagesRangeSchema
11+
>;

‎web/app/data/types.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,26 @@ export type CranDownloadsResponse = Array<{
120120
package: string;
121121
}>;
122122

123-
export type CranResponse = CranDownloadsResponse;
123+
export type CranTopDownloadedPackagesRes = {
124+
start: string; // e.g. "2015-05-01T00:00:00.000Z";
125+
end: string; // e.g. "2015-05-01T00:00:00.000Z";
126+
downloads: Array<{ package: string; downloads: number }>;
127+
};
128+
129+
/**
130+
* Trending packages are the ones that were downloaded at least 1000 times during last week,
131+
* and that substantially increased their download counts, compared to the average weekly downloads in the previous 24 weeks.
132+
* The percentage of increase is also shown in the output.
133+
*/
134+
export type CranTrendingPackagesRes = Array<{
135+
package: string;
136+
increase: number;
137+
}>;
138+
139+
export type CranResponse =
140+
| CranDownloadsResponse
141+
| CranTopDownloadedPackagesRes
142+
| CranTrendingPackagesRes;
124143

125144
export type PackageDownloadTrend = {
126145
trend: string;

‎web/app/modules/anchors.tsx‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,12 @@ export function AnchorLink(props: PropsWithChildren<{ fragment: string }>) {
5757
}
5858

5959
AnchorLink.displayName = "AnchorLink";
60+
61+
export function composeAnchorItems(
62+
anchors: string[],
63+
): Array<{ name: string; slug: string }> {
64+
return anchors.map((anchor) => ({
65+
name: anchor,
66+
slug: encodeURIComponent(anchor.toLowerCase().replaceAll(" ", "-")),
67+
}));
68+
}

‎web/app/modules/provided-by-label.tsx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { RiExternalLinkLine } from "@remixicon/react";
2+
import { ExternalLink } from "./external-link";
3+
4+
export function DataProvidedByCRANLabel() {
5+
return (
6+
<p className="text-gray-dim mt-16 text-right text-xs">
7+
Data provided by{" "}
8+
<ExternalLink
9+
href="https://github.com/r-hub/cranlogs.app"
10+
className="inline-flex items-center gap-1 underline underline-offset-4"
11+
>
12+
cranlogs
13+
<RiExternalLinkLine size={10} className="text-gray-dim" />
14+
</ExternalLink>
15+
</p>
16+
);
17+
}

‎web/app/root.tsx‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,23 @@ import "./tailwind.css";
1717
import { ENV } from "./data/env";
1818
import { BASE_URL } from "./modules/app";
1919
import { useEffect } from "react";
20-
import { randomInt } from "es-toolkit";
2120

22-
export const meta: MetaFunction = () => {
21+
export const meta: MetaFunction = ({ location }) => {
22+
// Pseudo-randomly select a cover image based on the length
23+
// of the current path (= stable index per site) and add
24+
// the current day of the week as a seed so that the cover
25+
// changes daily.
26+
const dayOfWeek = new Date().getDay();
27+
const coverIndex = ((location.pathname.length + dayOfWeek) & 9) + 1;
28+
2329
return [
2430
{ title: "CRAN/E" },
2531
{ name: "description", content: "The R package search engine, enhanced" },
2632
{ property: "og:type", content: "website" },
2733
{ property: "og:url", content: BASE_URL },
2834
{
2935
property: "og:image",
30-
content: BASE_URL + `/images/og/cover-${randomInt(9) + 1}.jpg`,
36+
content: BASE_URL + `/images/og/cover-${coverIndex}.jpg`,
3137
},
3238
];
3339
};

‎web/app/routes/$slug[.xml]._index.tsx‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ export async function loader(props: LoaderFunctionArgs) {
5252
lastmod: today,
5353
changefreq: "daily",
5454
})}
55+
${composeUrlElement({
56+
path: `/statistic/package`,
57+
lastmod: today,
58+
changefreq: "daily",
59+
})}
5560
</urlset>`.trim(),
5661
{
5762
headers: {

‎web/app/routes/_page.package.$packageId.tsx‎

Lines changed: 23 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,10 @@ import {
3232
} from "../modules/meta";
3333
import { BASE_URL } from "../modules/app";
3434
import { uniq } from "es-toolkit";
35-
import { PackageInsightService } from "../data/package-insight-service.server";
35+
import { PackageInsightService } from "../data/package-insight.service.server";
3636
import { slog } from "../modules/observability.server";
3737
import clsx from "clsx";
38+
import { DataProvidedByCRANLabel } from "../modules/provided-by-label";
3839

3940
const PackageDependencySearch = lazy(() =>
4041
import("../modules/package-dependency-search").then((mod) => ({
@@ -269,22 +270,26 @@ function AboveTheFoldSection(props: { item: Pkg; lastRelease: string }) {
269270
</ExternalLinkPill>
270271
</li>
271272
) : null}
272-
<li>
273-
<ExternalLinkPill
274-
href={item.cran_checks.link}
275-
icon={<RiExternalLinkLine size={18} />}
276-
>
277-
{item.cran_checks.label}
278-
</ExternalLinkPill>
279-
</li>
280-
<li>
281-
<ExternalLinkPill
282-
href={item.reference_manual.link}
283-
icon={<RiFilePdf2Line size={18} />}
284-
>
285-
{item.reference_manual.label}
286-
</ExternalLinkPill>
287-
</li>
273+
{item.cran_checks ? (
274+
<li>
275+
<ExternalLinkPill
276+
href={item.cran_checks.link}
277+
icon={<RiExternalLinkLine size={18} />}
278+
>
279+
{item.cran_checks.label}
280+
</ExternalLinkPill>
281+
</li>
282+
) : null}
283+
{item.reference_manual ? (
284+
<li>
285+
<ExternalLinkPill
286+
href={item.reference_manual.link}
287+
icon={<RiFilePdf2Line size={18} />}
288+
>
289+
{item.reference_manual.label}
290+
</ExternalLinkPill>
291+
</li>
292+
) : null}
288293
</ul>
289294
<ul className="flex flex-wrap items-start gap-4">
290295
<li>
@@ -525,16 +530,7 @@ function InsightsPageContentSection(props: {
525530
<p className="text-gray-dim">No downloads available</p>
526531
)}
527532

528-
<p className="text-gray-dim mt-16 text-right text-xs">
529-
Data provided by{" "}
530-
<ExternalLink
531-
href="https://github.com/r-hub/cranlogs.app"
532-
className="inline-flex items-center gap-1 underline underline-offset-4"
533-
>
534-
cranlogs
535-
<RiExternalLinkLine size={10} className="text-gray-dim" />
536-
</ExternalLink>
537-
</p>
533+
<DataProvidedByCRANLabel />
538534
</PageContentSection>
539535
);
540536
}

‎web/app/routes/_page.statistic._index.tsx‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
import { Link } from "@remix-run/react";
2-
import { Anchors, AnchorLink } from "../modules/anchors";
2+
import { Anchors, AnchorLink, composeAnchorItems } from "../modules/anchors";
33
import { PageContent } from "../modules/page-content";
44
import { PageContentSection } from "../modules/page-content-section";
55
import { InfoCard } from "../modules/info-card";
66
import { Header } from "../modules/header";
77
import { Tag } from "../modules/tag";
88
import { mergeMeta } from "../modules/meta";
9+
import { Separator } from "../modules/separator";
910

10-
const anchors = ["Site usage"];
11+
const anchors = composeAnchorItems(["Site usage", "CRAN data"]);
1112

1213
export const meta = mergeMeta(() => {
1314
return [
@@ -27,9 +28,9 @@ export default function StatisticsOverviewPage() {
2728
/>
2829

2930
<Anchors>
30-
{anchors.map((anchor) => (
31-
<AnchorLink key={anchor} fragment={anchor.toLowerCase()}>
32-
{anchor}
31+
{anchors.map(({ name, slug }) => (
32+
<AnchorLink key={slug} fragment={slug}>
33+
{name}
3334
</AnchorLink>
3435
))}
3536
</Anchors>
@@ -38,11 +39,11 @@ export default function StatisticsOverviewPage() {
3839
<PageContentSection
3940
headline="Site usage"
4041
subline="See what packages and authors are trending on CRAN/E"
41-
fragment="site-usage"
42+
fragment={"site-usage"}
4243
>
4344
<ul className="grid grid-cols-2 gap-4 md:grid-cols-3 lg:grid-cols-4">
4445
<li>
45-
<Link to="/statistic/crane/page-visits">
46+
<Link prefetch="intent" to="/statistic/crane/page-visits">
4647
<InfoCard variant="bronze" icon="internal" className="min-h-60">
4748
<div className="space-y-2">
4849
<h3>Page trends</h3>
@@ -55,6 +56,29 @@ export default function StatisticsOverviewPage() {
5556
</li>
5657
</ul>
5758
</PageContentSection>
59+
60+
<Separator />
61+
62+
<PageContentSection
63+
headline="CRAN data"
64+
subline="Get insights into CRAN data"
65+
fragment={"cran-data"}
66+
>
67+
<ul className="grid grid-cols-2 gap-4 md:grid-cols-3 lg:grid-cols-4">
68+
<li>
69+
<Link prefetch="intent" to="/statistic/packages">
70+
<InfoCard variant="bronze" icon="internal" className="min-h-60">
71+
<div className="space-y-2">
72+
<h3>Package downloads</h3>
73+
<p className="text-gray-dim">
74+
See what packages are trending on CRAN/E.
75+
</p>
76+
</div>
77+
</InfoCard>
78+
</Link>
79+
</li>
80+
</ul>
81+
</PageContentSection>
5882
</PageContent>
5983
</>
6084
);

0 commit comments

Comments
 (0)