Skip to content

Commit 432facf

Browse files
authored
Merge pull request #595 from kleros/fix/get-rid-of-unnecessary-extra-github-raw-usercontent
refactor: read snapshots from a manifest instead of scraping GitHub raw
2 parents e8c5a42 + 5cb6f0b commit 432facf

8 files changed

Lines changed: 318 additions & 68 deletions

File tree

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
# Production
77
/build/
88

9+
# Generated by scripts/emit-snapshots.js from src/assets/snapshots.json
10+
/public/snapshots.json
11+
912
# Logs
1013
/yarn-debug.log*
1114
/yarn-error.log*

.prettierignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# Ignore artifacts:
22
build
33
coverage
4+
public/snapshots.json
45

netlify.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@
22
YARN_FLAGS='--pure-lockfile'
33
NODE_VERSION='16.20.2'
44

5+
# snapshots.json is the published list of monthly reward snapshot CIDs.
6+
# It is read by various consumers, so it needs CORS.
7+
[[headers]]
8+
for = "/snapshots.json"
9+
[headers.values]
10+
Access-Control-Allow-Origin = "*"
11+
512
[[redirects]]
613
from = "/*"
714
to = "/index.html"

package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,14 @@
1414
"license": "MIT",
1515
"private": true,
1616
"scripts": {
17-
"prestart": "run-s build:theme",
17+
"prestart": "run-s build:theme build:snapshots",
1818
"start": "cross-env BROWSER=none react-app-rewired start",
1919
"test": "react-app-rewired test --watchAll=false",
20-
"prebuild": "run-s build:theme",
20+
"prebuild": "run-s build:theme build:snapshots",
2121
"build": "react-app-rewired build",
2222
"build:analyze": "source-map-explorer build/static/js/main.*",
2323
"build:theme": "lessc --js ./src/components/theme.less ./src/components/theme.css",
24+
"build:snapshots": "node scripts/emit-snapshots.js",
2425
"fix": "run-p -c fix:*",
2526
"fix:js": "eslint --fix '**/*.{js,jsx}'",
2627
"fix:css": "stylelint --fix '**/*.{less,css}'",

public/staking-rewards.html

Lines changed: 25 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -70,27 +70,19 @@
7070
const { useState, useEffect, useMemo } = React;
7171

7272
const IPFS = "https://cdn.kleros.link/ipfs/";
73-
const CLAIM_MODAL_URL = "https://raw.githubusercontent.com/kleros/court/master/src/components/claim-modal.js";
74-
75-
function parseSnapshotsFromSource(source) {
76-
const mainnetMatch = source.match(/\b1:\s*\{[^}]*?snapshots:\s*\[([\s\S]*?)\]/);
77-
const gnosisMatch = source.match(/\b100:\s*\{[^}]*?snapshots:\s*\[([\s\S]*?)\]/);
78-
function extractPaths(block) {
79-
if (!block) return [];
80-
const paths = [];
81-
const re = /"([^"]+\.json)"/g;
82-
let m;
83-
while ((m = re.exec(block)) !== null) paths.push(m[1]);
84-
return paths;
85-
}
86-
function toSnapshot(path) {
87-
const filename = path.split("/").pop();
88-
const dateMatch = filename.match(/(\d{4}-\d{2})/);
89-
return { path, label: dateMatch ? dateMatch[1] : "2021-01 & 02" };
90-
}
73+
// Generated at build time from src/assets/snapshots.json by scripts/emit-snapshots.js.
74+
const SNAPSHOTS_URL = "./snapshots.json";
75+
76+
function toSnapshot(path) {
77+
const filename = path.split("/").pop();
78+
const dateMatch = filename.match(/(\d{4}-\d{2})/);
79+
return { path, label: dateMatch ? dateMatch[1] : "2021-01 & 02" };
80+
}
81+
82+
function parseSnapshots(snapshotsByChainId) {
9183
return {
92-
mainnet: extractPaths(mainnetMatch?.[1]).map(toSnapshot),
93-
gnosis: extractPaths(gnosisMatch?.[1]).map(toSnapshot),
84+
mainnet: (snapshotsByChainId?.["1"] ?? []).map(toSnapshot),
85+
gnosis: (snapshotsByChainId?.["100"] ?? []).map(toSnapshot),
9486
};
9587
}
9688

@@ -456,8 +448,8 @@
456448
const [errors, setErrors] = useState([]);
457449
const [monthData, setMonthData] = useState(null);
458450
const [grandTotals, setGrandTotals] = useState(null);
459-
const [activeTab, setActiveTab] = useState("Summary");
460-
const [tabList, setTabList] = useState(["Summary"]);
451+
const [activeTab, setActiveTab] = useState("Monthly Totals");
452+
const [tabList, setTabList] = useState(["Monthly Totals"]);
461453
const [sourceError, setSourceError] = useState(null);
462454

463455
// Auto-start everything on mount
@@ -469,15 +461,14 @@
469461
setPhase("loading");
470462
setErrors([]);
471463

472-
// Step 1: fetch snapshot list from GitHub
464+
// Step 1: load the snapshot list
473465
let snapshots;
474466
try {
475-
const r = await fetch(CLAIM_MODAL_URL);
467+
const r = await fetch(SNAPSHOTS_URL);
476468
if (!r.ok) throw new Error("HTTP " + r.status);
477-
const source = await r.text();
478-
snapshots = parseSnapshotsFromSource(source);
469+
snapshots = parseSnapshots(await r.json());
479470
if (snapshots.mainnet.length === 0 && snapshots.gnosis.length === 0) {
480-
throw new Error("Could not parse any snapshots from claim-modal.js");
471+
throw new Error("snapshots.json contained no snapshots");
481472
}
482473
} catch (e) {
483474
setSourceError(e.message);
@@ -536,8 +527,8 @@
536527

537528
setMonthData(md);
538529
setGrandTotals(gt);
539-
setTabList(["Summary", "Monthly Totals", ...sorted]);
540-
setActiveTab("Summary");
530+
setTabList(["Monthly Totals", "Summary", ...sorted]);
531+
setActiveTab("Monthly Totals");
541532
setErrors(errs);
542533
setPhase("done");
543534
}
@@ -577,6 +568,9 @@
577568
if (!grandTotals || !monthData) return;
578569
const wb = XLSX.utils.book_new();
579570

571+
const monthlyTotalRows = sortedMonths.map((label) => monthTotalRow(label, monthData[label]));
572+
addSheet(wb, "Monthly Totals", monthlyTotalRows, [14, 18, 18, 18]);
573+
580574
const summaryRows = Object.entries(grandTotals)
581575
.map(([addr, t]) => ({
582576
Address: addr,
@@ -587,9 +581,6 @@
587581
.sort((a, b) => b["Grand Total PNK"] - a["Grand Total PNK"]);
588582
addSheet(wb, "Summary", summaryRows, [44, 20, 20, 20]);
589583

590-
const monthlyTotalRows = sortedMonths.map((label) => monthTotalRow(label, monthData[label]));
591-
addSheet(wb, "Monthly Totals", monthlyTotalRows, [14, 18, 18, 18]);
592-
593584
for (const label of sortedMonths) {
594585
const rows = monthWalletRows(monthData[label]);
595586
if (rows.length === 0) continue;
@@ -694,7 +685,7 @@
694685
fontWeight: 600,
695686
}}
696687
>
697-
{phase === "loading" ? "Fetching snapshot list from GitHub..." : "Fetching snapshots from IPFS..."}
688+
{phase === "loading" ? "Loading snapshot list..." : "Fetching snapshots from IPFS..."}
698689
</div>
699690
{phase === "fetching" && (
700691
<div style={{ width: 400, maxWidth: "90%" }}>
@@ -847,7 +838,7 @@
847838
gap: 8,
848839
}}
849840
>
850-
<span>Snapshot hashes fetched live from kleros/court master branch</span>
841+
<span>Reward data fetched from IPFS snapshots published by Kleros Court</span>
851842
<span>Tip: Use "Download XLSX" to import into Google Sheets or Excel</span>
852843
</div>
853844
</div>

scripts/emit-snapshots.js

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env node
2+
/**
3+
* 1. Verifies src/assets/snapshots.json still matches the `snapshots` arrays in
4+
* src/components/claim-modal.js, and fails the build if they have drifted.
5+
* 2. Copies the manifest to public/snapshots.json so the standalone
6+
* staking-rewards.html page can read it same-origin (files under public/ are
7+
* copied verbatim by the build and cannot import from src/).
8+
*
9+
* The arrays in claim-modal.js are duplicated here on purpose, and only
10+
* temporarily: raw.githubusercontent.com/kleros/court/master/src/components/claim-modal.js
11+
* is scraped by klerosboard and proof-of-humanity-v2-web to locate the monthly
12+
* reward snapshots. Once those repos point at snapshots.json instead, delete the
13+
* arrays from claim-modal.js, have it import this manifest, and drop assertParity().
14+
*
15+
* The array index of each snapshot is the on-chain `week` argument to
16+
* MerkleRedeem.claimWeek, so order and length must never change.
17+
*/
18+
const fs = require("fs");
19+
const path = require("path");
20+
21+
const root = path.join(__dirname, "..");
22+
const source = path.join(root, "src", "assets", "snapshots.json");
23+
const claimModal = path.join(root, "src", "components", "claim-modal.js");
24+
const destination = path.join(root, "public", "snapshots.json");
25+
26+
// Every chain the reward snapshots cover, declared once. To support a new chain, add one entry
27+
// here (with its filename prefix); CHAIN_IDS and the prefix lookup derive from it, so the set of
28+
// chains and their prefixes can never drift apart.
29+
const CHAINS = {
30+
1: { filenamePrefix: "snapshot-" },
31+
100: { filenamePrefix: "xdai-snapshot-" },
32+
};
33+
const CHAIN_IDS = Object.keys(CHAINS);
34+
const ENTRY = /^[A-Za-z0-9]+\/[A-Za-z0-9._-]+\.json$/;
35+
36+
// rewards.js derives the IPFS url, the month and the chain from these strings alone, so a
37+
// malformed entry there is a wrong reward figure at runtime rather than an error.
38+
function assertEntriesAreWellFormed(entries, chainId) {
39+
for (const entry of entries) {
40+
if (!ENTRY.test(entry)) {
41+
throw new Error(`snapshots.json: chain ${chainId} entry "${entry}" is not of the form "<cid>/<filename>.json"`);
42+
}
43+
const filename = entry.split("/")[1];
44+
const { filenamePrefix } = CHAINS[chainId];
45+
if (!filename.startsWith(filenamePrefix)) {
46+
throw new Error(
47+
`snapshots.json: chain ${chainId} entry "${entry}" must start with "${filenamePrefix}" ` +
48+
`— rewards.js tells the chains apart by that prefix.`
49+
);
50+
}
51+
}
52+
}
53+
54+
// A month published for one chain but not the other would make getSnapshotRewardAndSupply()
55+
// sum an incomplete reward set. Snapshots are appended, so only the newest month can be partial.
56+
function assertNewestMonthIsOnEveryChain(snapshots) {
57+
const newestMonth = (chainId) => {
58+
const entries = snapshots[chainId];
59+
const newest = entries[entries.length - 1];
60+
const month = (newest.match(/(\d{4}-\d{2})/) || [])[1];
61+
if (!month) {
62+
throw new Error(
63+
`snapshots.json: chain ${chainId}'s newest entry "${newest}" has no YYYY-MM, ` +
64+
`so it can't be checked against the other chains.`
65+
);
66+
}
67+
return month;
68+
};
69+
const [referenceChain, ...otherChains] = CHAIN_IDS;
70+
const referenceMonth = newestMonth(referenceChain);
71+
for (const chainId of otherChains) {
72+
const month = newestMonth(chainId);
73+
if (month !== referenceMonth) {
74+
throw new Error(
75+
`snapshots.json: newest month differs per chain ` +
76+
`(chain ${referenceChain} ${referenceMonth}, chain ${chainId} ${month}). ` +
77+
`Add the missing month, or the monthly reward total will be missing a chain.`
78+
);
79+
}
80+
}
81+
}
82+
83+
// Read the `snapshots: [...]` array literal for a chain out of claim-modal.js. This is coupled to
84+
// that file's current indentation; reformatting it will throw here rather than pass silently.
85+
function readClaimModalSnapshots(code, chainId) {
86+
const block = code.match(new RegExp(`\\n ${chainId}: \\{[\\s\\S]*?snapshots: \\[([\\s\\S]*?)\\n \\]`));
87+
if (!block) throw new Error(`claim-modal.js: could not find the snapshots array for chain ${chainId}`);
88+
return Array.from(block[1].matchAll(/"([^"]+)"/g), (match) => match[1]);
89+
}
90+
91+
function assertParity(snapshots) {
92+
const code = fs.readFileSync(claimModal, "utf8");
93+
for (const chainId of CHAIN_IDS) {
94+
const expected = readClaimModalSnapshots(code, chainId);
95+
const actual = snapshots[chainId];
96+
const drifted = expected.length !== actual.length || expected.some((entry, i) => entry !== actual[i]);
97+
if (drifted) {
98+
throw new Error(
99+
`snapshots.json and claim-modal.js disagree for chain ${chainId} ` +
100+
`(${actual.length} vs ${expected.length} entries). Update both, keeping the order identical.`
101+
);
102+
}
103+
}
104+
}
105+
106+
const snapshots = JSON.parse(fs.readFileSync(source, "utf8"));
107+
108+
// Everything below is verified per chain in CHAIN_IDS, but the whole object is written out. Reject
109+
// any other chain so nothing reaches public/snapshots.json without being validated first.
110+
const unverifiedChains = Object.keys(snapshots).filter((chainId) => !CHAIN_IDS.includes(chainId));
111+
if (unverifiedChains.length > 0) {
112+
throw new Error(
113+
`snapshots.json: chain(s) ${unverifiedChains.join(", ")} are not verified. Add them to CHAINS ` +
114+
`(and claim-modal.js) so they are validated before being published.`
115+
);
116+
}
117+
118+
for (const chainId of CHAIN_IDS) {
119+
if (!Array.isArray(snapshots[chainId]) || snapshots[chainId].length === 0) {
120+
throw new Error(`snapshots.json is missing snapshots for chain ${chainId}`);
121+
}
122+
assertEntriesAreWellFormed(snapshots[chainId], chainId);
123+
}
124+
assertNewestMonthIsOnEveryChain(snapshots);
125+
assertParity(snapshots);
126+
127+
fs.writeFileSync(destination, JSON.stringify(snapshots) + "\n");

0 commit comments

Comments
 (0)