Skip to content

Commit 2d94fb6

Browse files
ReynardoEWReynardoEW
committed
add ethereum, add dune utils, add interactive chart viewer
1 parent 8aef959 commit 2d94fb6

File tree

8 files changed

+536
-2
lines changed

8 files changed

+536
-2
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22
**/**/*.js
33
/utils/testCharts
44
result.txt
5-
/utils/result.png
5+
/utils/result.png
6+
.env

‎package.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,13 @@
33
"test": "mkdir -p utils/testCharts && ts-node utils/test.ts"
44
},
55
"dependencies": {
6-
"@defillama/sdk": "^5.0.101",
6+
"@defillama/sdk": "^5.0.126",
7+
"async-retry": "^1.3.3",
78
"axios": "^1.3.4",
9+
"chart.js": "^3.5.1",
810
"chartjs-node-canvas": "^4.1.6",
911
"dayjs": "^1.11.7",
12+
"dotenv": "^16.5.0",
1013
"form-data": "^4.0.0",
1114
"graphql-request": "^5.2.0",
1215
"node-fetch": "2.6.6"

‎protocols/ethereum.ts

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import { manualCliff, manualLinear } from "../adapters/manual";
2+
import { CliffAdapterResult, LinearAdapterResult, Protocol } from "../types/adapters";
3+
import { GAS_TOKEN } from "../utils/constants";
4+
import { queryDune } from "../utils/dune";
5+
import { periodToSeconds } from "../utils/time";
6+
7+
const chain = "ethereum";
8+
const start = 1438300800;
9+
10+
const genesis = 1438269988; // block 1 5eth reward
11+
const byzantiumFork = 1508131331; // block 4370000 3eth reward
12+
const constantinopleFork = 1551383524; // block 7280000 2eth reward
13+
const merge = 1663224162; // block 15537393 0eth reward
14+
15+
interface BurnDataPoint {
16+
eth_burn: number;
17+
timestamp: number;
18+
}
19+
20+
const burnData = async (type: 'pos' | 'pow'): Promise<LinearAdapterResult[]> => {
21+
const result: LinearAdapterResult[] = [];
22+
const burnData = await queryDune("5041563")
23+
24+
// Filter data based on type
25+
let filteredData = burnData;
26+
if (type === 'pow') {
27+
// Pre-merge data
28+
filteredData = burnData.filter((d: BurnDataPoint) => d.timestamp < merge);
29+
} else if (type === 'pos') {
30+
// Post-merge data
31+
filteredData = burnData.filter((d: BurnDataPoint) => d.timestamp >= merge);
32+
}
33+
34+
for (let i = 0; i < filteredData.length - 1; i++) {
35+
result.push({
36+
type: "linear",
37+
start: filteredData[i + 1].timestamp,
38+
end: filteredData[i].timestamp,
39+
amount: -filteredData[i].eth_burn
40+
});
41+
}
42+
43+
return result;
44+
}
45+
46+
const foundationOutflow = async (): Promise<CliffAdapterResult[]> => {
47+
const result: CliffAdapterResult[] = [];
48+
const foundationOutflowData = await queryDune("5041975")
49+
for (let i = 0; i < foundationOutflowData.length; i++) {
50+
result.push({
51+
type: "cliff",
52+
start: foundationOutflowData[i].start,
53+
amount: foundationOutflowData[i].amount,
54+
})
55+
}
56+
57+
return result;
58+
}
59+
60+
const stakingRewards = async (): Promise<LinearAdapterResult[]> => {
61+
const result: LinearAdapterResult[] = [];
62+
const issuanceData = await queryDune("5041721")
63+
64+
for (let i = 0; i < issuanceData.length - 1; i++) {
65+
result.push({
66+
type: "linear",
67+
start: issuanceData[i + 1].timestamp,
68+
end: issuanceData[i].timestamp,
69+
amount: issuanceData[i].eth_issued
70+
})
71+
}
72+
return result;
73+
}
74+
75+
76+
const ethereum: Protocol = {
77+
"Crowd Sale": manualCliff(start, 6e7),
78+
"Early Contributors": manualLinear(
79+
start,
80+
start + periodToSeconds.year * 4,
81+
6e6,
82+
),
83+
"Ethereum Foundation": foundationOutflow,
84+
"Issuance": [
85+
manualLinear(genesis, byzantiumFork, 4369999 * 5),
86+
manualLinear(byzantiumFork, constantinopleFork, 2910000 * 3),
87+
manualLinear(constantinopleFork, merge, 8257393 * 2),
88+
stakingRewards,
89+
burnData
90+
],
91+
meta: {
92+
token: `${chain}:${GAS_TOKEN}`,
93+
notes: [
94+
`Information on the Early Contributor vesting schedule structure could not be found, here we have assumed it as linearly unlocked over 4 years.`,
95+
`The Ethereum Foundation supply is assumed to be unlocked when there's an outflow from EthDev address.`,
96+
`Issuance is combination of PoW and PoS issuance, with EIP-1559 burning included`,
97+
`Uncle Block rewards are not included in the issuance calculation.`,
98+
],
99+
sources: [
100+
"https://dune.com/21co/ethereum-key-metrics",
101+
"https://www.galaxy.com/insights/research/breakdown-of-ethereum-supply-distribution-since-genesis/",
102+
"https://fastercapital.com/topics/common-token-vesting-strategies-for-airdrop-cryptocurrency.html",
103+
],
104+
protocolIds: ["4488"],
105+
},
106+
categories: {
107+
farming: ["Issuance"],
108+
insiders: ["Ethereum Foundation"],
109+
publicSale: ["Crowd Sale"],
110+
},
111+
};
112+
export default ethereum;

‎utils/dune.ts

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
import { httpGet, httpPost } from "./fetchURL";
2+
import { getEnv } from "./env";
3+
const plimit = require('p-limit');
4+
const limit = plimit(1);
5+
6+
const isRestrictedMode = getEnv('DUNE_RESTRICTED_MODE') === 'true'
7+
const API_KEYS = getEnv('DUNE_API_KEYS')?.split(',') ?? ["L0URsn5vwgyrWbBpQo9yS1E3C1DBJpZh"]
8+
let API_KEY_INDEX = 0;
9+
10+
const NOW_TIMESTAMP = Math.trunc((Date.now()) / 1000)
11+
12+
const getLatestData = async (queryId: string) => {
13+
checkCanRunDuneQuery()
14+
15+
const url = `https://api.dune.com/api/v1/query/${queryId}/results`
16+
try {
17+
const latest_result = (await limit(() => httpGet(url, {
18+
headers: {
19+
"x-dune-api-key": API_KEYS[API_KEY_INDEX]
20+
}
21+
})))
22+
const submitted_at = latest_result.submitted_at
23+
const submitted_at_timestamp = Math.trunc(new Date(submitted_at).getTime() / 1000)
24+
const diff = NOW_TIMESTAMP - submitted_at_timestamp
25+
if (diff < 60 * 60 * 3) {
26+
return latest_result.result.rows
27+
}
28+
return undefined
29+
} catch (e: any) {
30+
throw e;
31+
}
32+
}
33+
34+
35+
async function randomDelay() {
36+
const delay = Math.floor(Math.random() * 5) + 2
37+
return new Promise((resolve) => setTimeout(resolve, delay * 1000))
38+
}
39+
40+
const inquiryStatus = async (execution_id: string, queryId: string) => {
41+
checkCanRunDuneQuery()
42+
43+
let _status = undefined;
44+
do {
45+
try {
46+
_status = (await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/status`, {
47+
headers: {
48+
"x-dune-api-key": API_KEYS[API_KEY_INDEX]
49+
}
50+
}))).state
51+
if (['QUERY_STATE_PENDING', 'QUERY_STATE_EXECUTING'].includes(_status)) {
52+
console.info(`waiting for query id ${queryId} to complete...`)
53+
await randomDelay() // 1 - 4s
54+
}
55+
} catch (e: any) {
56+
throw e;
57+
}
58+
} while (_status !== 'QUERY_STATE_COMPLETED' && _status !== 'QUERY_STATE_FAILED');
59+
return _status
60+
}
61+
62+
const submitQuery = async (queryId: string, query_parameters = {}) => {
63+
checkCanRunDuneQuery()
64+
65+
let query: undefined | any = undefined
66+
try {
67+
query = await limit(() => httpPost(`https://api.dune.com/api/v1/query/${queryId}/execute`, { query_parameters }, {
68+
headers: {
69+
"x-dune-api-key": API_KEYS[API_KEY_INDEX],
70+
'Content-Type': 'application/json'
71+
}
72+
}))
73+
if (query?.execution_id) {
74+
return query?.execution_id
75+
} else {
76+
throw new Error("error query data: " + query)
77+
}
78+
} catch (e: any) {
79+
throw e;
80+
}
81+
}
82+
83+
84+
export const queryDune = async (queryId: string, query_parameters: any = {}) => {
85+
checkCanRunDuneQuery()
86+
87+
if (Object.keys(query_parameters).length === 0) {
88+
const latest_result = await getLatestData(queryId)
89+
if (latest_result !== undefined) return latest_result
90+
}
91+
const execution_id = await submitQuery(queryId, query_parameters)
92+
const _status = await inquiryStatus(execution_id, queryId)
93+
if (_status === 'QUERY_STATE_COMPLETED') {
94+
const API_KEY = API_KEYS[API_KEY_INDEX]
95+
try {
96+
const queryStatus = await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/results?limit=100000`, {
97+
headers: {
98+
"x-dune-api-key": API_KEY
99+
}
100+
}))
101+
return queryStatus.result.rows
102+
} catch (e: any) {
103+
throw e;
104+
}
105+
} else if(_status === "QUERY_STATE_FAILED"){
106+
if(query_parameters.fullQuery){
107+
console.log(`Dune query: ${query_parameters.fullQuery}`)
108+
} else {
109+
console.log("Dune parameters", query_parameters)
110+
}
111+
throw new Error(`Dune query failed: ${queryId}`)
112+
}
113+
}
114+
115+
const tableName = {
116+
bsc: "bnb",
117+
ethereum: "ethereum",
118+
base: "base",
119+
avax: "avalanche_c"
120+
} as any
121+
122+
export const queryDuneSql = (options: any, query: string) => {
123+
checkCanRunDuneQuery()
124+
125+
return queryDune("3996608", {
126+
fullQuery: query.replace("CHAIN", tableName[options.chain] ?? options.chain).split("TIME_RANGE").join(`block_time >= from_unixtime(${options.startTimestamp})
127+
AND block_time <= from_unixtime(${options.endTimestamp})`)
128+
})
129+
}
130+
131+
export function checkCanRunDuneQuery() {
132+
if (!isRestrictedMode) return;
133+
const currentHour = new Date().getUTCHours();
134+
if (currentHour >= 1 && currentHour <= 3) return; // 1am - 3am - any time other than this, throw error
135+
throw new Error(`Current hour is ${currentHour}. In restricted mode, can run dune queries only between 1am - 3am UTC`);
136+
}

‎utils/env.ts

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
const BOOL_KEYS = [
2+
''
3+
]
4+
5+
const DEFAULTS: any = {
6+
BITLAYER_RPC: "https://rpc.bitlayer.org,https://rpc.ankr.com/bitlayer,https://rpc.bitlayer-rpc.com,https://rpc-bitlayer.rockx.com",
7+
}
8+
9+
export const ENV_KEYS = new Set([
10+
...BOOL_KEYS,
11+
...Object.keys(DEFAULTS),
12+
'PANCAKESWAP_OPBNB_SUBGRAPH',
13+
'INDEXA_DB',
14+
'FLIPSIDE_API_KEY',
15+
'DUNE_API_KEYS',
16+
'DUNE_RESTRICTED_MODE',
17+
'ALLIUM_API_KEY',
18+
'BIT_QUERY_API_KEY',
19+
'SMARDEX_SUBGRAPH_API_KEY',
20+
'PROD_VYBE_API_KEY',
21+
'PERENNIAL_V2_SUBGRAPH_API_KEY',
22+
'LEVANA_API_KEY',
23+
'ZEROx_API_KEY',
24+
'ZEROX_API_KEY',
25+
'AGGREGATOR_0X_API_KEY',
26+
'SUI_RPC',
27+
'OKX_API_KEY',
28+
'ALCHEMIX_KEY',
29+
'ALCHEMIX_SECRET',
30+
'FLIPSIDE_RESTRICTED_MODE',
31+
'STARBASE_API_KEY',
32+
'ENSO_API_KEY',
33+
])
34+
35+
// This is done to support both ZEROx_API_KEY and ZEROX_API_KEY
36+
if (!process.env.ZEROX_API_KEY) process.env.ZEROX_API_KEY = process.env.ZEROx_API_KEY
37+
38+
Object.keys(DEFAULTS).forEach(i => {
39+
if (!process.env[i]) process.env[i] = DEFAULTS[i] // this is done to set the chain RPC details in @defillama/sdk
40+
})
41+
42+
43+
export function getEnv(key: string): any {
44+
if (!ENV_KEYS.has(key)) throw new Error(`Unknown env key: ${key}`)
45+
const value = process.env[key] ?? DEFAULTS[key]
46+
return BOOL_KEYS.includes(key) ? !!value : value
47+
}

‎utils/fetchURL.ts

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import axios, { AxiosRequestConfig } from "axios"
2+
3+
export default async function fetchURL(url: string, retries = 3): Promise<any> {
4+
try {
5+
const res = await httpGet(url)
6+
return res
7+
} catch (error) {
8+
if (retries > 0) return fetchURL(url, retries - 1)
9+
throw error
10+
}
11+
}
12+
13+
export async function postURL(url: string, data: any, retries = 3, options?: AxiosRequestConfig): Promise<any> {
14+
try {
15+
const res = await httpPost(url, data, options)
16+
return res
17+
} catch (error) {
18+
if (retries > 0) return postURL(url, data, retries - 1, options)
19+
throw error
20+
}
21+
}
22+
23+
function formAxiosError(url: string, error: any, options?: any) {
24+
let e = new Error((error as any)?.message)
25+
const axiosError = (error as any)?.response?.data?.message || (error as any)?.response?.data?.error || (error as any)?.response?.statusText || (error as any)?.response?.data;
26+
(e as any).url = url;
27+
Object.keys(options || {}).forEach((key) => (e as any)[key] = options[key]);
28+
if (axiosError) (e as any).axiosError = axiosError;
29+
delete (e as any).stack
30+
return e
31+
}
32+
33+
const successCodes: number[] = [200, 201, 202, 203, 204, 205, 206, 207, 208, 226];
34+
export async function httpGet(url: string, options?: AxiosRequestConfig, { withMetadata = false } = {}) {
35+
try {
36+
const res = await axios.get(url, options)
37+
if (!successCodes.includes(res.status)) throw new Error(`Error fetching ${url}: ${res.status} ${res.statusText}`)
38+
if (!res.data) throw new Error(`Error fetching ${url}: no data`)
39+
if (withMetadata) return res
40+
return res.data
41+
} catch (error) {
42+
throw formAxiosError(url, error, { method: 'GET' })
43+
}
44+
}
45+
46+
export async function httpPost(url: string, data: any, options?: AxiosRequestConfig, { withMetadata = false } = {}) {
47+
try {
48+
const res = await axios.post(url, data, options)
49+
if (!successCodes.includes(res.status)) throw new Error(`Error fetching ${url}: ${res.status} ${res.statusText}`)
50+
if (!res.data) throw new Error(`Error fetching ${url}: no data`)
51+
return res.data
52+
} catch (error) {
53+
if (withMetadata) throw error
54+
throw formAxiosError(url, error, { method: 'POST' })
55+
}
56+
}

0 commit comments

Comments
 (0)