add ethereum, add dune utils, add interactive chart viewer

.gitignore

@@ -2,4 +2,5 @@
 **/**/*.js
 /utils/testCharts
 result.txt
-/utils/result.png
+/utils/result.png
+.env

adapters/ethereum/index.ts

@@ -2,6 +2,7 @@ import fetch from "node-fetch";
 import { LinearAdapterResult } from "../../types/adapters";
 import { periodToSeconds } from "../../utils/time";
 import etherscan from "./etherscan.json";
+import uncleData from "./uncle.json";
 
 export const inflation = async (): Promise<LinearAdapterResult[]> => {
   const csvAmount = "120071885.806088302627129454";
@@ -28,3 +29,24 @@ export const inflation = async (): Promise<LinearAdapterResult[]> => {
 
   return sections;
 };
+
+export const uncle = async (): Promise<LinearAdapterResult[]> => {
+  const rawData = uncleData;
+  const linearSections: LinearAdapterResult[] = [];
+
+  const { startTime, rewards } = rawData;
+
+  for (let i = 0; i < rewards.length; i++) {
+    const start = startTime + i * 86400;
+    const end = startTime + 86400;
+
+    linearSections.push({
+      type: "linear",
+      amount: Number(rewards[i]),
+      start: start,
+      end: end
+    });
+  }
+
+  return linearSections;
+};

package.json

@@ -3,10 +3,12 @@
     "test": "mkdir -p utils/testCharts && ts-node utils/test.ts"
   },
   "dependencies": {
-    "@defillama/sdk": "^5.0.101",
+    "@defillama/sdk": "^5.0.126",
     "axios": "^1.3.4",
+    "chart.js": "^3.5.1",
     "chartjs-node-canvas": "^4.1.6",
     "dayjs": "^1.11.7",
+    "dotenv": "^16.5.0",
     "form-data": "^4.0.0",
     "graphql-request": "^5.2.0",
     "node-fetch": "2.6.6"

protocols/ethereum.ts

@@ -0,0 +1,113 @@
+import { uncle } from "../adapters/ethereum";
+import { manualCliff, manualLinear } from "../adapters/manual";
+import { CliffAdapterResult, LinearAdapterResult, Protocol } from "../types/adapters";
+import { GAS_TOKEN } from "../utils/constants";
+import { queryDune } from "../utils/dune";
+import { periodToSeconds } from "../utils/time";
+
+const chain = "ethereum";
+const start = 1438300800;
+
+const genesis = 1438269988; // block 1 5eth reward
+const byzantiumFork = 1508131331; // block 4370000 3eth reward
+const constantinopleFork = 1551383524; // block 7280000 2eth reward
+const merge = 1663224162; // block 15537393 0eth reward 
+
+interface BurnDataPoint {
+  eth_burn: number;
+  timestamp: number;
+}
+
+const burnData = async (type: 'pos' | 'pow'): Promise<LinearAdapterResult[]> => {
+  const result: LinearAdapterResult[] = [];
+  const burnData = await queryDune("5041563")
+
+  // Filter data based on type
+  let filteredData = burnData;
+  if (type === 'pow') {
+    // Pre-merge data
+    filteredData = burnData.filter((d: BurnDataPoint) => d.timestamp < merge);
+  } else if (type === 'pos') {
+    // Post-merge data
+    filteredData = burnData.filter((d: BurnDataPoint) => d.timestamp >= merge);
+  }
+
+  for (let i = 0; i < filteredData.length - 1; i++) {
+    result.push({
+      type: "linear",
+      start: filteredData[i + 1].timestamp,
+      end: filteredData[i].timestamp,
+      amount: -filteredData[i].eth_burn
+    });
+  }
+
+  return result;
+}
+
+const foundationOutflow = async (): Promise<CliffAdapterResult[]> => {
+  const result: CliffAdapterResult[] = [];
+  const foundationOutflowData = await queryDune("5041975")
+  for (let i = 0; i < foundationOutflowData.length; i++) {
+    result.push({
+      type: "cliff",
+      start: foundationOutflowData[i].start,
+      amount: foundationOutflowData[i].amount,
+    })
+  }
+
+  return result;
+}
+
+const stakingRewards = async (): Promise<LinearAdapterResult[]> => {
+  const result: LinearAdapterResult[] = [];
+  const issuanceData = await queryDune("5041721")
+
+  for (let i = 0; i < issuanceData.length - 1; i++) {
+    result.push({
+      type: "linear",
+      start: issuanceData[i + 1].timestamp,
+      end: issuanceData[i].timestamp,
+      amount: issuanceData[i].eth_issued
+    })
+  }
+  return result;
+}
+
+
+const ethereum: Protocol = {
+  "Crowd Sale": manualCliff(start, 6e7),
+  "Early Contributors": manualLinear(
+    start,
+    start + periodToSeconds.year * 4,
+    6e6,
+  ),
+  "Ethereum Foundation": foundationOutflow,
+  "Issuance": [
+    manualLinear(genesis, byzantiumFork, 4369999 * 5),
+    manualLinear(byzantiumFork, constantinopleFork, 2910000 * 3),
+    manualLinear(constantinopleFork, merge, 8257393 * 2),
+    uncle,
+    stakingRewards,
+    burnData
+  ],
+  meta: {
+    token: `${chain}:${GAS_TOKEN}`,
+    notes: [
+      `Information on the Early Contributor vesting schedule structure could not be found, here we have assumed it as linearly unlocked over 4 years.`,
+      `The Ethereum Foundation supply is assumed to be unlocked when there's an outflow from EthDev address.`,
+      `Issuance is combination of PoW and PoS issuance, with Uncle Rewards and EIP-1559 burning included`,
+    ],
+    sources: [
+      "https://dune.com/21co/ethereum-key-metrics",
+      "https://www.galaxy.com/insights/research/breakdown-of-ethereum-supply-distribution-since-genesis/",
+      "https://fastercapital.com/topics/common-token-vesting-strategies-for-airdrop-cryptocurrency.html",
+    ],
+    protocolIds: ["4488"],
+  },
+  categories: {
+    farming: ["Issuance"],
+    insiders: ["Ethereum Foundation"],
+    publicSale: ["Crowd Sale"],
+  },
+};
+export default ethereum;

utils/dune.ts

@@ -0,0 +1,136 @@
+import { httpGet, httpPost } from "./fetchURL";
+import { getEnv } from "./env";
+const plimit = require('p-limit');
+const limit = plimit(1);
+
+const isRestrictedMode = getEnv('DUNE_RESTRICTED_MODE') === 'true'
+const API_KEYS = getEnv('DUNE_API_KEYS')?.split(',') ?? ["L0URsn5vwgyrWbBpQo9yS1E3C1DBJpZh"]
+let API_KEY_INDEX = 0;
+
+const NOW_TIMESTAMP = Math.trunc((Date.now()) / 1000)
+
+const getLatestData = async (queryId: string) => {
+  checkCanRunDuneQuery()
+
+  const url = `https://api.dune.com/api/v1/query/${queryId}/results`
+  try {
+    const latest_result = (await limit(() => httpGet(url, {
+      headers: {
+        "x-dune-api-key": API_KEYS[API_KEY_INDEX]
+      }
+    })))
+    const submitted_at = latest_result.submitted_at
+    const submitted_at_timestamp = Math.trunc(new Date(submitted_at).getTime() / 1000)
+    const diff = NOW_TIMESTAMP - submitted_at_timestamp
+    if (diff < 60 * 60 * 3) {
+      return latest_result.result.rows
+    }
+    return undefined
+  } catch (e: any) {
+    throw e;
+  }
+}
+
+
+async function randomDelay() {
+  const delay = Math.floor(Math.random() * 5) + 2
+  return new Promise((resolve) => setTimeout(resolve, delay * 1000))
+}
+
+const inquiryStatus = async (execution_id: string, queryId: string) => {
+  checkCanRunDuneQuery()
+
+  let _status = undefined;
+  do {
+    try {
+      _status = (await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/status`, {
+        headers: {
+          "x-dune-api-key": API_KEYS[API_KEY_INDEX]
+        }
+      }))).state
+      if (['QUERY_STATE_PENDING', 'QUERY_STATE_EXECUTING'].includes(_status)) {
+        console.info(`waiting for query id ${queryId} to complete...`)
+        await randomDelay() // 1 - 4s
+      }
+    } catch (e: any) {
+      throw e;
+    }
+  } while (_status !== 'QUERY_STATE_COMPLETED' && _status !== 'QUERY_STATE_FAILED');
+  return _status
+}
+
+const submitQuery = async (queryId: string, query_parameters = {}) => {
+  checkCanRunDuneQuery()
+
+  let query: undefined | any = undefined
+  try {
+    query = await limit(() => httpPost(`https://api.dune.com/api/v1/query/${queryId}/execute`, { query_parameters }, {
+      headers: {
+        "x-dune-api-key": API_KEYS[API_KEY_INDEX],
+        'Content-Type': 'application/json'
+      }
+    }))
+    if (query?.execution_id) {
+      return query?.execution_id
+    } else {
+      throw new Error("error query data: " + query)
+    }
+  } catch (e: any) {
+    throw e;
+  }
+}
+
+
+export const queryDune = async (queryId: string, query_parameters: any = {}) => {
+  checkCanRunDuneQuery()
+
+  if (Object.keys(query_parameters).length === 0) {
+    const latest_result = await getLatestData(queryId)
+    if (latest_result !== undefined) return latest_result
+  }
+  const execution_id = await submitQuery(queryId, query_parameters)
+  const _status = await inquiryStatus(execution_id, queryId)
+  if (_status === 'QUERY_STATE_COMPLETED') {
+    const API_KEY = API_KEYS[API_KEY_INDEX]
+    try {
+      const queryStatus = await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/results?limit=100000`, {
+        headers: {
+          "x-dune-api-key": API_KEY
+        }
+      }))
+      return queryStatus.result.rows
+    } catch (e: any) {
+      throw e;
+    }
+  } else if(_status === "QUERY_STATE_FAILED"){
+    if(query_parameters.fullQuery){
+      console.log(`Dune query: ${query_parameters.fullQuery}`)
+    } else {
+      console.log("Dune parameters", query_parameters)
+    }
+    throw new Error(`Dune query failed: ${queryId}`)
+  }
+}
+
+const tableName = {
+  bsc: "bnb",
+  ethereum: "ethereum",
+  base: "base",
+  avax: "avalanche_c"
+} as any
+
+export const queryDuneSql = (options: any, query: string) => {
+  checkCanRunDuneQuery()
+
+  return queryDune("3996608", {
+    fullQuery: query.replace("CHAIN", tableName[options.chain] ?? options.chain).split("TIME_RANGE").join(`block_time >= from_unixtime(${options.startTimestamp})
+  AND block_time <= from_unixtime(${options.endTimestamp})`)
+  })
+}
+
+export function checkCanRunDuneQuery() {
+  if (!isRestrictedMode) return;
+  const currentHour = new Date().getUTCHours();
+  if (currentHour >= 1 && currentHour <= 3) return; // 1am - 3am - any time other than this, throw error
+  throw new Error(`Current hour is ${currentHour}. In restricted mode, can run dune queries only between 1am - 3am UTC`);
+}

utils/env.ts

@@ -0,0 +1,25 @@
+const BOOL_KEYS = [
+    ''
+]
+
+const DEFAULTS: any = {
+    BITLAYER_RPC: "https://rpc.bitlayer.org,https://rpc.ankr.com/bitlayer,https://rpc.bitlayer-rpc.com,https://rpc-bitlayer.rockx.com",
+}
+
+export const ENV_KEYS = new Set([
+    ...BOOL_KEYS,
+    ...Object.keys(DEFAULTS),
+    'DUNE_API_KEYS',
+    'DUNE_RESTRICTED_MODE'
+])
+
+Object.keys(DEFAULTS).forEach(i => {
+    if (!process.env[i]) process.env[i] = DEFAULTS[i] // this is done to set the chain RPC details in @defillama/sdk
+})
+
+
+export function getEnv(key: string): any {
+    if (!ENV_KEYS.has(key)) throw new Error(`Unknown env key: ${key}`)
+    const value = process.env[key] ?? DEFAULTS[key]
+    return BOOL_KEYS.includes(key) ? !!value : value
+}

utils/fetchURL.ts

@@ -0,0 +1,56 @@
+import axios, { AxiosRequestConfig } from "axios"
+
+export default async function fetchURL(url: string, retries = 3): Promise<any> {
+  try {
+    const res = await httpGet(url)
+    return res
+  } catch (error) {
+    if (retries > 0) return fetchURL(url, retries - 1)
+    throw error
+  }
+}
+
+export async function postURL(url: string, data: any, retries = 3, options?: AxiosRequestConfig): Promise<any> {
+  try {
+    const res = await httpPost(url, data, options)
+    return res
+  } catch (error) {
+    if (retries > 0) return postURL(url, data, retries - 1, options)
+    throw error
+  }
+}
+
+function formAxiosError(url: string, error: any, options?: any) {
+  let e = new Error((error as any)?.message)
+  const axiosError = (error as any)?.response?.data?.message || (error as any)?.response?.data?.error || (error as any)?.response?.statusText || (error as any)?.response?.data;
+  (e as any).url = url;
+  Object.keys(options || {}).forEach((key) => (e as any)[key] = options[key]);
+  if (axiosError) (e as any).axiosError = axiosError;
+  delete (e as any).stack
+  return e
+}
+
+const successCodes: number[] = [200, 201, 202, 203, 204, 205, 206, 207, 208, 226];
+export async function httpGet(url: string, options?: AxiosRequestConfig, { withMetadata = false } = {}) {
+  try {
+    const res = await axios.get(url, options)
+    if (!successCodes.includes(res.status)) throw new Error(`Error fetching ${url}: ${res.status} ${res.statusText}`)
+    if (!res.data) throw new Error(`Error fetching ${url}: no data`)
+    if (withMetadata) return res
+    return res.data
+  } catch (error) {
+    throw formAxiosError(url, error, { method: 'GET' })
+  }
+}
+
+export async function httpPost(url: string, data: any, options?: AxiosRequestConfig, { withMetadata = false } = {}) {
+  try {
+    const res = await axios.post(url, data, options)
+    if (!successCodes.includes(res.status)) throw new Error(`Error fetching ${url}: ${res.status} ${res.statusText}`)
+    if (!res.data) throw new Error(`Error fetching ${url}: no data`)
+    return res.data
+  } catch (error) {
+    if (withMetadata) throw error
+    throw formAxiosError(url, error, { method: 'POST' })
+  }
+}