|
| 1 | +import { httpGet, httpPost } from "./fetchURL"; |
| 2 | +import { getEnv } from "./env"; |
| 3 | +const plimit = require('p-limit'); |
| 4 | +const limit = plimit(1); |
| 5 | + |
| 6 | +const isRestrictedMode = getEnv('DUNE_RESTRICTED_MODE') === 'true' |
| 7 | +const API_KEYS = getEnv('DUNE_API_KEYS')?.split(',') ?? ["L0URsn5vwgyrWbBpQo9yS1E3C1DBJpZh"] |
| 8 | +let API_KEY_INDEX = 0; |
| 9 | + |
| 10 | +const NOW_TIMESTAMP = Math.trunc((Date.now()) / 1000) |
| 11 | + |
| 12 | +const getLatestData = async (queryId: string) => { |
| 13 | + checkCanRunDuneQuery() |
| 14 | + |
| 15 | + const url = `https://api.dune.com/api/v1/query/${queryId}/results` |
| 16 | + try { |
| 17 | + const latest_result = (await limit(() => httpGet(url, { |
| 18 | + headers: { |
| 19 | + "x-dune-api-key": API_KEYS[API_KEY_INDEX] |
| 20 | + } |
| 21 | + }))) |
| 22 | + const submitted_at = latest_result.submitted_at |
| 23 | + const submitted_at_timestamp = Math.trunc(new Date(submitted_at).getTime() / 1000) |
| 24 | + const diff = NOW_TIMESTAMP - submitted_at_timestamp |
| 25 | + if (diff < 60 * 60 * 3) { |
| 26 | + return latest_result.result.rows |
| 27 | + } |
| 28 | + return undefined |
| 29 | + } catch (e: any) { |
| 30 | + throw e; |
| 31 | + } |
| 32 | +} |
| 33 | + |
| 34 | + |
| 35 | +async function randomDelay() { |
| 36 | + const delay = Math.floor(Math.random() * 5) + 2 |
| 37 | + return new Promise((resolve) => setTimeout(resolve, delay * 1000)) |
| 38 | +} |
| 39 | + |
| 40 | +const inquiryStatus = async (execution_id: string, queryId: string) => { |
| 41 | + checkCanRunDuneQuery() |
| 42 | + |
| 43 | + let _status = undefined; |
| 44 | + do { |
| 45 | + try { |
| 46 | + _status = (await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/status`, { |
| 47 | + headers: { |
| 48 | + "x-dune-api-key": API_KEYS[API_KEY_INDEX] |
| 49 | + } |
| 50 | + }))).state |
| 51 | + if (['QUERY_STATE_PENDING', 'QUERY_STATE_EXECUTING'].includes(_status)) { |
| 52 | + console.info(`waiting for query id ${queryId} to complete...`) |
| 53 | + await randomDelay() // 1 - 4s |
| 54 | + } |
| 55 | + } catch (e: any) { |
| 56 | + throw e; |
| 57 | + } |
| 58 | + } while (_status !== 'QUERY_STATE_COMPLETED' && _status !== 'QUERY_STATE_FAILED'); |
| 59 | + return _status |
| 60 | +} |
| 61 | + |
| 62 | +const submitQuery = async (queryId: string, query_parameters = {}) => { |
| 63 | + checkCanRunDuneQuery() |
| 64 | + |
| 65 | + let query: undefined | any = undefined |
| 66 | + try { |
| 67 | + query = await limit(() => httpPost(`https://api.dune.com/api/v1/query/${queryId}/execute`, { query_parameters }, { |
| 68 | + headers: { |
| 69 | + "x-dune-api-key": API_KEYS[API_KEY_INDEX], |
| 70 | + 'Content-Type': 'application/json' |
| 71 | + } |
| 72 | + })) |
| 73 | + if (query?.execution_id) { |
| 74 | + return query?.execution_id |
| 75 | + } else { |
| 76 | + throw new Error("error query data: " + query) |
| 77 | + } |
| 78 | + } catch (e: any) { |
| 79 | + throw e; |
| 80 | + } |
| 81 | +} |
| 82 | + |
| 83 | + |
| 84 | +export const queryDune = async (queryId: string, query_parameters: any = {}) => { |
| 85 | + checkCanRunDuneQuery() |
| 86 | + |
| 87 | + if (Object.keys(query_parameters).length === 0) { |
| 88 | + const latest_result = await getLatestData(queryId) |
| 89 | + if (latest_result !== undefined) return latest_result |
| 90 | + } |
| 91 | + const execution_id = await submitQuery(queryId, query_parameters) |
| 92 | + const _status = await inquiryStatus(execution_id, queryId) |
| 93 | + if (_status === 'QUERY_STATE_COMPLETED') { |
| 94 | + const API_KEY = API_KEYS[API_KEY_INDEX] |
| 95 | + try { |
| 96 | + const queryStatus = await limit(() => httpGet(`https://api.dune.com/api/v1/execution/${execution_id}/results?limit=100000`, { |
| 97 | + headers: { |
| 98 | + "x-dune-api-key": API_KEY |
| 99 | + } |
| 100 | + })) |
| 101 | + return queryStatus.result.rows |
| 102 | + } catch (e: any) { |
| 103 | + throw e; |
| 104 | + } |
| 105 | + } else if(_status === "QUERY_STATE_FAILED"){ |
| 106 | + if(query_parameters.fullQuery){ |
| 107 | + console.log(`Dune query: ${query_parameters.fullQuery}`) |
| 108 | + } else { |
| 109 | + console.log("Dune parameters", query_parameters) |
| 110 | + } |
| 111 | + throw new Error(`Dune query failed: ${queryId}`) |
| 112 | + } |
| 113 | +} |
| 114 | + |
| 115 | +const tableName = { |
| 116 | + bsc: "bnb", |
| 117 | + ethereum: "ethereum", |
| 118 | + base: "base", |
| 119 | + avax: "avalanche_c" |
| 120 | +} as any |
| 121 | + |
| 122 | +export const queryDuneSql = (options: any, query: string) => { |
| 123 | + checkCanRunDuneQuery() |
| 124 | + |
| 125 | + return queryDune("3996608", { |
| 126 | + fullQuery: query.replace("CHAIN", tableName[options.chain] ?? options.chain).split("TIME_RANGE").join(`block_time >= from_unixtime(${options.startTimestamp}) |
| 127 | + AND block_time <= from_unixtime(${options.endTimestamp})`) |
| 128 | + }) |
| 129 | +} |
| 130 | + |
| 131 | +export function checkCanRunDuneQuery() { |
| 132 | + if (!isRestrictedMode) return; |
| 133 | + const currentHour = new Date().getUTCHours(); |
| 134 | + if (currentHour >= 1 && currentHour <= 3) return; // 1am - 3am - any time other than this, throw error |
| 135 | + throw new Error(`Current hour is ${currentHour}. In restricted mode, can run dune queries only between 1am - 3am UTC`); |
| 136 | +} |
0 commit comments