Skip to content

Commit bb5d70b

Browse files
mattromanomattromano
authored
AN-6016/create-daily-token-transfer-summary (#453)
* add transfers model * add transfer table to sources * add incremental * update summary table * update alias * remove transfer summary * remove amount test * update to jan 1 * remove silver transfers * update inc to snapshot method * update inc and yml * remove thresholds * remove comments * SO and add rec test * push back rec test, add not null filter * set to prod default date
1 parent 245a5a1 commit bb5d70b

File tree

3 files changed

+259
-3
lines changed

3 files changed

+259
-3
lines changed

‎models/silver/core/silver__transfers_summary.sql

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
{{ config(
2+
materialized = 'incremental',
3+
unique_key = ['transfers_id'],
4+
cluster_by = ['blockchain','block_day'],
5+
merge_exclude_columns = ['inserted_timestamp'],
6+
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address);",
7+
tags = ['daily']
8+
) }}
9+
10+
{% set default_date = "'2025-01-01'" %}
11+
12+
{% if is_incremental() %}
13+
{% set yesterday = "DATEADD('day', -1, SYSDATE()::DATE)" %}
14+
{% set block_ts_filter = "block_timestamp::date = " ~ yesterday %}
15+
{% set max_mod = yesterday %}
16+
{% else %}
17+
{% set block_ts_filter = "block_timestamp::date >= " ~ default_date ~ " AND block_timestamp::date < SYSDATE()::DATE" %}
18+
{% set max_mod = default_date %}
19+
{% endif %}
20+
21+
WITH evm_transfers AS (
22+
{{ dbt_utils.union_relations(
23+
relations=[
24+
source('arbitrum_core', 'ez_token_transfers'),
25+
source('avalanche_core', 'ez_token_transfers'),
26+
source('base_core', 'ez_token_transfers'),
27+
source('blast_core', 'ez_token_transfers'),
28+
source('boba_core', 'ez_token_transfers'),
29+
source('bsc_core', 'ez_token_transfers'),
30+
source('core_core', 'ez_token_transfers'),
31+
source('ethereum_core', 'ez_token_transfers'),
32+
source('gnosis_core', 'ez_token_transfers'),
33+
source('ink_core', 'ez_token_transfers'),
34+
source('kaia_core', 'ez_token_transfers'),
35+
source('mantle_core', 'ez_token_transfers'),
36+
source('optimism_core', 'ez_token_transfers'),
37+
source('polygon_core', 'ez_token_transfers'),
38+
source('sei_evm_core', 'ez_token_transfers'),
39+
source('flow_evm_core', 'ez_token_transfers'),
40+
],
41+
where=block_ts_filter
42+
) }}
43+
),
44+
45+
all_transfers AS (
46+
-- EVM Chains
47+
SELECT
48+
DATE(block_timestamp) as block_day,
49+
contract_address as address,
50+
LOWER(SPLIT_PART(_dbt_source_relation, '.', 1)) as blockchain,
51+
tx_hash,
52+
from_address,
53+
to_address,
54+
amount
55+
FROM
56+
evm_transfers
57+
58+
UNION ALL
59+
60+
-- Non-EVM Chains
61+
SELECT
62+
DATE(block_timestamp) as block_day,
63+
token_address as address,
64+
'aleo' as blockchain,
65+
tx_id as tx_hash,
66+
sender as from_address,
67+
receiver as to_address,
68+
amount
69+
FROM {{ source('aleo_core', 'fact_transfers') }}
70+
WHERE {{ block_ts_filter }}
71+
AND token_address is not null
72+
73+
UNION ALL
74+
75+
SELECT
76+
DATE(block_timestamp) as block_day,
77+
token_address as address,
78+
'aptos' as blockchain,
79+
tx_hash,
80+
CASE
81+
WHEN transfer_event = 'WithdrawEvent' THEN account_address
82+
ELSE NULL
83+
END as from_address,
84+
CASE
85+
WHEN transfer_event = 'DepositEvent' THEN account_address
86+
ELSE NULL
87+
END as to_address,
88+
amount
89+
FROM {{ source('aptos_core', 'fact_transfers') }}
90+
WHERE {{ block_ts_filter }}
91+
92+
UNION ALL
93+
94+
SELECT
95+
DATE(block_timestamp) as block_day,
96+
currency as address,
97+
'axelar' as blockchain,
98+
tx_id as tx_hash,
99+
sender as from_address,
100+
receiver as to_address,
101+
amount
102+
FROM {{ source('axelar_core', 'fact_transfers') }}
103+
WHERE {{ block_ts_filter }}
104+
105+
UNION ALL
106+
107+
SELECT
108+
DATE(block_timestamp) as block_day,
109+
currency as address,
110+
'cosmos' as blockchain,
111+
tx_id as tx_hash,
112+
sender as from_address,
113+
receiver as to_address,
114+
amount
115+
FROM {{ source('cosmos_core', 'fact_transfers') }}
116+
WHERE {{ block_ts_filter }}
117+
118+
UNION ALL
119+
120+
SELECT
121+
DATE(block_timestamp) as block_day,
122+
mint as address,
123+
'eclipse' as blockchain,
124+
tx_id as tx_hash,
125+
tx_from as from_address,
126+
tx_to as to_address,
127+
amount
128+
FROM {{ source('eclipse_core', 'fact_transfers') }}
129+
WHERE {{ block_ts_filter }}
130+
131+
UNION ALL
132+
133+
SELECT
134+
DATE(block_timestamp) as block_day,
135+
token_contract as address,
136+
'flow' as blockchain,
137+
tx_id as tx_hash,
138+
sender as from_address,
139+
recipient as to_address,
140+
amount
141+
FROM {{ source('flow_core', 'ez_token_transfers') }}
142+
WHERE {{ block_ts_filter }}
143+
144+
UNION ALL
145+
146+
SELECT
147+
DATE(block_timestamp) as block_day,
148+
contract_address as address,
149+
'near' as blockchain,
150+
tx_hash,
151+
from_address,
152+
to_address,
153+
amount
154+
FROM {{ source('near_core', 'ez_token_transfers') }}
155+
WHERE {{ block_ts_filter }}
156+
157+
UNION ALL
158+
159+
SELECT
160+
DATE(block_timestamp) as block_day,
161+
currency as address,
162+
'osmosis' as blockchain,
163+
tx_id as tx_hash,
164+
sender as from_address,
165+
receiver as to_address,
166+
amount
167+
FROM {{ source('osmosis_core', 'fact_transfers') }}
168+
WHERE {{ block_ts_filter }}
169+
170+
UNION ALL
171+
172+
SELECT
173+
DATE(block_timestamp) as block_day,
174+
mint as address,
175+
'solana' as blockchain,
176+
tx_id as tx_hash,
177+
tx_from as from_address,
178+
tx_to as to_address,
179+
amount
180+
FROM {{ source('solana_core', 'fact_transfers') }}
181+
WHERE {{ block_ts_filter }}
182+
183+
),
184+
185+
aggregated_transfers AS (
186+
SELECT
187+
block_day,
188+
address,
189+
blockchain,
190+
count(distinct tx_hash) as tx_count,
191+
count(distinct from_address) as unique_senders,
192+
sum(amount) as amount
193+
FROM all_transfers
194+
GROUP BY 1,2,3
195+
where address is not null
196+
)
197+
198+
SELECT
199+
block_day,
200+
address,
201+
blockchain,
202+
tx_count,
203+
unique_senders,
204+
amount,
205+
{{ dbt_utils.generate_surrogate_key(['address','blockchain','block_day']) }} AS transfers_id,
206+
SYSDATE() AS inserted_timestamp,
207+
SYSDATE() AS modified_timestamp,
208+
'{{ invocation_id }}' AS _invocation_id
209+
FROM
210+
aggregated_transfers

‎models/silver/core/silver__transfers_summary.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
version: 2
2+
3+
models:
4+
- name: silver__transfers_summary
5+
tests:
6+
- dbt_utils.unique_combination_of_columns:
7+
combination_of_columns:
8+
- ADDRESS
9+
- BLOCKCHAIN
10+
- BLOCK_DAY
11+
columns:
12+
- name: BLOCK_DAY
13+
tests:
14+
- not_null
15+
- dbt_expectations.expect_row_values_to_have_recent_data:
16+
datepart: day
17+
interval: 2
18+
19+
- name: ADDRESS
20+
tests:
21+
- not_null
22+
23+
- name: BLOCKCHAIN
24+
tests:
25+
- not_null
26+
27+
- name: TX_COUNT
28+
tests:
29+
- not_null
30+
31+
- name: UNIQUE_SENDERS
32+
tests:
33+
- not_null
34+
35+
- name: AMOUNT
36+
37+
- name: TRANSFERS_ID
38+
tests:
39+
- unique
40+
- not_null

‎models/sources.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ sources:
99
schema: core
1010
tables:
1111
- name: dim_token_registrations
12+
- name: fact_transfers
1213

1314
- name: aleo_observ
1415
database: aleo
@@ -379,6 +380,7 @@ sources:
379380
schema: core
380381
tables:
381382
- name: dim_contracts
383+
- name: ez_token_transfers
382384

383385
- name: bob_silver
384386
database: bob
@@ -395,6 +397,7 @@ sources:
395397
schema: core
396398
tables:
397399
- name: dim_contracts
400+
- name: ez_token_transfers
398401

399402
- name: boba_silver
400403
database: boba
@@ -506,6 +509,7 @@ sources:
506509
schema: core
507510
tables:
508511
- name: dim_contracts
512+
- name: ez_token_transfers
509513

510514
- name: core_silver
511515
database: core
@@ -646,6 +650,7 @@ sources:
646650
- name: contracts
647651
- name: complete_event_abis
648652
- name: blocks
653+
- name: transfers
649654

650655
- name: ethereum_silver_dex
651656
database: ethereum
@@ -719,6 +724,7 @@ sources:
719724
schema: core_evm
720725
tables:
721726
- name: dim_contracts
727+
- name: ez_token_transfers
722728

723729
- name: flow_observ
724730
database: flow
@@ -794,6 +800,7 @@ sources:
794800
- name: contracts
795801
- name: traces
796802
- name: blocks
803+
797804

798805
- name: gnosis_silver_dex
799806
database: gnosis
@@ -830,7 +837,7 @@ sources:
830837
schema: core
831838
tables:
832839
- name: dim_contracts
833-
840+
- name: ez_token_transfers
834841
- name: ink_silver
835842
database: ink
836843
schema: silver
@@ -1271,7 +1278,7 @@ sources:
12711278
schema: core
12721279
tables:
12731280
- name: dim_contracts
1274-
1281+
- name: ez_token_transfers
12751282
- name: swell_silver
12761283
database: swell
12771284
schema: silver
@@ -1311,7 +1318,6 @@ sources:
13111318
tables:
13121319
- name: fact_swaps
13131320
- name: fact_liquidity_actions
1314-
13151321
#=============================================================================
13161322
# TON
13171323
#=============================================================================

0 commit comments

Comments
 (0)