From f26184e0fe1b5ee4d1fde543f3954572945905c2 Mon Sep 17 00:00:00 2001 From: LJ Date: Mon, 17 Nov 2025 12:48:01 -0500 Subject: [PATCH] linters etc --- mezo/__init__.py | 1 + mezo/clients.py | 29 +++++++++++-- mezo/currency_config.py | 62 +++++++++++++--------------- pyproject.toml | 8 +++- requirements.txt | 1 + scripts/fetch_market_transactions.py | 19 +++++---- scripts/fetch_mezo_users.py | 5 +-- scripts/process_dapp_data.py | 15 +++---- scripts/process_musd_data.py | 14 +++---- scripts/process_pools_data.py | 11 +++-- scripts/process_swaps_data.py | 60 ++++++++++++++++++--------- scripts/process_vaults_data.py | 10 ++--- 12 files changed, 142 insertions(+), 93 deletions(-) diff --git a/mezo/__init__.py b/mezo/__init__.py index cfaa784..d68d9cd 100644 --- a/mezo/__init__.py +++ b/mezo/__init__.py @@ -6,6 +6,7 @@ # sys.path.insert(0, project_root) import os + from dotenv import load_dotenv load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '..', '.env'), override=True) diff --git a/mezo/clients.py b/mezo/clients.py index 216d44a..078f085 100644 --- a/mezo/clients.py +++ b/mezo/clients.py @@ -94,9 +94,9 @@ def get_subgraph_data(subgraph_url, query, query_key): MUSD_STABILITY_POOL_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-stability-pool/1.0.0/gn" MUSD_TROVE_MANAGER_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-trove-manager/1.0.0/gn" AUGUST_VAULT_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/mezo-vaults-mezo/1.0.0/gn" - SWAPS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.0.0/gn" - POOLS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.1.0/gn" - TIGRIS_POOLS_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/tigris-pools-mezo/1.0.0/gn' + SWAPS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.2.0/gn" + POOLS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.2.0/gn" + TIGRIS_POOLS_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/tigris-pools-mezo/1.2.0/gn' WORMHOLE_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/wormhole-bridge-mezo/1.0.0/gn' LOLLI_WIT_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/btc-as-erc20-mezo/1.0.0/gn' @@ -408,6 +408,29 @@ def __init__(self, key: str = None, project_id: str = None): self.client = bigquery.Client(project=project_id, credentials=credentials) + def query(self, query_string: str) -> pd.DataFrame: + """ + Execute a SQL query against BigQuery and return results as a pandas DataFrame. + + Args: + query_string: SQL query string to execute + + Returns: + pandas.DataFrame: Query results as a DataFrame + + Example: + >>> client = BigQueryClient(key="BIGQUERY_KEY", project_id="my-project") + >>> df = client.query("SELECT * FROM `my-project.my_dataset.my_table` LIMIT 100") + """ + try: + query_job = self.client.query(query_string) + df = query_job.to_dataframe() + print(f"āœ… Query executed successfully. Returned {len(df)} rows.") + return df + except Exception as e: + print(f"āŒ Error executing query: {e}") + raise + def create_dataset(self, dataset_id: str, location: str = "US"): dataset_ref = bigquery.Dataset(self.client.dataset(dataset_id)) try: diff --git a/mezo/currency_config.py b/mezo/currency_config.py index 06c488b..525d28d 100644 --- a/mezo/currency_config.py +++ b/mezo/currency_config.py @@ -56,25 +56,20 @@ } POOLS_MAP = { - '0x52e604c44417233b6ccedddc0d640a405caacefb': 'btc_musd_pool', - '0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356': 'musdc_musd_pool', - '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'musd_musdt_pool', - '0x2a1ab0224a7a608d3a992cb15594a2934f74f4c0': 'musdc_musdt_pool', - '0xb07c2172c4b7bbc3ac52088d30cb103853b0b403': 'musdc_btc_pool', - '0x329d64572f8922c3fe90d23a3c74a360d8ea6235': 'btc_xsolvbtc_pool', - '0x58c8f6d2e589928c46425eaf4254b6a41c45a584': 'upmusd_musd_pool', - '0x5cd2a025c001e07ae354a4c22c3009908de1ac59': 'solvbtc_musd_pool', - '0x9e60cd4d5b718178fab0137200a36a5472191302': 'btc_musdt_pool', - '0xb7fd1db5228e4d9f4109c5635f66375e5af0d8f5': 'musd_xsolvbtc_pool', - '0xf6f950485b0a65828f07581ca979ef1271778d6a': 'btc_solvbtc_pool', - '0xfbcc89586780ac6f41e9cec97663e5592be41331': 'upmusd_btc_pool', - '0x58b18C64F0483D6b742c54C23B625c65F7A0974d': 't_musd_pool' -} - -POOL_TOKEN0_MAP = { - '0x52e604c44417233b6ccedddc0d640a405caacefb': 'BTC', - '0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356': 'USDC', - '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'MUSD', + "0x52e604c44417233b6ccedddc0d640a405caacefb": "btc_musd_pool", + "0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356": "musdc_musd_pool", + "0x10906a9e9215939561597b4c8e4b98f93c02031a": "musd_musdt_pool", + "0x2a1ab0224a7a608d3a992cb15594a2934f74f4c0": "musdc_musdt_pool", + "0xb07c2172c4b7bbc3ac52088d30cb103853b0b403": "musdc_btc_pool", + "0x329d64572f8922c3fe90d23a3c74a360d8ea6235": "btc_xsolvbtc_pool", + "0x58c8f6d2e589928c46425eaf4254b6a41c45a584": "upmusd_musd_pool", + "0x5cd2a025c001e07ae354a4c22c3009908de1ac59": "solvbtc_musd_pool", + "0x9e60cd4d5b718178fab0137200a36a5472191302": "btc_musdt_pool", + "0xb7fd1db5228e4d9f4109c5635f66375e5af0d8f5": "musd_xsolvbtc_pool", + "0xf6f950485b0a65828f07581ca979ef1271778d6a": "btc_solvbtc_pool", + "0xfbcc89586780ac6f41e9cec97663e5592be41331": "upmusd_btc_pool", + "0x6688f868e9c81ee671867e77fbc618bbea2e9782": "t_musd_pool", + "0x72e6b3f126cf4f6c90c08114ac29038a0e269210": "cbbtc_btc_pool" } POOL_TOKEN_PAIRS = { @@ -90,23 +85,24 @@ '0xb7fd1db5228e4d9f4109c5635f66375e5af0d8f5': {'token0': 'MUSD', 'token1': 'xSolvBTC'}, '0xf6f950485b0a65828f07581ca979ef1271778d6a': {'token0': 'BTC', 'token1': 'SolvBTC'}, '0xfbcc89586780ac6f41e9cec97663e5592be41331': {'token0': 'upMUSD', 'token1': 'BTC'}, - '0x58b18C64F0483D6b742c54C23B625c65F7A0974d': {'token0': 'T', 'token1': 'MUSD'} + '0x6688f868e9c81ee671867e77fbc618bbea2e9782': {'token0': 'T', 'token1': 'MUSD'}, + '0x72e6b3f126cf4f6c90c08114ac29038a0e269210': {'token0': 'cbBTC', 'token1': 'BTC'} } MEZO_TOKEN_ADDRESSES = { - '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'USDT', - '0x52e604c44417233b6ccedddc0d640a405caacefb': 'BTC', - '0xa10aD2570ea7b93d19fDae6Bd7189fF4929Bc747': 'SolvBTC', - '0xdF708431162Ba247dDaE362D2c919e0fbAfcf9DE': 'xSolvBTC', - '0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356': 'USDC', - '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'MUSD', - '0x221B2D9aD7B994861Af3f4c8A80c86C4aa86Bf53': 'upMUSD', - '0xaaC423eDC4E3ee9ef81517e8093d52737165b71F': 'T', - '0x29fA8F46CBB9562b87773c8f50a7F9F27178261c': 'swBTC', - '0x1531b6e3d51BF80f634957dF81A990B92dA4b154': 'DAI', - '0x812fcC0Bb8C207Fd8D6165a7a1173037F43B2dB8': 'FBTC', - '0xdf6542260a9F768f07030E4895083F804241F4C4': 'USDe', - '0x6a7CD8E1384d49f502b4A4CE9aC9eb320835c5d7': 'cbBTC' + "0x10906a9e9215939561597b4c8e4b98f93c02031a": "USDT", + "0x52e604c44417233b6ccedddc0d640a405caacefb": "BTC", + "0xa10aD2570ea7b93d19fDae6Bd7189fF4929Bc747": "SolvBTC", + "0xdF708431162Ba247dDaE362D2c919e0fbAfcf9DE": "xSolvBTC", + "0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356": "USDC", + "0xdD468A1DDc392dcdbEf6db6e34E89AA338F9F186": "MUSD", + "0x221B2D9aD7B994861Af3f4c8A80c86C4aa86Bf53": "upMUSD", + "0xaaC423eDC4E3ee9ef81517e8093d52737165b71F": "T", + "0x29fA8F46CBB9562b87773c8f50a7F9F27178261c": "swBTC", + "0x1531b6e3d51BF80f634957dF81A990B92dA4b154": "DAI", + "0x812fcC0Bb8C207Fd8D6165a7a1173037F43B2dB8": "FBTC", + "0xdf6542260a9F768f07030E4895083F804241F4C4": "USDe", + "0x6a7CD8E1384d49f502b4A4CE9aC9eb320835c5d7": "cbBTC", } TIGRIS_MAP = { diff --git a/pyproject.toml b/pyproject.toml index 1de2c56..472d1f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,13 +21,17 @@ requires-python = "~=3.13.0" [tool.ruff] line-length = 99 src = ["mezo"] -include = ["pyproject.toml", "mezo/**/*.py"] +include = ["pyproject.toml", "mezo/**/*.py", "docs/**/*.py"] [tool.ruff.lint] extend-select = ["I"] # Add import sorting -ignore = ["E501", "W291", "W293"] # Ignore line-too-long and trailing whitespace +ignore = ["E501", "W291", "W293", "F401", "F811", "F841"] # Ignore line-too-long, trailing whitespace, unused imports, and unused variables + +[tool.ruff.per-file-ignores] +"**/docs/models/*.py" = ["F401", "F811", "F821", "F841"] [tool.ruff.lint.isort] known-first-party = ["mezo"] force-sort-within-sections = true + diff --git a/requirements.txt b/requirements.txt index 6d09059..71c8196 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ python-dotenv>=1.0.0 # Database and API clients db-dtypes>=1.4.0 +dune-client>=1.9.3 google-api-core>=2.20.0 google-auth>=2.30.0 google-cloud-bigquery>=3.30.0 diff --git a/scripts/fetch_market_transactions.py b/scripts/fetch_market_transactions.py index e1e2656..16fcac5 100644 --- a/scripts/fetch_market_transactions.py +++ b/scripts/fetch_market_transactions.py @@ -6,11 +6,12 @@ MUSD Market smart contracts via the Mezo block explorer API. """ -import pandas as pd -import requests import time from typing import Dict, List, Tuple +import pandas as pd +import requests + CONTRACTS = { "Store": "0xB6881e8b21a3cd6D23c4F90724E26e35BB8980bE", "Donations": "0x6aD9E8e5236C0E2cF6D755Bb7BE4eABCbC03f76d" @@ -204,12 +205,12 @@ def main() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: # Add delay between contracts if contract_name != list(CONTRACTS.keys())[-1]: - print(f"\nā³ Waiting before fetching next contract...") + print("\nā³ Waiting before fetching next contract...") time.sleep(1) # Create combined DataFrame print(f"\n{'='*60}") - print(f"šŸ“Š COMBINED ANALYSIS") + print("šŸ“Š COMBINED ANALYSIS") print(f"{'='*60}") if all_transactions: @@ -230,15 +231,15 @@ def main() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: print(f"šŸ”§ Unique methods across all contracts: {stats['unique_methods']}") print(f"šŸ’° Total fees paid: {stats['total_fees_eth']:.8f} ETH") - print(f"\nšŸ“Š Transactions by contract:") + print("\nšŸ“Š Transactions by contract:") for contract, count in stats['transactions_by_contract'].items(): print(f" {contract}: {count}") - print(f"\nšŸ“ˆ Top methods across all contracts:") + print("\nšŸ“ˆ Top methods across all contracts:") for method, count in list(stats['methods_breakdown'].items())[:10]: print(f" {method}: {count}") - print(f"\nšŸ” Sample transactions from combined dataset:") + print("\nšŸ” Sample transactions from combined dataset:") sample_cols = ['contract_name', 'date', 'method', 'market_item', 'fee_value'] print(transactions_df[sample_cols].head(10).to_string(index=False)) @@ -246,12 +247,12 @@ def main() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: store_df = transactions_df[transactions_df['contract_name'] == 'Store'].copy() donations_df = transactions_df[transactions_df['contract_name'] == 'Donations'].copy() - print(f"\nšŸ’” Individual DataFrames created:") + print("\nšŸ’” Individual DataFrames created:") print(f" store_df: {len(store_df)} transactions") print(f" donations_df: {len(donations_df)} transactions") print(f" transactions_df: {len(transactions_df)} transactions (combined)") - print(f"\nšŸŽ‰ Multi-contract data collection complete!") + print("\nšŸŽ‰ Multi-contract data collection complete!") return transactions_df, store_df, donations_df diff --git a/scripts/fetch_mezo_users.py b/scripts/fetch_mezo_users.py index a00c06d..e59aefd 100644 --- a/scripts/fetch_mezo_users.py +++ b/scripts/fetch_mezo_users.py @@ -1,5 +1,4 @@ from datetime import date -import os from dotenv import load_dotenv import pandas as pd @@ -36,8 +35,8 @@ def fetch_btc_users(users): @with_progress("Saving to csv") def save_to_csv(df, name): - os.makedirs('./outputs', exist_ok=True) - output_path = f'./outputs/{name}_{date.today()}.csv' + # os.makedirs('../outputs', exist_ok=True) + output_path = f'../mezo-analytics-1/outputs/{name}_{date.today()}.csv' df.to_csv(output_path) print(f"Saved CSV to {output_path}") diff --git a/scripts/process_dapp_data.py b/scripts/process_dapp_data.py index e92fc89..e80fdca 100644 --- a/scripts/process_dapp_data.py +++ b/scripts/process_dapp_data.py @@ -10,14 +10,15 @@ 5. Uploads aggregated data to BigQuery marts dataset """ +from datetime import datetime + from dotenv import load_dotenv import pandas as pd -import os import requests -from datetime import datetime + from mezo.clients import BigQueryClient -from mezo.visual_utils import ProgressIndicators, ExceptionHandler, with_progress -from mezo.datetime_utils import format_datetimes +from mezo.visual_utils import ExceptionHandler, ProgressIndicators, with_progress + @with_progress("Fetching transactions from API") def fetch_transactions_data(api_url: str) -> dict: @@ -247,10 +248,10 @@ def main(): ProgressIndicators.print_step("Summary statistics calculated", "success") # Display sample data for verification - print(f"\nšŸ“„ Sample Raw Data (first 3 rows):") + print("\nšŸ“„ Sample Raw Data (first 3 rows):") print(raw_transactions_df[['transaction_hash', 'timestamp', 'amount', 'transaction_type']].head(3)) - print(f"\nšŸ“Š Daily Aggregations Summary:") + print("\nšŸ“Š Daily Aggregations Summary:") print(daily_aggregations_df[['date'] + [col for col in daily_aggregations_df.columns if 'total' in col]].head(3)) ProgressIndicators.print_summary_box( @@ -270,7 +271,7 @@ def main(): except Exception as e: ProgressIndicators.print_step(f"Critical error in main processing: {str(e)}", "error") ProgressIndicators.print_header("āŒ PROCESSING FAILED") - print(f"\nšŸ“ Error traceback:") + print("\nšŸ“ Error traceback:") print(f"{'ā”€' * 50}") import traceback traceback.print_exc() diff --git a/scripts/process_musd_data.py b/scripts/process_musd_data.py index e49ba8d..7d6d2df 100644 --- a/scripts/process_musd_data.py +++ b/scripts/process_musd_data.py @@ -1,18 +1,16 @@ +from datetime import datetime + from dotenv import load_dotenv import pandas as pd -from datetime import datetime import requests + +from mezo.clients import BigQueryClient, SubgraphClient from mezo.currency_utils import Conversions +from mezo.data_utils import add_cumulative_columns, add_pct_change_columns, add_rolling_values from mezo.datetime_utils import format_datetimes -from mezo.data_utils import ( - add_rolling_values, - add_pct_change_columns, - add_cumulative_columns -) -from mezo.clients import BigQueryClient, SubgraphClient from mezo.queries import MUSDQueries -from mezo.visual_utils import ProgressIndicators, ExceptionHandler, with_progress from mezo.test_utils import tests +from mezo.visual_utils import ExceptionHandler, ProgressIndicators, with_progress # ================================================== # helper functions diff --git a/scripts/process_pools_data.py b/scripts/process_pools_data.py index 26e5d35..ac83dab 100644 --- a/scripts/process_pools_data.py +++ b/scripts/process_pools_data.py @@ -460,7 +460,7 @@ def calculate_fee_metrics(fees_df): df = fees_df.copy() df = df.sort_values(['pool', 'timestamp']) - df['total_fees_usd'] = df['totalFees0_usd'] + df['totalFees1_usd'] + df['total_fees_usd'] = df['totalFees0_usd'].fillna(0) + df['totalFees1_usd'].fillna(0) daily_pool_fees = df.groupby(['pool', 'timestamp']).agg({ 'totalFees0_usd': 'sum', @@ -597,11 +597,13 @@ def main(test_mode=False, sample_size=False, skip_bigquery=False): if test_mode: snapshots = [ (tvl_snapshot, 'm_pools_tvl_snapshot'), - (efficiency_metrics, 'm_pools_efficiency') + (efficiency_metrics, 'm_pools_efficiency'), + (daily_pool_fees, 'm_pools_daily_fees_by_pool'), + (daily_pool_fees_all, 'm_pools_daily_fees') ] for dataset, name in snapshots: - dataset.to_csv(f'{name}.csv') + dataset.to_csv(f"/Users/laurenjackson/Desktop/mezo-analytics-1/outputs/{name}.csv") print(tvl_snapshot) print(daily_pool_tvl[daily_pool_tvl['date'] >= current_date - timedelta(days=7)]) @@ -744,4 +746,5 @@ def main(test_mode=False, sample_size=False, skip_bigquery=False): # test = tests() # results = tests.quick_test(sample_size=500) # test.inspect_data(results) - # tests.save_test_outputs(results) \ No newline at end of file + # tests.save_test_outputs(results) + diff --git a/scripts/process_swaps_data.py b/scripts/process_swaps_data.py index c8f67dc..42bbe21 100644 --- a/scripts/process_swaps_data.py +++ b/scripts/process_swaps_data.py @@ -1,14 +1,17 @@ +from datetime import datetime + from dotenv import load_dotenv import pandas as pd -from datetime import datetime + +from mezo.clients import BigQueryClient, SubgraphClient +from mezo.currency_config import POOL_TOKEN_PAIRS, POOLS_MAP from mezo.currency_utils import Conversions from mezo.datetime_utils import format_datetimes -from mezo.clients import BigQueryClient, SubgraphClient from mezo.queries import MUSDQueries -from mezo.test_utils import tests from mezo.report_utils import save_metrics_snapshot -from mezo.currency_config import POOL_TOKEN_PAIRS, POOLS_MAP -from mezo.visual_utils import ProgressIndicators, ExceptionHandler, with_progress + +# from mezo.test_utils import tests +from mezo.visual_utils import ExceptionHandler, ProgressIndicators, with_progress ################################################ # HELPER FUNCTIONS @@ -27,14 +30,6 @@ def clean_swap_and_fee_data(raw): df = format_datetimes(df, ['timestamp_']) df = conv.map_pool_to_tokens(df, pool_column='contractId_', pool_token_mapping=POOL_TOKEN_PAIRS) - # determine which amount columns to process based on swap flag - # if swap: - # amount0_cols = [col for col in df.columns if col.startswith('amount0') and col in ['amount0In', 'amount0Out']] - # amount1_cols = [col for col in df.columns if col.startswith('amount1') and col in ['amount1In', 'amount1Out']] - # else: - # amount0_cols = ['amount0'] - # amount1_cols = ['amount1'] - amount0_cols = [col for col in df.columns if col.startswith(('amount0'))] amount1_cols = [col for col in df.columns if col.startswith(('amount1'))] @@ -107,7 +102,7 @@ def get_daily_swaps_by_pool(df): @with_progress("Aggregating pool-level metrics") def get_swaps_by_pool(df): - """Aggregate swap data by pool""" + """Aggregate swap data by pool, ensuring all pools from POOLS_MAP are included""" pool_metrics = df.groupby('pool').agg( total_volume=('total_volume', 'sum'), total_fees=('total_fees', 'sum'), @@ -116,6 +111,23 @@ def get_swaps_by_pool(df): avg_swap_size=('total_volume', 'mean') ).reset_index() + # Ensure all pools from POOLS_MAP are included, even if they have no swaps + all_pools = set(POOLS_MAP.values()) + existing_pools = set(pool_metrics['pool'].unique()) + missing_pools = all_pools - existing_pools + + if missing_pools: + # Create rows for pools with no swaps (zero values) + missing_rows = pd.DataFrame({ + 'pool': list(missing_pools), + 'total_volume': 0.0, + 'total_fees': 0.0, + 'swap_count': 0, + 'users': 0, + 'avg_swap_size': 0.0 + }) + pool_metrics = pd.concat([pool_metrics, missing_rows], ignore_index=True) + pool_metrics = pool_metrics.sort_values('total_volume', ascending=False) return pool_metrics @@ -258,10 +270,12 @@ def main(test_mode=False, sample_size=False, skip_bigquery=False): ProgressIndicators.print_step("Uploading clean data to BigQuery", "start") clean_datasets = [ - (swaps_df_clean, 'swaps_clean', 'transactionHash_'), - (fees_df_clean, 'swap_fees_clean', 'transactionHash_') + (swaps_df_clean, 'stg_swaps_clean', 'transactionHash_'), + (fees_df_clean, 'stg_swap_fees_clean', 'transactionHash_') ] + print(fees_df_clean['pool'].unique()) + for dataset, table_name, id_column in clean_datasets: if dataset is not None and len(dataset) > 0: bq.update_table(dataset, 'staging', table_name, id_column) @@ -280,6 +294,8 @@ def main(test_mode=False, sample_size=False, skip_bigquery=False): suffixes=('', '_fee') ) + print(swaps_with_fees['pool'].unique()) + # Select and rename columns for clarity col_map = { 'timestamp_': 'timestamp', @@ -319,14 +335,20 @@ def main(test_mode=False, sample_size=False, skip_bigquery=False): pool_daily_metrics = create_swaps_daily_metrics(swaps_final) summary_metrics = create_summary_metrics(swaps_final, daily_metrics) + int_swaps_with_fees.to_csv('int_swaps_with_fees.csv') + swaps_final.to_csv('swaps_final.csv') + daily_metrics.to_csv('daily_metrics.csv') + pool_daily_metrics.to_csv('pool_daily_metrics.csv') + summary_metrics.to_csv('summary_metrics.csv') + if not skip_bigquery: ProgressIndicators.print_step("Uploading aggregated data to BigQuery", "start") analytics_datasets = [ (int_swaps_with_fees, 'intermediate', 'int_swaps_with_fees', 'transactionHash_'), - (swaps_final, 'marts', 'swaps_with_metrics', 'transactionHash_'), - (daily_metrics, 'marts', 'swap_daily_metrics', 'date'), - (pool_daily_metrics, 'marts', 'swap_pool_daily_metrics', 'date') + (swaps_final, 'marts', 'm_swaps_with_metrics', 'transactionHash_'), + (daily_metrics, 'marts', 'm_swap_daily_metrics', 'date'), + (pool_daily_metrics, 'marts', 'm_swap_pool_daily_metrics', 'date') ] for dataset, schema, table_name, id_column in analytics_datasets: diff --git a/scripts/process_vaults_data.py b/scripts/process_vaults_data.py index cbe743d..d005f79 100644 --- a/scripts/process_vaults_data.py +++ b/scripts/process_vaults_data.py @@ -1,12 +1,12 @@ import pandas as pd -from dotenv import load_dotenv -import os + +from mezo.clients import BigQueryClient, SubgraphClient from mezo.currency_utils import Conversions -from mezo.test_utils import tests -from mezo.datetime_utils import format_datetimes from mezo.data_utils import add_cumulative_columns, add_rolling_values -from mezo.clients import BigQueryClient, SubgraphClient +from mezo.datetime_utils import format_datetimes from mezo.queries import VaultQueries + +# from mezo.test_utils import tests from mezo.visual_utils import ProgressIndicators # ========================================