General cleanups

mezo/__init__.py

@@ -6,6 +6,7 @@
 #     sys.path.insert(0, project_root)
 
 import os
+
 from dotenv import load_dotenv
 
 load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '..', '.env'), override=True)

mezo/clients.py

@@ -94,9 +94,9 @@ def get_subgraph_data(subgraph_url, query, query_key):
     MUSD_STABILITY_POOL_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-stability-pool/1.0.0/gn"
     MUSD_TROVE_MANAGER_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-trove-manager/1.0.0/gn"
     AUGUST_VAULT_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/mezo-vaults-mezo/1.0.0/gn"
-    SWAPS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.0.0/gn"
-    POOLS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.1.0/gn"
-    TIGRIS_POOLS_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/tigris-pools-mezo/1.0.0/gn'
+    SWAPS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.2.0/gn"
+    POOLS_SUBGRAPH = "https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/musd-pools-mezo/1.2.0/gn"
+    TIGRIS_POOLS_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/tigris-pools-mezo/1.2.0/gn'
     WORMHOLE_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/wormhole-bridge-mezo/1.0.0/gn'
     LOLLI_WIT_SUBGRAPH = 'https://api.goldsky.com/api/public/project_cm6ks2x8um4aj01uj8nwg1f6r/subgraphs/btc-as-erc20-mezo/1.0.0/gn'
 
@@ -408,6 +408,29 @@ def __init__(self, key: str = None, project_id: str = None):
 
         self.client = bigquery.Client(project=project_id, credentials=credentials)
 
+    def query(self, query_string: str) -> pd.DataFrame:
+        """
+        Execute a SQL query against BigQuery and return results as a pandas DataFrame.
+
+        Args:
+            query_string: SQL query string to execute
+
+        Returns:
+            pandas.DataFrame: Query results as a DataFrame
+
+        Example:
+            >>> client = BigQueryClient(key="BIGQUERY_KEY", project_id="my-project")
+            >>> df = client.query("SELECT * FROM `my-project.my_dataset.my_table` LIMIT 100")
+        """
+        try:
+            query_job = self.client.query(query_string)
+            df = query_job.to_dataframe()
+            print(f"✅ Query executed successfully. Returned {len(df)} rows.")
+            return df
+        except Exception as e:
+            print(f"❌ Error executing query: {e}")
+            raise
+
     def create_dataset(self, dataset_id: str, location: str = "US"):
         dataset_ref = bigquery.Dataset(self.client.dataset(dataset_id))
         try:

mezo/currency_config.py

@@ -56,25 +56,20 @@
 }
 
 POOLS_MAP = {
-    '0x52e604c44417233b6ccedddc0d640a405caacefb': 'btc_musd_pool',
-    '0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356': 'musdc_musd_pool',
-    '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'musd_musdt_pool',
-    '0x2a1ab0224a7a608d3a992cb15594a2934f74f4c0': 'musdc_musdt_pool',
-    '0xb07c2172c4b7bbc3ac52088d30cb103853b0b403': 'musdc_btc_pool',
-    '0x329d64572f8922c3fe90d23a3c74a360d8ea6235': 'btc_xsolvbtc_pool',
-    '0x58c8f6d2e589928c46425eaf4254b6a41c45a584': 'upmusd_musd_pool',
-    '0x5cd2a025c001e07ae354a4c22c3009908de1ac59': 'solvbtc_musd_pool',
-    '0x9e60cd4d5b718178fab0137200a36a5472191302': 'btc_musdt_pool',
-    '0xb7fd1db5228e4d9f4109c5635f66375e5af0d8f5': 'musd_xsolvbtc_pool',
-    '0xf6f950485b0a65828f07581ca979ef1271778d6a': 'btc_solvbtc_pool',
-    '0xfbcc89586780ac6f41e9cec97663e5592be41331': 'upmusd_btc_pool',
-    '0x58b18C64F0483D6b742c54C23B625c65F7A0974d': 't_musd_pool'
-}
-
-POOL_TOKEN0_MAP = {
-    '0x52e604c44417233b6ccedddc0d640a405caacefb': 'BTC',
-    '0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356': 'USDC',
-    '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'MUSD',
+    "0x52e604c44417233b6ccedddc0d640a405caacefb": "btc_musd_pool",
+    "0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356": "musdc_musd_pool",
+    "0x10906a9e9215939561597b4c8e4b98f93c02031a": "musd_musdt_pool",
+    "0x2a1ab0224a7a608d3a992cb15594a2934f74f4c0": "musdc_musdt_pool",
+    "0xb07c2172c4b7bbc3ac52088d30cb103853b0b403": "musdc_btc_pool",
+    "0x329d64572f8922c3fe90d23a3c74a360d8ea6235": "btc_xsolvbtc_pool",
+    "0x58c8f6d2e589928c46425eaf4254b6a41c45a584": "upmusd_musd_pool",
+    "0x5cd2a025c001e07ae354a4c22c3009908de1ac59": "solvbtc_musd_pool",
+    "0x9e60cd4d5b718178fab0137200a36a5472191302": "btc_musdt_pool",
+    "0xb7fd1db5228e4d9f4109c5635f66375e5af0d8f5": "musd_xsolvbtc_pool",
+    "0xf6f950485b0a65828f07581ca979ef1271778d6a": "btc_solvbtc_pool",
+    "0xfbcc89586780ac6f41e9cec97663e5592be41331": "upmusd_btc_pool",
+    "0x6688f868e9c81ee671867e77fbc618bbea2e9782": "t_musd_pool",
+    "0x72e6b3f126cf4f6c90c08114ac29038a0e269210": "cbbtc_btc_pool"
 }
 
 POOL_TOKEN_PAIRS = {
@@ -90,23 +85,24 @@
     '0xb7fd1db5228e4d9f4109c5635f66375e5af0d8f5': {'token0': 'MUSD', 'token1': 'xSolvBTC'},
     '0xf6f950485b0a65828f07581ca979ef1271778d6a': {'token0': 'BTC', 'token1': 'SolvBTC'},
     '0xfbcc89586780ac6f41e9cec97663e5592be41331': {'token0': 'upMUSD', 'token1': 'BTC'},
-    '0x58b18C64F0483D6b742c54C23B625c65F7A0974d': {'token0': 'T', 'token1': 'MUSD'}
+    '0x6688f868e9c81ee671867e77fbc618bbea2e9782': {'token0': 'T', 'token1': 'MUSD'},
+    '0x72e6b3f126cf4f6c90c08114ac29038a0e269210': {'token0': 'cbBTC', 'token1': 'BTC'}
 }
 
 MEZO_TOKEN_ADDRESSES = {
-    '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'USDT',
-    '0x52e604c44417233b6ccedddc0d640a405caacefb': 'BTC',
-    '0xa10aD2570ea7b93d19fDae6Bd7189fF4929Bc747': 'SolvBTC',
-    '0xdF708431162Ba247dDaE362D2c919e0fbAfcf9DE': 'xSolvBTC',
-    '0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356': 'USDC',
-    '0x10906a9e9215939561597b4c8e4b98f93c02031a': 'MUSD',
-    '0x221B2D9aD7B994861Af3f4c8A80c86C4aa86Bf53': 'upMUSD',
-    '0xaaC423eDC4E3ee9ef81517e8093d52737165b71F': 'T', 
-    '0x29fA8F46CBB9562b87773c8f50a7F9F27178261c': 'swBTC',
-    '0x1531b6e3d51BF80f634957dF81A990B92dA4b154': 'DAI',
-    '0x812fcC0Bb8C207Fd8D6165a7a1173037F43B2dB8': 'FBTC',
-    '0xdf6542260a9F768f07030E4895083F804241F4C4': 'USDe',
-    '0x6a7CD8E1384d49f502b4A4CE9aC9eb320835c5d7': 'cbBTC'
+    "0x10906a9e9215939561597b4c8e4b98f93c02031a": "USDT",
+    "0x52e604c44417233b6ccedddc0d640a405caacefb": "BTC",
+    "0xa10aD2570ea7b93d19fDae6Bd7189fF4929Bc747": "SolvBTC",
+    "0xdF708431162Ba247dDaE362D2c919e0fbAfcf9DE": "xSolvBTC",
+    "0xed812aec0fecc8fd882ac3eccc43f3aa80a6c356": "USDC",
+    "0xdD468A1DDc392dcdbEf6db6e34E89AA338F9F186": "MUSD",
+    "0x221B2D9aD7B994861Af3f4c8A80c86C4aa86Bf53": "upMUSD",
+    "0xaaC423eDC4E3ee9ef81517e8093d52737165b71F": "T",
+    "0x29fA8F46CBB9562b87773c8f50a7F9F27178261c": "swBTC",
+    "0x1531b6e3d51BF80f634957dF81A990B92dA4b154": "DAI",
+    "0x812fcC0Bb8C207Fd8D6165a7a1173037F43B2dB8": "FBTC",
+    "0xdf6542260a9F768f07030E4895083F804241F4C4": "USDe",
+    "0x6a7CD8E1384d49f502b4A4CE9aC9eb320835c5d7": "cbBTC",
 }
 
 TIGRIS_MAP = {

pyproject.toml

@@ -21,13 +21,17 @@ requires-python = "~=3.13.0"
 [tool.ruff]
 line-length = 99
 src = ["mezo"]
-include = ["pyproject.toml", "mezo/**/*.py"]
+include = ["pyproject.toml", "mezo/**/*.py", "docs/**/*.py"]
 
 [tool.ruff.lint]
 extend-select = ["I"]  # Add import sorting
-ignore = ["E501", "W291", "W293"]  # Ignore line-too-long and trailing whitespace
+ignore = ["E501", "W291", "W293", "F401", "F811", "F841"]  # Ignore line-too-long, trailing whitespace, unused imports, and unused variables
+
+[tool.ruff.per-file-ignores]
+"**/docs/models/*.py" = ["F401", "F811", "F821", "F841"]
 
 [tool.ruff.lint.isort]
 known-first-party = ["mezo"]
 force-sort-within-sections = true
 
+

requirements.txt

@@ -15,6 +15,7 @@ python-dotenv>=1.0.0
 
 # Database and API clients
 db-dtypes>=1.4.0
+dune-client>=1.9.3
 google-api-core>=2.20.0
 google-auth>=2.30.0
 google-cloud-bigquery>=3.30.0

scripts/fetch_market_transactions.py

@@ -6,11 +6,12 @@
 MUSD Market smart contracts via the Mezo block explorer API.
 """
 
-import pandas as pd
-import requests
 import time
 from typing import Dict, List, Tuple
 
+import pandas as pd
+import requests
+
 CONTRACTS = {
     "Store": "0xB6881e8b21a3cd6D23c4F90724E26e35BB8980bE",
     "Donations": "0x6aD9E8e5236C0E2cF6D755Bb7BE4eABCbC03f76d"
@@ -204,12 +205,12 @@ def main() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
 
         # Add delay between contracts
         if contract_name != list(CONTRACTS.keys())[-1]:
-            print(f"\n⏳ Waiting before fetching next contract...")
+            print("\n⏳ Waiting before fetching next contract...")
             time.sleep(1)
 
     # Create combined DataFrame
     print(f"\n{'='*60}")
-    print(f"📊 COMBINED ANALYSIS")
+    print("📊 COMBINED ANALYSIS")
     print(f"{'='*60}")
 
     if all_transactions:
@@ -230,28 +231,28 @@ def main() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
         print(f"🔧 Unique methods across all contracts: {stats['unique_methods']}")
         print(f"💰 Total fees paid: {stats['total_fees_eth']:.8f} ETH")
 
-        print(f"\n📊 Transactions by contract:")
+        print("\n📊 Transactions by contract:")
         for contract, count in stats['transactions_by_contract'].items():
             print(f"  {contract}: {count}")
 
-        print(f"\n📈 Top methods across all contracts:")
+        print("\n📈 Top methods across all contracts:")
         for method, count in list(stats['methods_breakdown'].items())[:10]:
             print(f"  {method}: {count}")
 
-        print(f"\n🔍 Sample transactions from combined dataset:")
+        print("\n🔍 Sample transactions from combined dataset:")
         sample_cols = ['contract_name', 'date', 'method', 'market_item', 'fee_value']
         print(transactions_df[sample_cols].head(10).to_string(index=False))
 
         # Create individual contract DataFrames
         store_df = transactions_df[transactions_df['contract_name'] == 'Store'].copy()
         donations_df = transactions_df[transactions_df['contract_name'] == 'Donations'].copy()
 
-        print(f"\n💡 Individual DataFrames created:")
+        print("\n💡 Individual DataFrames created:")
         print(f"  store_df: {len(store_df)} transactions")
         print(f"  donations_df: {len(donations_df)} transactions")
         print(f"  transactions_df: {len(transactions_df)} transactions (combined)")
 
-        print(f"\n🎉 Multi-contract data collection complete!")
+        print("\n🎉 Multi-contract data collection complete!")
 
         return transactions_df, store_df, donations_df
 

scripts/fetch_mezo_users.py

@@ -1,5 +1,4 @@
 from datetime import date
-import os
 
 from dotenv import load_dotenv
 import pandas as pd
@@ -36,8 +35,8 @@ def fetch_btc_users(users):
 
 @with_progress("Saving to csv")
 def save_to_csv(df, name):
-    os.makedirs('./outputs', exist_ok=True)
-    output_path = f'./outputs/{name}_{date.today()}.csv'
+    # os.makedirs('../outputs', exist_ok=True)
+    output_path = f'../mezo-analytics-1/outputs/{name}_{date.today()}.csv'
     df.to_csv(output_path)
     print(f"Saved CSV to {output_path}")
 

scripts/process_dapp_data.py

@@ -10,14 +10,15 @@
 5. Uploads aggregated data to BigQuery marts dataset
 """
 
+from datetime import datetime
+
 from dotenv import load_dotenv
 import pandas as pd
-import os
 import requests
-from datetime import datetime
+
 from mezo.clients import BigQueryClient
-from mezo.visual_utils import ProgressIndicators, ExceptionHandler, with_progress
-from mezo.datetime_utils import format_datetimes
+from mezo.visual_utils import ExceptionHandler, ProgressIndicators, with_progress
+
 
 @with_progress("Fetching transactions from API")
 def fetch_transactions_data(api_url: str) -> dict:
@@ -247,10 +248,10 @@ def main():
         ProgressIndicators.print_step("Summary statistics calculated", "success")
 
         # Display sample data for verification
-        print(f"\n📄 Sample Raw Data (first 3 rows):")
+        print("\n📄 Sample Raw Data (first 3 rows):")
         print(raw_transactions_df[['transaction_hash', 'timestamp', 'amount', 'transaction_type']].head(3))
 
-        print(f"\n📊 Daily Aggregations Summary:")
+        print("\n📊 Daily Aggregations Summary:")
         print(daily_aggregations_df[['date'] + [col for col in daily_aggregations_df.columns if 'total' in col]].head(3))
 
         ProgressIndicators.print_summary_box(
@@ -270,7 +271,7 @@ def main():
     except Exception as e:
         ProgressIndicators.print_step(f"Critical error in main processing: {str(e)}", "error")
         ProgressIndicators.print_header("❌ PROCESSING FAILED")
-        print(f"\n📍 Error traceback:")
+        print("\n📍 Error traceback:")
         print(f"{'─' * 50}")
         import traceback
         traceback.print_exc()

scripts/process_musd_data.py

@@ -1,18 +1,16 @@
+from datetime import datetime
+
 from dotenv import load_dotenv
 import pandas as pd
-from datetime import datetime
 import requests
+
+from mezo.clients import BigQueryClient, SubgraphClient
 from mezo.currency_utils import Conversions
+from mezo.data_utils import add_cumulative_columns, add_pct_change_columns, add_rolling_values
 from mezo.datetime_utils import format_datetimes
-from mezo.data_utils import (
-    add_rolling_values, 
-    add_pct_change_columns, 
-    add_cumulative_columns
-)
-from mezo.clients import BigQueryClient, SubgraphClient
 from mezo.queries import MUSDQueries
-from mezo.visual_utils import ProgressIndicators, ExceptionHandler, with_progress
 from mezo.test_utils import tests
+from mezo.visual_utils import ExceptionHandler, ProgressIndicators, with_progress
 
 # ==================================================
 # helper functions

scripts/process_pools_data.py

@@ -460,7 +460,7 @@ def calculate_fee_metrics(fees_df):
     df = fees_df.copy()
     df = df.sort_values(['pool', 'timestamp'])
 
-    df['total_fees_usd'] = df['totalFees0_usd'] + df['totalFees1_usd']
+    df['total_fees_usd'] = df['totalFees0_usd'].fillna(0) + df['totalFees1_usd'].fillna(0)
 
     daily_pool_fees = df.groupby(['pool', 'timestamp']).agg({
         'totalFees0_usd': 'sum',
@@ -597,11 +597,13 @@ def main(test_mode=False, sample_size=False, skip_bigquery=False):
         if test_mode:            
             snapshots = [
                 (tvl_snapshot, 'm_pools_tvl_snapshot'),
-                (efficiency_metrics, 'm_pools_efficiency')
+                (efficiency_metrics, 'm_pools_efficiency'),
+                (daily_pool_fees, 'm_pools_daily_fees_by_pool'),
+                (daily_pool_fees_all, 'm_pools_daily_fees')
             ]
 
             for dataset, name in snapshots:
-                dataset.to_csv(f'{name}.csv')
+                dataset.to_csv(f"/Users/laurenjackson/Desktop/mezo-analytics-1/outputs/{name}.csv")
 
             print(tvl_snapshot)
             print(daily_pool_tvl[daily_pool_tvl['date'] >=  current_date - timedelta(days=7)])
@@ -744,4 +746,5 @@ def main(test_mode=False, sample_size=False, skip_bigquery=False):
     # test = tests()
     # results = tests.quick_test(sample_size=500)
     # test.inspect_data(results)
-    # tests.save_test_outputs(results)
+    # tests.save_test_outputs(results)
+