Data Processing Pipeline #1482
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Processing Pipeline | |
| on: | |
| schedule: | |
| # Run every 6 hours (at 00:00, 06:00, 12:00, 18:00 UTC) | |
| - cron: '0 */6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| run_bridge_data: | |
| description: 'Run bridge data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_mezo_users: | |
| description: 'Run Mezo users fetching' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_musd_data: | |
| description: 'Run MUSD data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_market_data: | |
| description: 'Run market data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_swaps_data: | |
| description: 'Run swaps data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_pools_data: | |
| description: 'Run pools data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_dapp_data: | |
| description: 'Run DApp data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_vaults_data: | |
| description: 'Run vaults data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_vote_data: | |
| description: 'Run vote data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_token_registrations: | |
| description: 'Run token registrations processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| env: | |
| PYTHON_VERSION: 3.13 | |
| jobs: | |
| process-bridge-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_bridge_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🌉 Process Bridge Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting bridge data processing..." | |
| python scripts/process_bridge_data.py | |
| echo "✅ Bridge data processing completed" | |
| fetch-mezo-users: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_mezo_users == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 👥 Fetch Mezo Users | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting Mezo users fetch..." | |
| python scripts/fetch_mezo_users.py | |
| echo "✅ Mezo users fetch completed" | |
| process-musd-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_musd_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 💰 Process MUSD Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting MUSD data processing..." | |
| python scripts/process_musd_data.py | |
| echo "✅ MUSD data processing completed" | |
| process-market-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_market_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🛒 Process Market Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting market data processing..." | |
| python scripts/process_market_data.py | |
| echo "✅ Market data processing completed" | |
| process-swaps-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_swaps_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🔄 Process Swaps Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting swaps data processing..." | |
| python scripts/process_swaps_data.py | |
| echo "✅ Swaps data processing completed" | |
| process-pools-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_pools_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🏊 Process Pools Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting pools data processing..." | |
| python scripts/process_pools_data.py | |
| echo "✅ Pools data processing completed" | |
| process-dapp-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_dapp_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🏢 Process DApp Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting DApp data processing..." | |
| python scripts/process_dapp_data.py | |
| echo "✅ DApp data processing completed" | |
| process-vaults-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_vaults_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🏦 Process Vaults Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting vaults data processing..." | |
| python scripts/process_vaults_data.py | |
| echo "✅ Vaults data processing completed" | |
| process-vote-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_vote_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🗳️ Process Vote Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting vote data processing..." | |
| python scripts/process_vote_data.py | |
| echo "✅ Vote data processing completed" | |
| process-token-registrations: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_token_registrations == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 📌 Process Token Registrations | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| GOOGLE_CLOUD_KEY: ${{ secrets.GOOGLE_CLOUD_KEY }} | |
| run: | | |
| echo "🚀 Starting token registrations processing..." | |
| python scripts/process_token_registrations.py | |
| echo "✅ Token registrations processing completed" | |
| notify-on-failure: | |
| runs-on: ubuntu-latest | |
| if: always() && contains(needs.*.result, 'failure') | |
| needs: [process-bridge-data, fetch-mezo-users, process-musd-data, process-market-data, process-swaps-data, process-pools-data, process-dapp-data, process-vaults-data, process-vote-data, process-token-registrations] | |
| steps: | |
| - name: 📱 Discord notification on failure | |
| run: | | |
| # Determine which jobs failed | |
| FAILED_JOBS="" | |
| if [[ "${{ needs.process-bridge-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Bridge Data Processing\n" | |
| fi | |
| if [[ "${{ needs.fetch-mezo-users.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Mezo Users Fetch\n" | |
| fi | |
| if [[ "${{ needs.process-musd-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• MUSD Data Processing\n" | |
| fi | |
| if [[ "${{ needs.process-market-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Market Data Processing\n" | |
| fi | |
| if [[ "${{ needs.process-swaps-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Swaps Data Processing\n" | |
| fi | |
| if [[ "${{ needs.process-pools-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Pools Data Processing\n" | |
| fi | |
| if [[ "${{ needs.process-dapp-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• DApp Data Processing\n" | |
| fi | |
| if [[ "${{ needs.process-vaults-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Vaults Data Processing\n" | |
| fi | |
| if [[ "${{ needs.process-vote-data.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Vote Data Processing\n" | |
| fi | |
| if [[ "${{ needs.process-token-registrations.result }}" == "failure" ]]; then | |
| FAILED_JOBS="${FAILED_JOBS}• Token Registrations Processing\n" | |
| fi | |
| curl -X POST -H "Content-Type: application/json" \ | |
| -d '{ | |
| "embeds": [{ | |
| "title": "🚨 Data Processing Pipeline Failed", | |
| "description": "One or more jobs failed in `${{ github.repository }}`", | |
| "color": 15158332, | |
| "fields": [ | |
| { | |
| "name": "Failed Jobs", | |
| "value": "'"${FAILED_JOBS}"'", | |
| "inline": false | |
| }, | |
| { | |
| "name": "Workflow", | |
| "value": "${{ github.workflow }}", | |
| "inline": true | |
| }, | |
| { | |
| "name": "Branch", | |
| "value": "${{ github.ref_name }}", | |
| "inline": true | |
| }, | |
| { | |
| "name": "Triggered by", | |
| "value": "${{ github.event_name }}", | |
| "inline": true | |
| }, | |
| { | |
| "name": "View Logs", | |
| "value": "[Run #${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})", | |
| "inline": false | |
| } | |
| ], | |
| "timestamp": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'" | |
| }] | |
| }' \ | |
| ${{ secrets.DISCORD_WEBHOOK }} |