feat(export): Migrate parquet updates to CI + create iceberg metadata files

.github/workflows/export-chunked.yml

@@ -0,0 +1,63 @@
+name: Export Chunked Tables
+
+on:
+  schedule:
+    - cron: '0 */6 * * *'  # Every 6 hours
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to use'
+        required: true
+        type: choice
+        options:
+          - production
+          - staging
+        default: production
+      dry_run:
+        description: 'Dry run only'
+        type: boolean
+        default: false
+
+jobs:
+  export:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Export chunked tables
+        env:
+          # Production secrets
+          CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_PASSWORD }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          # Staging secrets
+          CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.CLICKHOUSE_STAGING_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD }}
+          STAGING_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          STAGING_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          ENVIRONMENT="${{ github.event.inputs.environment || 'production' }}"
+          DRY_RUN_FLAG="${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}"
+
+          python scripts/export_all.py -e $ENVIRONMENT -g chunked -p 8 $DRY_RUN_FLAG
+
+      - name: Export summary
+        if: always()
+        run: |
+          echo "### Export Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- Environment: ${{ github.event.inputs.environment || 'production' }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Type: Chunked tables" >> $GITHUB_STEP_SUMMARY
+          echo "- Dry Run: ${{ github.event.inputs.dry_run || 'false' }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/export-daily.yml

@@ -0,0 +1,72 @@
+name: Export Daily Tables
+
+on:
+  schedule:
+    - cron: '0 4 * * *'  # 4 AM UTC
+  workflow_dispatch:
+    inputs:
+      date:
+        description: 'Date to export (YYYY-MM-DD)'
+        required: false
+        type: string
+      environment:
+        description: 'Environment to use'
+        required: true
+        type: choice
+        options:
+          - production
+          - staging
+        default: production
+      dry_run:
+        description: 'Dry run only'
+        type: boolean
+        default: false
+
+jobs:
+  export:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Export daily tables
+        env:
+          # Production secrets
+          CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_PASSWORD }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          # Staging secrets
+          CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.CLICKHOUSE_STAGING_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD }}
+          STAGING_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          STAGING_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          ENVIRONMENT="${{ github.event.inputs.environment || 'production' }}"
+          DATE="${{ github.event.inputs.date || '' }}"
+          DRY_RUN_FLAG="${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}"
+
+          if [ -n "$DATE" ]; then
+            python scripts/export_all.py -e $ENVIRONMENT -g daily -d $DATE $DRY_RUN_FLAG
+          else
+            python scripts/export_all.py -e $ENVIRONMENT -g daily $DRY_RUN_FLAG
+          fi
+
+      - name: Export summary
+        if: always()
+        run: |
+          echo "### Export Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- Environment: ${{ github.event.inputs.environment || 'production' }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Date: ${{ github.event.inputs.date || 'yesterday' }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Dry Run: ${{ github.event.inputs.dry_run || 'false' }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/export-hourly.yml

@@ -0,0 +1,72 @@
+name: Export Hourly Tables
+
+on:
+  schedule:
+    - cron: '15 * * * *'  # 15 minutes past every hour
+  workflow_dispatch:
+    inputs:
+      date:
+        description: 'Date to export (YYYY-MM-DD)'
+        required: false
+        type: string
+      environment:
+        description: 'Environment to use'
+        required: true
+        type: choice
+        options:
+          - production
+          - staging
+        default: production
+      dry_run:
+        description: 'Dry run only'
+        type: boolean
+        default: false
+
+jobs:
+  export:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Export hourly tables
+        env:
+          # Production secrets
+          CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_PASSWORD }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          # Staging secrets
+          CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.CLICKHOUSE_STAGING_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD }}
+          STAGING_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          STAGING_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          ENVIRONMENT="${{ github.event.inputs.environment || 'production' }}"
+          DATE="${{ github.event.inputs.date || '' }}"
+          DRY_RUN_FLAG="${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}"
+
+          if [ -n "$DATE" ]; then
+            python scripts/export_all.py -e $ENVIRONMENT -g hourly -d $DATE $DRY_RUN_FLAG
+          else
+            python scripts/export_all.py -e $ENVIRONMENT -g hourly $DRY_RUN_FLAG
+          fi
+
+      - name: Export summary
+        if: always()
+        run: |
+          echo "### Export Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- Environment: ${{ github.event.inputs.environment || 'production' }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Date: ${{ github.event.inputs.date || 'last hour' }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Dry Run: ${{ github.event.inputs.dry_run || 'false' }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/export-manual.yml

@@ -0,0 +1,95 @@
+name: Manual Table Export
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment'
+        required: true
+        type: choice
+        options:
+          - staging
+          - production
+      table:
+        description: 'Table name'
+        required: true
+        type: string
+      network:
+        description: 'Network'
+        required: true
+        type: choice
+        options:
+          - mainnet
+          - holesky
+          - sepolia
+      date:
+        description: 'Date (YYYY-MM-DD) for daily/hourly tables'
+        required: false
+        type: string
+      force:
+        description: 'Force re-export even if file exists'
+        type: boolean
+        default: false
+      dry_run:
+        description: 'Dry run only'
+        type: boolean
+        default: false
+
+jobs:
+  export:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Export single table
+        env:
+          # Production secrets
+          CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_PASSWORD }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          # Staging secrets
+          CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.CLICKHOUSE_STAGING_PASSWORD }}
+          EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD: ${{ secrets.EXPERIMENTAL_CLICKHOUSE_STAGING_PASSWORD }}
+          STAGING_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          STAGING_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          ARGS="-e ${{ inputs.environment }} -t ${{ inputs.table }} -n ${{ inputs.network }}"
+
+          if [ -n "${{ inputs.date }}" ]; then
+            ARGS="$ARGS -d ${{ inputs.date }}"
+          fi
+
+          if [ "${{ inputs.force }}" = "true" ]; then
+            ARGS="$ARGS --force"
+          fi
+
+          if [ "${{ inputs.dry_run }}" = "true" ]; then
+            ARGS="$ARGS --dry-run"
+          fi
+
+          echo "Running: python scripts/export_table.py $ARGS"
+          python scripts/export_table.py $ARGS
+
+      - name: Export summary
+        if: always()
+        run: |
+          echo "### Export Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Table: ${{ inputs.table }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Network: ${{ inputs.network }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Date: ${{ inputs.date || 'latest' }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Force: ${{ inputs.force }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Dry Run: ${{ inputs.dry_run }}" >> $GITHUB_STEP_SUMMARY