Iceberg Data Collection and Analysis #306
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Iceberg Data Collection and Analysis | |
| on: | |
| schedule: | |
| # Run daily at 6:00 AM UTC (typical Antarctic data update time) | |
| - cron: "0 6 * * *" | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: # Allow manual triggering | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| jobs: | |
| test: | |
| name: Run Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirement.txt | |
| - name: Run linting | |
| run: | | |
| python -m py_compile src/*.py | |
| python -m py_compile main.py setup.py | |
| - name: Run tests | |
| run: | | |
| python src/tests.py | |
| - name: Test CLI commands | |
| run: | | |
| python main.py info || echo "No data available yet - OK for fresh install" | |
| python main.py animations || echo "No data available yet - OK for fresh install" | |
| echo "Testing project setup..." | |
| python setup.py | |
| collect-data: | |
| name: Collect Iceberg Data | |
| runs-on: ubuntu-latest | |
| needs: test | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirement.txt | |
| - name: Setup project | |
| run: python setup.py | |
| - name: Download existing data | |
| continue-on-error: true | |
| run: | | |
| # Try to download existing data file if it exists | |
| if [ -f data/iceberg_location.json ]; then | |
| echo "Using existing data file" | |
| else | |
| echo "Creating new data file" | |
| mkdir -p data | |
| echo '{}' > data/iceberg_location.json | |
| fi | |
| - name: Scrape iceberg data | |
| run: | | |
| echo "Scraping latest iceberg data..." | |
| python main.py scrape | |
| - name: Generate interactive map | |
| run: | | |
| echo "Generating interactive map..." | |
| python main.py map | |
| - name: Generate API endpoints | |
| run: | | |
| echo "Generating API endpoints for external access..." | |
| python main.py api | |
| - name: Display data summary | |
| run: | | |
| echo "Displaying data summary..." | |
| python main.py info | |
| - name: Create backup | |
| run: | | |
| mkdir -p backups | |
| cp data/iceberg_location.json "backups/iceberg_location_$(date +%Y%m%d_%H%M%S).json" | |
| - name: Upload data as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: iceberg-data-${{ github.run_number }} | |
| path: | | |
| data/iceberg_location.json | |
| output/iceberg_map.html | |
| api/*.json | |
| api/*.jsonp | |
| *.log | |
| retention-days: 90 | |
| - name: Display summary | |
| run: | | |
| echo "Data Collection Summary:" | |
| python main.py info | |
| echo "" | |
| echo "File sizes:" | |
| ls -lh data/*.json output/*.html *.log || true | |
| deploy: | |
| name: Deploy Data | |
| runs-on: ubuntu-latest | |
| needs: collect-data | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: iceberg-data-${{ github.run_number }} | |
| - name: Commit and push data | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| # Add updated files | |
| git add data/iceberg_location.json output/iceberg_map.html api/ | |
| # Check if there are changes to commit | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| else | |
| # Count records for commit message | |
| RECORD_COUNT=$(python -c "import json; data=json.load(open('data/iceberg_location.json')); print(sum(len(v) for v in data.values()))") | |
| DATE=$(date '+%Y-%m-%d') | |
| git commit -m "Update iceberg data - $DATE ($RECORD_COUNT total records)" | |
| git push | |
| echo "Data committed and pushed" | |
| fi |