Routine Data Update #905
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Routine Data Update | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '45 9 * * *' | |
| concurrency: | |
| group: data-updates | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| runDataUpdate: | |
| name: Run Data Update | |
| runs-on: ubuntu-latest | |
| env: | |
| REPO_DIR: main | |
| TIMESTAMP: X | |
| steps: | |
| - name: Store timestamp | |
| run: | | |
| echo "TIMESTAMP=$(date -u +"%F-%H")" >> "$GITHUB_ENV" | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| path: ${{ env.REPO_DIR }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| persist-credentials: false | |
| - name: Setup python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: '${{ env.REPO_DIR }}/_visualize/scripts/requirements.txt' | |
| - name: Install dependencies | |
| run: pip install -r $REPO_DIR/_visualize/scripts/requirements.txt | |
| - name: Create GitHub App Installation Token1 | |
| uses: actions/create-github-app-token@v2 | |
| id: app-token1 | |
| with: | |
| app-id: ${{ vars.CLIENT_ID }} | |
| private-key: ${{ secrets.PRIVATE_KEY }} | |
| - name: Run data collection script with App Installation Token | |
| run: | | |
| set -eu | |
| cd $REPO_DIR/_visualize/scripts | |
| ./UPDATE.sh $TAG | |
| env: | |
| GITHUB_API_TOKEN: ${{ steps.app-token1.outputs.token }} | |
| TAG: MEMBERS | |
| - name: Validate members data updates | |
| run: ./$REPO_DIR/.github/scripts/validate.sh | |
| env: | |
| TAG: MEMBERS | |
| - name: Run data collection script with Action Token | |
| run: | | |
| set -eu | |
| cd $REPO_DIR/_visualize/scripts | |
| ./UPDATE.sh $TAG | |
| env: | |
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TAG: CORE | |
| - name: Validate core data updates | |
| run: ./$REPO_DIR/.github/scripts/validate.sh | |
| env: | |
| TAG: CORE | |
| - name: Create GitHub App Installation Token | |
| uses: actions/create-github-app-token@v2 | |
| id: app-token | |
| with: | |
| app-id: ${{ vars.CLIENT_ID }} | |
| private-key: ${{ secrets.PRIVATE_KEY }} | |
| - name: Get GitHub App User ID | |
| id: get-user-id | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" | |
| - name: Configure git | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| gh auth setup-git | |
| git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]' | |
| git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com' | |
| - name: Commit updated data | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| set -eu | |
| cd $REPO_DIR | |
| git stash | |
| git pull --ff-only | |
| git stash pop | |
| git add -A . | |
| git commit -m "$TIMESTAMP Data Update by ${{ steps.app-token.outputs.app-slug }}" | |
| git push | |
| - name: Show health stats | |
| if: ${{ always() }} | |
| run: | | |
| for TAG in MEMBERS CORE; do | |
| cat $REPO_DIR/_visualize/LAST_${TAG}_UPDATE.txt || true | |
| echo "Warning Count: $(grep -c 'Warning' $REPO_DIR/_visualize/LAST_${TAG}_UPDATE.log)" | |
| echo "From Timeouts: $(grep -c 'but failed' $REPO_DIR/_visualize/LAST_${TAG}_UPDATE.log)" | |
| echo "Limit Reached: $(grep -c 'rate limit exceeded' $REPO_DIR/_visualize/LAST_${TAG}_UPDATE.log)" | |
| done | |
| - name: Save log files | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: logfiles_${{ env.TIMESTAMP }}_update | |
| path: | | |
| ${{ env.REPO_DIR }}/_visualize/LAST_MEMBERS_UPDATE.txt | |
| ${{ env.REPO_DIR }}/_visualize/LAST_MEMBERS_UPDATE.log | |
| ${{ env.REPO_DIR }}/_visualize/LAST_CORE_UPDATE.txt | |
| ${{ env.REPO_DIR }}/_visualize/LAST_CORE_UPDATE.log |