Major data update script and workflow refactoring.

.github/scripts/update.sh → .github/scripts/validate.sh

@@ -6,30 +6,15 @@ set -eu
 
 # From action env:
 #   REPO_DIR
+#   TAG
 
-ACT_LOG_PATH=_visualize/LAST_MASTER_UPDATE.txt
+ACT_LOG_PATH=_visualize/LAST_${TAG}_UPDATE.txt
 ACT_INPUT_PATH=_visualize
 ACT_DATA_PATH=visualize/github-data
-ACT_SCRIPT_PATH=_visualize/scripts
-
-### SETUP ###
-
-# Store absolute path
-cd $REPO_DIR
-REPO_ROOT=$(pwd)
-
-# Store previous END timestamp
-OLD_END=$(cat $ACT_LOG_PATH | grep END | cut -f 2)
-OLD_END=$(date --date="$OLD_END" "+%s")
-
-### RUN MASTER SCRIPT ###
-
-cd $REPO_ROOT/$ACT_SCRIPT_PATH
-./MASTER.sh
 
 ### VALIDATE UPDATE ###
 
-cd $REPO_ROOT
+cd $REPO_DIR
 
 #   Timestamp log changed
 cat $ACT_LOG_PATH
@@ -50,17 +35,6 @@ if [ $(cat $ACT_LOG_PATH | grep -c FAILED) -ne "0" ] || [ $(cat $ACT_LOG_PATH |
         echo "Timestamp log valid"
 fi
 
-#   New START is later than previous END
-NEW_START=$(cat $ACT_LOG_PATH | grep START | cut -f 2)
-NEW_START=$(date --date="$NEW_START" "+%s")
-if [ "$OLD_END" -gt "$NEW_START" ]
-    then
-        echo "UPDATE FAILED - New START is earlier than previous END"
-        exit 1
-    else
-        echo "START timestamp valid"
-fi
-
 #   All changes are to valid files only
 git diff --name-only HEAD
 CHANGE_COUNT=$(git diff --name-only HEAD | grep -c -E ".+")

.github/workflows/cache.yml

@@ -3,12 +3,18 @@ name: Routine Data Cache Request
 on:
   workflow_dispatch:
   schedule:
-    - cron: "45 8 * * *"
+    - cron: '45 8 * * *'
+
+concurrency:
+  group: data-cache
 
 defaults:
   run:
     shell: bash
 
+env:
+  TAG: CACHE
+
 jobs:
   runDataUpdate:
     name: Run Cache Request
@@ -20,35 +26,44 @@ jobs:
       - name: Store timestamp
         run: |
           echo "TIMESTAMP=$(date -u +"%F-%H")" >> "$GITHUB_ENV"
+
       - name: Checkout
         uses: actions/checkout@v6
         with:
           path: ${{ env.REPO_DIR }}
           token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Setup python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.11"
-          cache: "pip"
-          cache-dependency-path: "${{ env.REPO_DIR }}/_visualize/scripts/requirements.txt"
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: '${{ env.REPO_DIR }}/_visualize/scripts/requirements.txt'
+
       - name: Install dependencies
-        run: pip install -r ${{ env.REPO_DIR }}/_visualize/scripts/requirements.txt
-      - name: Run cache script
-        run: ./${{ env.REPO_DIR }}/.github/scripts/cache.sh
+        run: pip install -r $REPO_DIR/_visualize/scripts/requirements.txt
+
+      - name: Run data collection script
+        run: |
+          set -eu
+          cd $REPO_DIR/_visualize/scripts
+          ./UPDATE.sh $TAG
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Show health stats
         if: ${{ always() }}
         run: |
-          cat ${{ env.REPO_DIR }}/_visualize/LAST_CACHE_REQUEST.txt || true
-          echo "Warning Count: $(grep -c 'Warning' ${{ env.REPO_DIR }}/_visualize/LAST_CACHE_REQUEST.log)"
-          echo "From Timeouts: $(grep -c 'but failed' ${{ env.REPO_DIR }}/_visualize/LAST_CACHE_REQUEST.log)"
-          echo "Limit Reached: $(grep -c 'rate limit exceeded' ${{ env.REPO_DIR }}/_visualize/LAST_CACHE_REQUEST.log)"
+          cat $REPO_DIR/_visualize/LAST_$TAG_UPDATE.txt || true
+          echo "Warning Count: $(grep -c 'Warning' $REPO_DIR/_visualize/LAST_$TAG_UPDATE.log)"
+          echo "From Timeouts: $(grep -c 'but failed' $REPO_DIR/_visualize/LAST_$TAG_UPDATE.log)"
+          echo "Limit Reached: $(grep -c 'rate limit exceeded' $REPO_DIR/_visualize/LAST_$TAG_UPDATE.log)"
+
       - name: Save log files
         if: ${{ always() }}
         uses: actions/upload-artifact@v6
         with:
           name: logfiles_${{ env.TIMESTAMP }}_cache
           path: |
-            ${{ env.REPO_DIR }}/_visualize/LAST_CACHE_REQUEST.txt
-            ${{ env.REPO_DIR }}/_visualize/LAST_CACHE_REQUEST.log
+            ${{ env.REPO_DIR }}/_visualize/LAST_${{ env.TAG }}_UPDATE.txt
+            ${{ env.REPO_DIR }}/_visualize/LAST_${{ env.TAG }}_UPDATE.log

.github/workflows/update.yml

@@ -3,7 +3,11 @@ name: Routine Data Update
 on:
   workflow_dispatch:
   schedule:
-    - cron: "45 10 * * *"
+    - cron: '45 10 * * *'
+
+concurrency:
+  group: data-updates
+  cancel-in-progress: true
 
 defaults:
   run:
@@ -20,66 +24,110 @@ jobs:
       - name: Store timestamp
         run: |
           echo "TIMESTAMP=$(date -u +"%F-%H")" >> "$GITHUB_ENV"
+
       - name: Checkout
         uses: actions/checkout@v6
         with:
           path: ${{ env.REPO_DIR }}
           token: ${{ secrets.GITHUB_TOKEN }}
           persist-credentials: false
+
       - name: Setup python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.11"
-          cache: "pip"
-          cache-dependency-path: "${{ env.REPO_DIR }}/_visualize/scripts/requirements.txt"
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: '${{ env.REPO_DIR }}/_visualize/scripts/requirements.txt'
+
       - name: Install dependencies
-        run: pip install -r ${{ env.REPO_DIR }}/_visualize/scripts/requirements.txt
-      - name: Run update script
+        run: pip install -r $REPO_DIR/_visualize/scripts/requirements.txt
+
+      - name: Create GitHub App Installation Token1
+        uses: actions/create-github-app-token@v2
+        id: app-token1
+        with:
+          app-id: ${{ vars.APP_ID }}
+          private-key: ${{ secrets.PRIVATE_KEY }}
+
+      - name: Run data collection script with App Installation Token
+        run: |
+          set -eu
+          cd $REPO_DIR/_visualize/scripts
+          ./UPDATE.sh $TAG
+        env:
+          GITHUB_API_TOKEN: ${{ steps.app-token1.outputs.token }}
+          TAG: MEMBERS
+
+      - name: Validate members data updates
+        run: ./$REPO_DIR/.github/scripts/validate.sh
+        env:
+          TAG: MEMBERS
+
+      - name: Run data collection script with Action Token
+        run: |
+          set -eu
+          cd $REPO_DIR/_visualize/scripts
+          ./UPDATE.sh $TAG
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: ./${{ env.REPO_DIR }}/.github/scripts/update.sh
+          TAG: CORE
+
+      - name: Validate core data updates
+        run: ./$REPO_DIR/.github/scripts/validate.sh
+        env:
+          TAG: CORE
+
       - name: Create GitHub App Installation Token
         uses: actions/create-github-app-token@v2
         id: app-token
         with:
           app-id: ${{ vars.APP_ID }}
           private-key: ${{ secrets.PRIVATE_KEY }}
+
       - name: Get GitHub App User ID
         id: get-user-id
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT"
+
       - name: Configure git
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: |
           gh auth setup-git
           git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]'
           git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com'
+
       - name: Commit updated data
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
         run: |
-          pushd ${{ env.REPO_DIR }}
+          set -eu
+          cd $REPO_DIR
           git stash
           git pull --ff-only
           git stash pop
           git add -A .
-          git commit -m "${{ env.TIMESTAMP }} Data Update by ${{ steps.app-token.outputs.app-slug }}"
+          git commit -m "$TIMESTAMP Data Update by ${{ steps.app-token.outputs.app-slug }}"
           git push
-          popd
+
       - name: Show health stats
         if: ${{ always() }}
         run: |
-          cat ${{ env.REPO_DIR }}/_visualize/LAST_MASTER_UPDATE.txt || true
-          echo "Warning Count: $(grep -c 'Warning' ${{ env.REPO_DIR }}/_visualize/LAST_MASTER_UPDATE.log)"
-          echo "From Timeouts: $(grep -c 'but failed' ${{ env.REPO_DIR }}/_visualize/LAST_MASTER_UPDATE.log)"
-          echo "Limit Reached: $(grep -c 'rate limit exceeded' ${{ env.REPO_DIR }}/_visualize/LAST_MASTER_UPDATE.log)"
+          for TAG in MEMBERS CORE; do
+            cat $REPO_DIR/_visualize/LAST_$TAG_UPDATE.txt || true
+            echo "Warning Count: $(grep -c 'Warning' $REPO_DIR/_visualize/LAST_$TAG_UPDATE.log)"
+            echo "From Timeouts: $(grep -c 'but failed' $REPO_DIR/_visualize/LAST_$TAG_UPDATE.log)"
+            echo "Limit Reached: $(grep -c 'rate limit exceeded' $REPO_DIR/_visualize/LAST_$TAG_UPDATE.log)"
+          done
+
       - name: Save log files
         if: ${{ always() }}
         uses: actions/upload-artifact@v6
         with:
           name: logfiles_${{ env.TIMESTAMP }}_update
           path: |
-            ${{ env.REPO_DIR }}/_visualize/LAST_MASTER_UPDATE.txt
-            ${{ env.REPO_DIR }}/_visualize/LAST_MASTER_UPDATE.log
+            ${{ env.REPO_DIR }}/_visualize/LAST_MEMBERS_UPDATE.txt
+            ${{ env.REPO_DIR }}/_visualize/LAST_MEMBERS_UPDATE.log
+            ${{ env.REPO_DIR }}/_visualize/LAST_CORE_UPDATE.txt
+            ${{ env.REPO_DIR }}/_visualize/LAST_CORE_UPDATE.log

.gitignore

@@ -2,7 +2,6 @@ _site
 Gemfile.lock
 *.pyc
 _visualize/*.log
-_visualize/LAST_CACHE_REQUEST.txt
 .DS_Store
 .vscode/
 .bundle

_visualize/README.md

@@ -2,7 +2,7 @@
 
 ```bash
 cd _visualize/scripts/
-./MASTER.sh
+./UPDATE.sh
 ```
 
 _(Additional script functionality detailed in the [`./scripts` section below][jump2 scripts].)_
@@ -29,7 +29,11 @@ New files are created for each type of data structure.
 For most files, data is overwritten each time the scripts are run.  
 Other scripts may collect cumulative data with a daily timestamp. If one of these scripts is run multiple times in a single day, the entry for that day will be overwritten.
 
-Running [`MASTER.sh`][mastersh] will run all of the necessary scripts in the appropriate order to fetch the latest data. It will also update [`LAST_MASTER_UPDATE.txt`][lastmasterup] to record when this complete data update was last run.
+Running [`UPDATE.sh`][updatesh] will run all of the necessary scripts in the appropriate order to fetch the latest data. It will also update `LAST_FULL_UPDATE.txt` to record when this complete data update was last run.  
+
+You can also run the script with an argument `UPDATE.sh <TAG>` to select a custom set of scripts defined in `UPDATE_<TAG>.txt`.
+(See [`UPDATE_FULL.txt`][updatefull] for the default set of scripts.)
+Timestamps will also be recorded in `LAST_<TAG>_UPDATE.txt` instead.
 
 The scripts are only for gathering new data. You do not need them to run in order to view the webpage visualizations.
 
@@ -39,8 +43,8 @@ The scripts are only for gathering new data. You do not need them to run in orde
 [queries dir]: queries
 [scripts dir]: scripts
 [requires]: scripts/requirements.txt
-[mastersh]: scripts/MASTER.sh
-[lastmasterup]: LAST_MASTER_UPDATE.txt
+[updatesh]: scripts/UPDATE.sh
+[updatefull]: scripts/UPDATE_FULL.txt
 [gitgraphql]: https://developer.github.com/v4/
 [oauth]: https://github.com/settings/developers
 [personaltoken]: https://github.com/settings/tokens