Append to Historical Data #457
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Append to Historical Data | |
| on: | |
| schedule: | |
| # Run 1 hour after the main scrape (which runs at 9am and 9pm UTC) | |
| # This runs at 10am and 10pm UTC | |
| - cron: '0 10,22 * * *' | |
| workflow_dispatch: # Allow manual trigger | |
| permissions: | |
| contents: write | |
| jobs: | |
| append-historical: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: true | |
| fetch-depth: 0 # Need full history to commit | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| pip install jq | |
| - name: Append incremental data to historical | |
| run: | | |
| python3 append_to_historical.py | |
| - name: Commit and push historical files | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| FILES=(github_flakiness_historical.json gitlab_flakiness_historical.json github_flakiness_current.json) | |
| COMMIT_MSG="chore: update historical retest metrics [bot]" | |
| git add "${FILES[@]}" | |
| if git diff --staged --quiet; then | |
| echo "📊 No changes to historical data" | |
| exit 0 | |
| fi | |
| mapfile -t CHANGED < <(git diff --staged --name-only) | |
| echo "📊 Committing ${#CHANGED[@]} changed file(s) via API..." | |
| git reset HEAD -- . > /dev/null 2>&1 || true | |
| CURRENT_SHA=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq '.object.sha') | |
| BASE_TREE=$(gh api "repos/${GITHUB_REPOSITORY}/git/commits/${CURRENT_SHA}" --jq '.tree.sha') | |
| TREE_JSON="[]" | |
| for file in "${CHANGED[@]}"; do | |
| base64 < "$file" | jq -Rs '{content: ., encoding: "base64"}' > /tmp/blob.json | |
| BLOB_SHA=$(gh api "repos/${GITHUB_REPOSITORY}/git/blobs" \ | |
| --input /tmp/blob.json --jq '.sha') | |
| TREE_JSON=$(echo "$TREE_JSON" | jq \ | |
| --arg path "$file" --arg sha "$BLOB_SHA" \ | |
| '. + [{"path": $path, "mode": "100644", "type": "blob", "sha": $sha}]') | |
| done | |
| NEW_TREE=$(jq -n --arg base "$BASE_TREE" --argjson tree "$TREE_JSON" \ | |
| '{base_tree: $base, tree: $tree}' | \ | |
| gh api "repos/${GITHUB_REPOSITORY}/git/trees" --input - --jq '.sha') | |
| NEW_COMMIT=$(gh api "repos/${GITHUB_REPOSITORY}/git/commits" \ | |
| -f message="${COMMIT_MSG}" \ | |
| -f tree="${NEW_TREE}" \ | |
| -f "parents[]=${CURRENT_SHA}" \ | |
| --jq '.sha') | |
| gh api "repos/${GITHUB_REPOSITORY}/git/refs/heads/main" -X PATCH -f sha="${NEW_COMMIT}" > /dev/null | |
| echo "✅ Committed ${NEW_COMMIT:0:7} (verified/signed by GitHub)" | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: historical-metrics-${{ github.run_number }} | |
| path: | | |
| github_flakiness_historical.json | |
| gitlab_flakiness_historical.json | |
| retention-days: 90 |