Scheduled Update #3161
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scheduled Update | |
| on: | |
| schedule: | |
| - cron: "0 */2 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: scheduled-updates | |
| cancel-in-progress: false | |
| jobs: | |
| update: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v7 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 1 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Validate JSON (pre-check) | |
| run: python scripts/validate_json.py | |
| - name: Run smart aggregator | |
| run: python scripts/smart_aggregator.py | |
| - name: Validate and clean all lists | |
| run: python scripts/validate_and_clean.py | |
| - name: Validate JSON (post-check) | |
| run: python scripts/validate_json.py | |
| - name: Update count badges | |
| run: python scripts/update_counts.py | |
| - name: Convert JSON to TXT | |
| run: python scripts/json_to_txt.py | |
| - name: Build rootlists | |
| run: python scripts/build_rootlist.py | |
| - name: Build array shards | |
| run: python scripts/build_shards.py | |
| - name: Commit and push | |
| run: | | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git config user.name "github-actions[bot]" | |
| git add list.json list.txt count.json \ | |
| community/blocklist.json community/blocklist.txt \ | |
| community/live_blocklist.json community/live_blocklist.txt \ | |
| community/content_live.json community/content_live.txt \ | |
| community/count.json community/live_count.json \ | |
| community/content_active_count.json \ | |
| community/state.json community/commit_message.txt \ | |
| dns/*.json dns/*.txt \ | |
| rootlist/ | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| exit 0 | |
| fi | |
| CHANGES=$(git diff --staged --stat | tail -1) | |
| git diff --staged --name-only > /tmp/staged_files.txt | |
| tar -cf /tmp/processed.tar -T /tmp/staged_files.txt | |
| for i in {1..5}; do | |
| git fetch origin main | |
| git reset --hard origin/main | |
| tar -xf /tmp/processed.tar | |
| git add $(cat /tmp/staged_files.txt | tr '\n' ' ') | |
| if git diff --staged --quiet; then | |
| echo "Changes already present in remote" | |
| exit 0 | |
| fi | |
| git commit -m "Daily update: ${CHANGES} [skip ci]" | |
| git push && exit 0 | |
| echo "Push attempt $i failed, retrying in $((i * 2))s..." | |
| sleep $((i * 2)) | |
| done | |
| exit 1 | |
| - name: Mirror to Codeberg | |
| env: | |
| CODEBERG_TOKEN: ${{ secrets.CODEBERG_TOKEN }} | |
| run: | | |
| if [ -z "$CODEBERG_TOKEN" ]; then exit 0; fi | |
| REPO_NAME="${GITHUB_REPOSITORY##*/}" | |
| BRANCH="${GITHUB_REF_NAME}" | |
| git remote add codeberg https://phishdestroy:${CODEBERG_TOKEN}@codeberg.org/phishdestroy/${REPO_NAME}.git 2>/dev/null || git remote set-url codeberg https://phishdestroy:${CODEBERG_TOKEN}@codeberg.org/phishdestroy/${REPO_NAME}.git | |
| if git ls-remote codeberg HEAD 2>/dev/null | grep -q HEAD; then | |
| git push codeberg HEAD:${BRANCH} --force || echo "Mirror push failed, continuing" | |
| else | |
| git fetch --unshallow 2>/dev/null || true | |
| git push codeberg HEAD:${BRANCH} --force || echo "Mirror push failed, continuing" | |
| fi |