destroylist/.github/workflows/on_list_update.yml at e757887a9a9ac6054be288a2dac386125f9e0284 · phishdestroy/destroylist · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
name: On List Update
on:
  push:
    paths:
      - "list.json"
      - "dns/active_domains.json"
      - "dns/content_active.json"
      - "community/blocklist.json"
      - "community/live_blocklist.json"
      - "community/content_live.json"
    branches:
      - main
  workflow_dispatch:

jobs:
  process:
    if: "!contains(github.event.head_commit.message || '', '[skip ci]')"
    runs-on: ubuntu-latest
    timeout-minutes: 60
    permissions:
      contents: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 1

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Validate JSON
        run: python scripts/validate_json.py

      - name: Clean all lists against allowlist
        run: python scripts/validate_and_clean.py

      - name: Convert JSON to TXT
        run: python scripts/json_to_txt.py

      - name: Update domain counts
        run: python scripts/update_counts.py

      - name: Build rootlist
        run: python scripts/build_rootlist.py

      - name: Build array shards
        run: python scripts/build_shards.py

      - name: Commit and push
        run: |
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config user.name "github-actions[bot]"

          git add list.json list.txt count.json \
                 community/blocklist.json community/blocklist.txt \
                 community/live_blocklist.json community/live_blocklist.txt \
                 community/content_live.json community/content_live.txt \
                 community/count.json community/live_count.json \
                 community/content_active_count.json \
                 dns/*.json dns/*.txt \
                 rootlist/

          if git diff --staged --quiet; then
            echo "No changes to commit"
            exit 0
          fi

          # Save processed outputs before any git reset can overwrite them
          git diff --staged --name-only > /tmp/staged_files.txt
          tar -cf /tmp/processed.tar -T /tmp/staged_files.txt

          for i in {1..10}; do
            git fetch origin main
            git reset --hard origin/main

            # Restore our processed file versions on top of latest remote
            tar -xf /tmp/processed.tar

            git add $(cat /tmp/staged_files.txt | tr '\n' ' ')
            if git diff --staged --quiet; then
              echo "Changes already present in remote"
              exit 0
            fi

            git commit -m "Update processed files [skip ci]"
            git push && exit 0

            WAIT=$(( RANDOM % 10 + i * 3 ))
            echo "Push attempt $i failed, retrying in ${WAIT}s..."
            sleep $WAIT
          done
          exit 1