scp_crawler/.github/workflows/scp-items.yml at main · scp-data/scp_crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
name: Crawl SCP Wiki

on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

permissions:
  contents: write

jobs:
  update-main-scp:
    runs-on: ubuntu-latest
    steps:
      - name: "Checkout Crawler"
        uses: actions/checkout@v6

      - name: "Clone API Repository"
        uses: actions/checkout@v6
        with:
          repository: heroheman/scp-api
          path: scp-api

      - name: "Setup Python"
        uses: actions/setup-python@v6
        with:
          python-version: '3.13'

      - name: "Install Crawler"
        run: make install

      - name: "Crawl Titles"
        run: make data/scp_titles.json

      - name: "Crawl Hubs"
        run: make data/scp_hubs.json

      - name: "Crawl Items"
        run: make data/scp_items.json

      - name: "Process Items"
        run: make data/processed/items

      - name: "Crawl Tales"
        run: make data/scp_tales.json

      - name: "Process Tales"
        run: make data/processed/tales

      - name: "Crawl GOI"
        run: make data/goi.json

      - name: "Process GOI"
        run: make data/processed/goi

      - name: "Crawl Supplements"
        run: make data/scp_supplement.json

      - name: "Process Supplements"
        run: make data/processed/supplement

      - name: "Move Files into API"
        run: cp -Rf ./data/processed/* ./scp-api/docs/data/scp/

      # - name: "Push"
      #   shell: bash
      #   run: >
      #     cd scp-api;
      #     ./bin/push.sh;

      #   env:
      #     GIT_USER: "SCP Bot"
      #     GIT_EMAIL: "scp@tedivm.com"