website-docs/.github/workflows/docsearch-all-stable-full.yml at master · pingcap/website-docs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
name: Doc Search Scripts (All Stable, Full)

on:
  workflow_dispatch:
    inputs:
      language:
        description: 'Language scope for full crawl'
        required: true
        default: both
        type: choice
        options:
          - both
          - en
          - zh
      scraper_repo:
        description: 'Docsearch-scraper repo used to patch workflow image'
        required: true
        default: 'https://github.com/shczhen/docsearch-scraper.git'
        type: string
      scraper_ref:
        description: 'Docsearch-scraper branch/ref used to patch workflow image'
        required: true
        default: 'incrementalCrawl'
        type: string

permissions:
  contents: write

concurrency:
  group: docsearch-all-stable-full
  cancel-in-progress: false

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
        with:
          ref: 'doc-search'

      - name: Login to Container Registry
        uses: docker/login-action@v1
        with:
          registry: ${{ secrets.DOCKER_REGISTRY }}
          username: ${{ secrets.DOCKER_REGISTRY_USERNAME }}
          password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}

      - name: Run scripts
        run: |
          set -o pipefail
          cd docsearch
          touch .env
          echo "APPLICATION_ID=${{ secrets.ALGOLIA_APPLICATION_ID }}" >> .env
          echo "API_KEY=${{ secrets.ALGOLIA_API_KEY }}" >> .env
          echo "GITHUB_AUTH_TOKEN=${{ secrets.GH_TOKEN }}" >> .env
          export GITHUB_AUTH_TOKEN=${{ secrets.GH_TOKEN }}

          export CRAWL_LANG="${{ inputs.language }}"

          export BASE_DOCKER_REGISTRY="${{ secrets.DOCKER_REGISTRY }}"
          export PATCH_SCRAPER_REPO="${{ inputs.scraper_repo }}"
          export PATCH_SCRAPER_REF="${{ inputs.scraper_ref }}"
          export PATCHED_DOCKER_REGISTRY="local"
          export PATCH_IMAGE_ENV_FILE="/tmp/docsearch-patched-image.env"

          ./all-stable/scripts/prepare-patched-scraper-image.sh "$(pwd)/all-stable"
          . "$PATCH_IMAGE_ENV_FILE"
          export DOCKER_REGISTRY="$PATCHED_DOCKER_REGISTRY"

          echo "Run full crawl against production domain with patched image: $PATCHED_SCRAPER_IMAGE"
          ./all-stable/scripts/crawl-full.sh "$(pwd)/all-stable" 2>&1 | tee /tmp/docsearch-all-stable-full.log

          ./all-stable/scripts/check-crawl-errors.sh /tmp/docsearch-all-stable-full.log

      - name: Git push latest_commit
        run: |
          cd docsearch
          cat all-stable/configs/latest_commit.json
          git status
          git config user.name github-actions
          git config user.email github-actions@github.com
          git add all-stable/configs/latest_commit.json
          if git diff --cached --quiet; then
            echo "No latest_commit.json changes."
            exit 0
          fi
          git commit -m "update all-stable latest_commit.json (full)"
          git pull --rebase origin doc-search
          git push origin HEAD:doc-search