Doc Search Scripts (All Stable, Incremental) #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Doc Search Scripts (All Stable, Incremental) | |
| on: | |
| schedule: | |
| - cron: '00 06 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| scraper_repo: | |
| description: 'Docsearch-scraper repo used to patch workflow image' | |
| required: true | |
| default: 'https://github.com/shczhen/docsearch-scraper.git' | |
| type: string | |
| scraper_ref: | |
| description: 'Docsearch-scraper branch/ref used to patch workflow image' | |
| required: true | |
| default: 'incrementalCrawl' | |
| type: string | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: docsearch-all-stable-incremental | |
| cancel-in-progress: false | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: 'doc-search' | |
| - name: Login to Container Registry | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: ${{ secrets.DOCKER_REGISTRY }} | |
| username: ${{ secrets.DOCKER_REGISTRY_USERNAME }} | |
| password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | |
| - name: Run scripts | |
| run: | | |
| set -o pipefail | |
| cd docsearch | |
| touch .env | |
| echo "APPLICATION_ID=${{ secrets.ALGOLIA_APPLICATION_ID }}" >> .env | |
| echo "API_KEY=${{ secrets.ALGOLIA_API_KEY }}" >> .env | |
| echo "GITHUB_AUTH_TOKEN=${{ secrets.GH_TOKEN }}" >> .env | |
| export GITHUB_AUTH_TOKEN=${{ secrets.GH_TOKEN }} | |
| export BASE_DOCKER_REGISTRY="${{ secrets.DOCKER_REGISTRY }}" | |
| export PATCH_SCRAPER_REPO="${{ inputs.scraper_repo }}" | |
| export PATCH_SCRAPER_REF="${{ inputs.scraper_ref }}" | |
| export PATCHED_DOCKER_REGISTRY="local" | |
| export PATCH_IMAGE_ENV_FILE="/tmp/docsearch-patched-image.env" | |
| ./all-stable/scripts/prepare-patched-scraper-image.sh "$(pwd)/all-stable" | |
| . "$PATCH_IMAGE_ENV_FILE" | |
| export DOCKER_REGISTRY="$PATCHED_DOCKER_REGISTRY" | |
| echo "Run incremental crawl with patched image: $PATCHED_SCRAPER_IMAGE" | |
| ./all-stable/scripts/crawl-incremental.sh "$(pwd)/all-stable" 2>&1 | tee /tmp/docsearch-all-stable-incremental.log | |
| ./all-stable/scripts/check-crawl-errors.sh /tmp/docsearch-all-stable-incremental.log | |
| - name: Git push latest_commit | |
| run: | | |
| cd docsearch | |
| cat all-stable/configs/latest_commit.json | |
| git status | |
| git config user.name github-actions | |
| git config user.email github-actions@github.com | |
| git add all-stable/configs/latest_commit.json | |
| if git diff --cached --quiet; then | |
| echo "No latest_commit.json changes." | |
| exit 0 | |
| fi | |
| git commit -m "update all-stable latest_commit.json" | |
| git pull --rebase origin doc-search | |
| git push origin HEAD:doc-search |