JobPilot/.github/workflows/scrape.yml at 21f9b018f7d8be0f219dd5f9379fe5a35aeb5ad9 · Mahi6384/JobPilot · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
name: Scheduled Job Scraper

on:
  schedule:
    - cron: '0 */6 * * *' # Fires every 6 hours; script self-throttles to ~18h
  workflow_dispatch: # Manual trigger button in GitHub UI

jobs:
  scrape:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Install server dependencies
        working-directory: ./server
        run: npm ci

      - name: Install Playwright Chromium
        working-directory: ./server
        run: npx playwright install chromium --with-deps

      - name: Run scraper
        working-directory: ./server
        env:
          MONGODB_URI: ${{ secrets.MONGODB_URI }}
          LINKEDIN_SESSION: ${{ secrets.LINKEDIN_SESSION }}
          CI: 'true'
          NODE_ENV: production
        run: node scraper/index.js