dats/.github/workflows/backend_checks.yml at b2cd4ab9bef8a68a9f0b1338a59acb9419847a78 · uhh-lt/dats · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
name: backend checks
on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - "backend/**"
      - ".github/workflows/backend_checks.yml"
      - "docker/**"
      - "tools/**"

jobs:
  backend-checks:
    runs-on: self-hosted
    env:
      RAY_HAS_CHANGED: false
    steps:
      - uses: actions/checkout@v3
      - name: Checkout PR
        if: github.event_name == 'pull_request'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh pr checkout ${{ github.event.pull_request.number }}
      - name: Check for changes in ray (pull request)
        id: check_ray_changes_pr
        if: github.event_name == 'pull_request'
        run: |
          git fetch origin ${{ github.event.pull_request.base.ref }}
          base_branch="origin/${{ github.event.pull_request.base.ref }}"
          if git diff --name-only $base_branch HEAD | grep -q -e '^ray/'; then
            echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV
          fi
      - name: Check for changes in ray (push)
        id: check_ray_changes_push
        if: github.event_name == 'push'
        run: |
          if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^ray/'; then
            echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV
          fi
      - name: Prepare environment
        run: |
          ./bin/setup-folders.sh
          ./bin/setup-envs.sh --project_name action-runner --port_prefix 131
          sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
          sed -i 's/^RAY_DEVICE_IDS=.*/RAY_DEVICE_IDS=1/' docker/.env
          if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
            sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
            sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.ray.yml
            sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.yml
            echo "RAY_CACHE_DIR=$HOME/ray_cache" >> docker/.env
            mkdir -p $HOME/ray_cache
            chmod 777 $HOME/ray_cache
          fi
      - name: Build & Start Docker Containers
        working-directory: docker
        run: |
          source .env
          if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
            docker compose -f compose.ray.yml build
            docker compose -f compose.ray.yml up --wait
          fi
          COMPOSE_PROFILES="rq,backend" docker compose build
          COMPOSE_PROFILES="rq" docker compose up --wait --quiet-pull
      - name: Check 1 - pytest runs without errors
        working-directory: docker
        run: |
          source .env
          docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/python -m pytest /dats_code/test
      - name: Check 2 - Database migrates without errors database
        working-directory: docker
        run: |
          source .env
          docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/python src/migrations/run_migrations.py
      - name: Check 3 - Database schema is up-to-date after migration
        working-directory: docker
        run: |
          source .env
          docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/alembic -c src/migrations/alembic.ini check
      - name: Check 4 - install uv environment
        working-directory: backend
        run: |
          uv sync
      - name: Check 5 - pyright runs without errors
        working-directory: backend
        run: |
          uv run pyright
      - name: Start Remaining Docker Containers
        working-directory: docker
        run: |
          source .env
          COMPOSE_PROFILES="rq,backend" docker compose up --wait --quiet-pull
      - name: Check 6 - Test End-2-End importer script
        working-directory: backend
        env:
          TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }}
        run: |
          source .venv/bin/activate
          cd ../tools/importer
          uv pip install -r requirements.txt
          wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip
          unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip
          python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text
          python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image
      - name: Cleanup
        working-directory: docker
        if: always()
        run: |
          source .env
          if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
            docker compose -f compose.ray.yml down -v
            RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env)
            docker rmi $RAY_IMAGE
          fi
          docker compose down -v --remove-orphans
          BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env)
          docker rmi $BACKEND_IMAGE