AngelosAI/.github/workflows/angelos.yml at 871806c2bf5d03f230a314ede8a7affd669303bc · ls1intum/AngelosAI · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
name: Deploy Angelos

on:
  push:
    branches:
      - main
    paths:
      - 'rag/**'
      - '.github/workflows/angelos.yml'

jobs:
  build:
    if: ${{ vars.ENABLE_DEPLOYMENT == 'true' }}
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build and Push Angelos Image
        uses: docker/build-push-action@v6
        with:
          platforms: amd64, arm64
          file: rag/Dockerfile
          context: rag
          tags: ghcr.io/ls1intum/angelosai/angelos-app:latest
          push: true

  deploy:
    if: ${{ vars.ENABLE_DEPLOYMENT == 'true' }}
    needs: build
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      # Copy the main docker-compose file (for Angelos) into the 'rag' folder on the VM
      - name: Copy Docker Compose File to VM
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ vars.VM_HOST }}
          username: ${{ vars.VM_USERNAME }}
          key: ${{ secrets.VM_SSH_PRIVATE_KEY }}
          proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }}
          proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
          proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
          proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
          source: "rag/docker-compose.yml"
          target: "/home/${{ vars.VM_USERNAME }}/${{ github.repository }}"

      # Copy the Weaviate compose file into the appropriate subfolder on the VM
      - name: Copy Weaviate Compose File to VM
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ vars.VM_HOST }}
          username: ${{ vars.VM_USERNAME }}
          key: ${{ secrets.VM_SSH_PRIVATE_KEY }}
          proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }}
          proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
          proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
          proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
          source: "rag/docker/weaviate.yml"
          target: "/home/${{ vars.VM_USERNAME }}/${{ github.repository }}"

      # Copy the default Weaviate env file into its proper folder so that the compose file’s relative path works
      - name: Copy Weaviate Default Env File to VM
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ vars.VM_HOST }}
          username: ${{ vars.VM_USERNAME }}
          key: ${{ secrets.VM_SSH_PRIVATE_KEY }}
          proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }}
          proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
          proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
          proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
          source: "rag/docker/weaviate/default.env"
          target: "/home/${{ vars.VM_USERNAME }}/${{ github.repository }}"

      - name: Copy Weaviate backup script to VM
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ vars.VM_HOST }}
          username: ${{ vars.VM_USERNAME }}
          key: ${{ secrets.VM_SSH_PRIVATE_KEY }}
          proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }}
          proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
          proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
          proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
          source: "rag/weaviate_backup.sh"
          target: "/home/${{ vars.VM_USERNAME }}/${{ github.repository }}"

      # Create the .env.prod file in the 'rag' folder on the VM (so it’s alongside the compose files)
      - name: SSH to VM and create .env.prod
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ vars.VM_HOST }}
          username: ${{ vars.VM_USERNAME }}
          key: ${{ secrets.VM_SSH_PRIVATE_KEY }}
          proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }}
          proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
          proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
          proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
          script: |
            rm /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod || true
            touch /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            # Add relevant environment variables
            echo "WEAVIATE_URL=${{ vars.WEAVIATE_URL }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "WEAVIATE_PORT=${{ vars.WEAVIATE_PORT }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "DELETE_BEFORE_INIT=${{ vars.DELETE_BEFORE_INIT }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "USE_OLLAMA=${{ vars.USE_OLLAMA }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "GPU_URL=${{ vars.GPU_URL }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "GPU_USER=${{ vars.GPU_USER }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "GPU_PASSWORD=${{ secrets.GPU_PASSWORD }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "GPU_MODEL=${{ vars.GPU_MODEL }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "GPU_EMBED_MODEL=${{ vars.GPU_EMBED_MODEL }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "GPU_HOST=${{ vars.GPU_HOST }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "OPENAI_MODEL=${{ vars.OPENAI_MODEL }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "OPENAI_EMBEDDING_MODEL=${{ vars.OPENAI_EMBEDDING_MODEL }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "USE_AZURE=${{ vars.USE_AZURE }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "AZURE_OPENAI_DEPLOYMENT=${{ vars.AZURE_OPENAI_DEPLOYMENT }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "AZURE_OPENAI_EMBEDDING_DEPLOYMENT=${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "AZURE_OPENAI_ENDPOINT=${{ secrets.AZURE_OPENAI_ENDPOINT }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "AZURE_OPENAI_VERSION=${{ vars.AZURE_OPENAI_VERSION }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "COHERE_API_KEY=${{ secrets.COHERE_API_KEY }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "COHERE_API_KEY_MULTI=${{ secrets.COHERE_API_KEY_MULTI }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "COHERE_API_KEY_EN=${{ secrets.COHERE_API_KEY_EN }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod
            echo "ANGELOS_APP_API_KEY=${{ secrets.ANGELOS_SECRET }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod

      - name: SSH to VM and Execute Docker-Compose Up
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ vars.VM_HOST }}
          username: ${{ vars.VM_USERNAME }}
          key: ${{ secrets.VM_SSH_PRIVATE_KEY }}
          proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }}
          proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
          proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
          proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
          script: |
            if ! docker network inspect angelos-network >/dev/null 2>&1; then
              echo "Network 'angelos-network' does not exist, creating it..."
              docker network create angelos-network
            fi
            docker network ls
            docker compose -f /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/docker-compose.yml -f /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/docker/weaviate.yml --env-file=/home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/.env.prod up --pull=always -d --force-recreate --remove-orphans


      - name: SSH to VM and install/run weekly Weaviate backup
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ vars.VM_HOST }}
          username: ${{ vars.VM_USERNAME }}
          key: ${{ secrets.VM_SSH_PRIVATE_KEY }}
          proxy_host: ${{ vars.DEPLOYMENT_GATEWAY_HOST }}
          proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
          proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
          proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
          script: |
            # Make sure the data folder is owned by our user
            sudo chown -R ${VM_USERNAME}:${VM_USERNAME} /home/${VM_USERNAME}/${{ github.repository }}/rag/.docker-data

            # Ensure script is executable
            chmod +x /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/weaviate_backup.sh

            # Install an idempotent cron entry (Sun 05:15)
            CRON_WEAVIATE="15 5 * * 0 /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/weaviate_backup.sh >> /var/log/weaviate_backup.log 2>&1"
            ( crontab -l 2>/dev/null | grep -v weaviate_backup.sh ; echo "$CRON_WEAVIATE" ) | crontab -
            echo '🗓️ Weekly Weaviate backup cron installed.'

            # Run one now
            sudo /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/rag/weaviate_backup.sh