Merge pull request #405 from hotosm/feature/offline-prediction

Feature : Docker optimization and offline prediction

Author: kshitijrajsharma

.dockerignore

@@ -0,0 +1,99 @@
+# Ignore Python cache and virtualenv files
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.pyc
+*.pyo
+.Python
+env/
+venv/
+ENV/
+VENV/
+*.env
+*.venv
+
+backend/env/
+backend/venv/
+backend/ENV/
+backend/VENV/
+backend/*.env
+backend/*.venv
+
+
+# Ignore pip and poetry files/directories
+pip-log.txt
+pip-delete-this-directory.txt
+poetry.lock
+
+# Ignore Django static/media output (customize as needed)
+static/
+media/
+
+# Ignore test and coverage artifacts
+.coverage
+.tox/
+.nox/
+.pytest_cache/
+htmlcov/
+.coverage.*
+.cache
+.mypy_cache/
+.dmypy.json
+.pyre/
+.pytype/
+
+# Ignore log files
+*.log
+
+# Ignore IDE/editor files
+.vscode/
+.idea/
+*.sublime-workspace
+*.sublime-project
+*.swp
+*.swo
+
+# Ignore OS files
+.DS_Store
+Thumbs.db
+
+# Ignore frontend build and dependency directories
+frontend/node_modules/
+frontend/build/
+frontend/dist/
+frontend/.env
+frontend/.next/
+frontend/.cache/
+frontend/.turbo/
+
+# Ignore backend specific (if any)
+backend/node_modules/
+backend/staticfiles/
+backend/.env
+backend/.venv
+backend/.pytest_cache/
+backend/migrations/
+backend/db.sqlite3
+
+# Ignore git
+.git/
+.gitignore
+
+# Ignore Docker files themselves (don't copy into image)
+Dockerfile
+backend/Dockerfile.*
+frontend/Dockerfile.*
+.dockerignore
+
+# Ignore miscellaneous
+*.sqlite3
+*.db
+*.bak
+*.orig
+*.tmp
+tmp/
+*.pem
+*.crt
+*.key
+*.lock

.github/workflows/backend_build.yml

@@ -1,4 +1,4 @@
-name: Build Docker Images for backend
+name: Build Docker Images
 
 on:
   push:
@@ -23,11 +23,12 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-api-image:
+  build-api-image:
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
+      actions: write
     steps:
       - name: Remove unnecessary files
         run: |
@@ -56,20 +57,24 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: backend/
-          file: backend/Dockerfile_CPU
+          file: backend/Dockerfile.API
           push: false
           tags: ${{ steps.meta_api.outputs.tags }}
           labels: ${{ steps.meta_api.outputs.labels }}
           cache-from: type=gha,scope=api,timeout=20m
           cache-to: type=gha,mode=max,scope=api,timeout=20m,ignore-error=true
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-  build-and-push-worker-image:
-    needs: build-and-push-api-image
+  build-worker-image:
+    needs: build-api-image
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        build_type: [cpu, gpu]
     permissions:
       contents: read
       packages: write
+      actions: write
     steps:
       - name: Remove unnecessary files
         run: |
@@ -89,19 +94,69 @@ jobs:
         id: meta_worker
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_worker
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_worker
+          tags: |
+            type=raw,value=latest-${{ matrix.build_type }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Worker Docker image
+      - name: Build and push Worker Docker image (${{ matrix.build_type }})
         uses: docker/build-push-action@v6
         with:
           context: backend/
-          file: backend/Dockerfile
+          file: backend/Dockerfile.workers
           push: false
+          build-args: |
+            BUILD_TYPE=${{ matrix.build_type }}
           tags: ${{ steps.meta_worker.outputs.tags }}
           labels: ${{ steps.meta_worker.outputs.labels }}
-          cache-from: type=gha,scope=docker-worker,timeout=20m
-          cache-to: type=gha,mode=min,scope=docker-worker,timeout=20m,ignore-error=true
+          cache-from: type=gha,scope=docker-worker-${{ matrix.build_type }},timeout=20m
+          cache-to: type=gha,mode=min,scope=docker-worker-${{ matrix.build_type }},timeout=20m,ignore-error=true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  build-offline-predictor-image:
+    runs-on: ubuntu-24.04
+    needs: build-api-image
+    permissions:
+      contents: read
+      packages: write
+      actions: write
+    steps:
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Offline Predictor Docker
+        id: meta_offline_predictor
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_offline_predictor
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push Offline Predictor Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: backend/
+          file: backend/Dockerfile.API
+          push: false
+          build-args: |
+            BUILD_TARGET=predictor
+          tags: ${{ steps.meta_offline_predictor.outputs.tags }}
+          labels: ${{ steps.meta_offline_predictor.outputs.labels }}
+          cache-from: type=gha,scope=offline-predictor,timeout=20m
+          cache-to: type=gha,mode=max,scope=offline-predictor,timeout=20m,ignore-error=true
           github-token: ${{ secrets.GITHUB_TOKEN }}