Remove Azure functions from workflows/compose

.github/labeler.yml

@@ -4,13 +4,9 @@
   - "app/shared/jobs/migrations/*"
 
 dependencies:
-  - "poetry.lock"
   - "pyproject.toml"
-  - "app/workers/poetry.lock"
-  - "app/workers/pyproject.toml"
-  - "app/shared/poetry.lock"
+  - "app/airflow/pyproject.toml"
   - "app/shared/pyproject.toml"
-  - "app/api/poetry.lock"
   - "app/api/pyproject.toml"
   - "app/next-client-app/package-lock.json"
   - "app/next-client-app/package.json"
@@ -25,5 +21,5 @@ dependencies:
 "Area: Frontend":
   - "app/next-client-app/**"
 
-"Area: Workers":
-  - "app/workers/**"
+"Area: Airflow":
+  - "app/airflow/**"

.github/workflows/ci.yml

@@ -1,6 +1,9 @@
 name: ci
 on: [pull_request]
 
+permissions:
+  contents: read
+
 jobs:
   format_python_code:
     runs-on: ubuntu-latest

.github/workflows/deploy.production.yml

@@ -5,9 +5,7 @@ on:
     types: [released]
 
 env:
-  AZURE_FUNCTIONAPP_PACKAGE_PATH: "./app/workers"
   AZURE_WEBAPP_PACKAGE_PATH: "./app/next-client-app"
-  PYTHON_VERSION: "3.11"
   NODE_VERSION: 20
   app-name: carrot
   repo-owner: health-informatics-uon
@@ -18,43 +16,6 @@ permissions:
   packages: write # Required to publish packages
 
 jobs:   
-  build-and-publish-workers:
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Checkout GitHub Action"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Setup Python ${{ env.PYTHON_VERSION }} Environment
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: "Install UV and Pip"
-        run: |
-          python -m pip install --upgrade pip
-          pip install uv
-
-      - name: "Resolve Project Dependencies Using UV"
-        run: |
-          pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}'
-          uv pip compile pyproject.toml --output-file requirements.txt
-          popd
-        shell: bash
-
-      - name: "Install Dependencies Using Pip"
-        shell: bash
-        run: |
-          pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}'
-          pip install -r requirements.txt --target=".python_packages/lib/site-packages"
-          popd
-
-      - name: Upload artifact for deployment job
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: functionsapp
-          path: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}
-          include-hidden-files: true
-
   build-and-publish-web:
     runs-on: "ubuntu-latest"
 
@@ -191,32 +152,10 @@ jobs:
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.scheduler_meta.outputs.tags }}
 
-  # Deploy Workers Production
-  deploy-workers-production:
-    runs-on: ubuntu-latest
-    needs: build-and-publish-workers
-    environment: production
-
-    steps:
-      - name: Download artifact from build job
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          path: functionsapp
-          name: functionsapp
-
-      - name: "Run Azure Functions Action"
-        uses: Azure/functions-action@0bd707f87c0b6385742bab336c74e1afc61f6369 # v1.5.3
-        id: fa
-        with:
-          app-name: ${{ vars.AZURE_FUNCTIONS_NAME }}
-          slot-name: "production"
-          package: ${{ github.workspace }}/functionsapp
-          publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE  }}
-
   # Deploy Web App Production
   deploy-web-production:
     runs-on: ubuntu-latest
-    needs: [build-and-publish-web, deploy-workers-production]
+    needs: [build-and-publish-web]
     environment:
       name: "production"
       url: ${{ steps.deploy-to-webapp.outputs.webapp-url }}

.github/workflows/deploy.yml

@@ -7,9 +7,7 @@ on:
   workflow_dispatch:
 
 env:
-  AZURE_FUNCTIONAPP_PACKAGE_PATH: "./app/workers"
   AZURE_WEBAPP_PACKAGE_PATH: "./app/next-client-app"
-  PYTHON_VERSION: "3.11"
   NODE_VERSION: 20
   app-name: carrot
   repo-owner: health-informatics-uon
@@ -20,43 +18,6 @@ permissions:
   packages: write # Required to publish packages
 
 jobs:
-  build-and-publish-workers:
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Checkout GitHub Action"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Setup Python ${{ env.PYTHON_VERSION }} Environment
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: "Install UV and Pip"
-        run: |
-          python -m pip install --upgrade pip
-          pip install uv
-
-      - name: "Resolve Project Dependencies Using UV"
-        run: |
-          pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}'
-          uv pip compile pyproject.toml --output-file requirements.txt
-          popd
-        shell: bash
-
-      - name: "Install Dependencies Using Pip"
-        shell: bash
-        run: |
-          pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}'
-          pip install -r requirements.txt --target=".python_packages/lib/site-packages"
-          popd
-
-      - name: Upload artifact for deployment job
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: functionsapp
-          path: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}
-          include-hidden-files: true
-
   build-and-publish-web:
     runs-on: "ubuntu-latest"
 
@@ -193,32 +154,10 @@ jobs:
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.scheduler_meta.outputs.tags }}
 
-  # Deploy Workers Dev
-  deploy-workers-dev:
-    runs-on: ubuntu-latest
-    needs: [build-and-publish-workers, build-and-publish-web]
-    environment: dev
-
-    steps:
-      - name: Download artifact from build job
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          path: functionsapp
-          name: functionsapp
-
-      - name: "Run Azure Functions Action"
-        uses: Azure/functions-action@0bd707f87c0b6385742bab336c74e1afc61f6369 # v1.5.3
-        id: fa
-        with:
-          app-name: ${{ vars.AZURE_FUNCTIONS_NAME }}
-          slot-name: "production"
-          package: ${{ github.workspace }}/functionsapp
-          publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE  }}
-
   # Deploy Web App Dev
   deploy-web-dev:
     runs-on: ubuntu-latest
-    needs: [build-and-publish-web, deploy-workers-dev]
+    needs: [build-and-publish-web]
     environment:
       name: "dev"
       url: ${{ steps.deploy-web-dev.outputs.webapp-url }}
@@ -330,32 +269,10 @@ jobs:
           publish-profile: ${{ secrets.AIRFLOW_SCHEDULER_PUBLISH_PROFILE }}
           images: ${{ env.registry }}/${{ env.repo-owner }}/${{ env.app-name }}/airflow-scheduler:${{ github.sha }}
 
-  # Deploy Workers Test
-  deploy-workers-test:
-    runs-on: ubuntu-latest
-    needs: deploy-workers-dev
-    environment: test
-
-    steps:
-      - name: Download artifact from build job
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          path: functionsapp
-          name: functionsapp
-
-      - name: "Run Azure Functions Action"
-        uses: Azure/functions-action@0bd707f87c0b6385742bab336c74e1afc61f6369 # v1.5.3
-        id: fa
-        with:
-          app-name: ${{ vars.AZURE_FUNCTIONS_NAME }}
-          slot-name: "production"
-          package: ${{ github.workspace }}/functionsapp
-          publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE  }}
-
   # Deploy Web App Test
   deploy-web-test:
     runs-on: ubuntu-latest
-    needs: [deploy-web-dev, deploy-workers-test]
+    needs: [deploy-web-dev]
     environment:
       name: "test"
       url: ${{ steps.deploy-to-webapp.outputs.webapp-url }}

.github/workflows/publish.yml

@@ -73,23 +73,3 @@ jobs:
           push: true
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.backend_meta.outputs.tags }}
-
-      # Tag and build workers
-      - name: Extract workers Docker metadata
-        id: workers_meta
-        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
-        with:
-          images: ${{ env.registry }}/${{ env.repo-owner }}/${{ env.app-name }}/workers
-          tags: |
-            ${{ github.event.release.tag_name }}
-            ${{ github.sha }}
-            ${{ env.RUN_TIMESTAMP }}
-
-      - name: Build and push workers image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
-        with:
-          context: ./app
-          file: ./app/workers/Dockerfile
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.workers_meta.outputs.tags }}

.github/workflows/test.yml

@@ -1,4 +1,6 @@
 name: test
+permissions:
+  contents: read
 
 on: [pull_request]
 

docker-compose.yml

@@ -98,7 +98,7 @@ services:
       - 8000:8000
     environment:
       - FRONTEND_URL=http://frontend:3000
-      - ALLOWED_HOSTS=['localhost', '127.0.0.1','api', 'workers']
+      - ALLOWED_HOSTS=['localhost', '127.0.0.1','api']
       - DB_ENGINE=django.db.backends.postgresql
       - DB_HOST=db
       - DB_PORT=5432
@@ -107,12 +107,7 @@ services:
       - DB_PASSWORD=postgres
       - DEBUG=True
       - RULES_QUEUE_NAME=rules-queue
-      - WORKERS_UPLOAD_NAME=upload-reports-queue
       - SECRET_KEY=secret
-      - WORKERS_URL=http://workers:80
-      - WORKERS_RULES_NAME=RulesOrchestrator
-      - WORKERS_RULES_KEY=rules_key
-      - WORKERS_RULES_EXPORT_NAME=rules-exports-queue
       - STORAGE_CONN_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1;
       - SIGNING_KEY=secret
       - STORAGE_TYPE=${STORAGE_TYPE:-minio}
@@ -146,51 +141,6 @@ services:
       omop-lite:
         condition: service_completed_successfully
 
-  workers:
-    profiles: ["azure"]
-    image: carrot-workers
-    build:
-      context: app
-      dockerfile: workers/Dockerfile
-    ports:
-      - 8080:80
-      - 7071:80
-    environment:
-      # Set Docker to look for secrets locally
-      - AzureWebJobsSecretStorageType=files
-      - IsEncrypted=false
-      - AzureWebJobsStorage=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1;
-      - FUNCTIONS_WORKER_RUNTIME=python
-      - STORAGE_CONN_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1;
-      - APP_URL=http://api:8000/
-      - WORKERS_UPLOAD_NAME=upload-reports-queue
-      - RULES_QUEUE_NAME=rules-queue
-      - RULES_FILE_QUEUE_NAME=rules-exports-queue
-      # The address that can be used to reach the function app from outside
-      - WEBSITE_HOSTNAME=localhost:7071
-      # Database setup
-      - DB_ENGINE=django.db.backends.postgresql
-      - DB_HOST=db
-      - DB_PORT=5432
-      - DB_NAME=postgres
-      - DB_USER=postgres
-      - DB_PASSWORD=postgres
-      - PYTHONPATH=/home/site/wwwroot:/home/site/wwwroot/shared
-      - STORAGE_TYPE=azure
-      - MINIO_ENDPOINT=minio:9000
-      - MINIO_ACCESS_KEY=minioadmin
-      - MINIO_SECRET_KEY=minioadmin
-    volumes:
-      - ./app/workers:/home/site/wwwroot
-      - ./app/shared:/home/site/wwwroot/shared
-      # Mount the local secret to Docker container
-      - ./app/workers/Secrets:/azure-functions-host/Secrets/
-    depends_on:
-      azurite:
-        condition: service_started
-      api:
-        condition: service_healthy
-
   azurite:
     profiles: ["azure"]
     image: mcr.microsoft.com/azure-storage/azurite