diff --git a/.github/labeler.yml b/.github/labeler.yml index cbd5b0584..d7e0c5df0 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -4,13 +4,9 @@ - "app/shared/jobs/migrations/*" dependencies: - - "poetry.lock" - "pyproject.toml" - - "app/workers/poetry.lock" - - "app/workers/pyproject.toml" - - "app/shared/poetry.lock" + - "app/airflow/pyproject.toml" - "app/shared/pyproject.toml" - - "app/api/poetry.lock" - "app/api/pyproject.toml" - "app/next-client-app/package-lock.json" - "app/next-client-app/package.json" @@ -25,5 +21,5 @@ dependencies: "Area: Frontend": - "app/next-client-app/**" -"Area: Workers": - - "app/workers/**" \ No newline at end of file +"Area: Airflow": + - "app/airflow/**" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7d85d8f1..2f7cd6d9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,9 @@ name: ci on: [pull_request] +permissions: + contents: read + jobs: format_python_code: runs-on: ubuntu-latest diff --git a/.github/workflows/deploy.production.yml b/.github/workflows/deploy.production.yml index c99474560..f714a5a1f 100644 --- a/.github/workflows/deploy.production.yml +++ b/.github/workflows/deploy.production.yml @@ -5,9 +5,7 @@ on: types: [released] env: - AZURE_FUNCTIONAPP_PACKAGE_PATH: "./app/workers" AZURE_WEBAPP_PACKAGE_PATH: "./app/next-client-app" - PYTHON_VERSION: "3.11" NODE_VERSION: 20 app-name: carrot repo-owner: health-informatics-uon @@ -18,43 +16,6 @@ permissions: packages: write # Required to publish packages jobs: - build-and-publish-workers: - runs-on: ubuntu-latest - steps: - - name: "Checkout GitHub Action" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Python ${{ env.PYTHON_VERSION }} Environment - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: "Install UV and Pip" - run: | - python -m pip install --upgrade pip - pip install uv - - - name: "Resolve Project Dependencies Using UV" - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - uv pip compile pyproject.toml --output-file requirements.txt - popd - shell: bash - - - name: "Install Dependencies Using Pip" - shell: bash - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - pip install -r requirements.txt --target=".python_packages/lib/site-packages" - popd - - - name: Upload artifact for deployment job - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: functionsapp - path: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }} - include-hidden-files: true - build-and-publish-web: runs-on: "ubuntu-latest" @@ -191,32 +152,10 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.scheduler_meta.outputs.tags }} - # Deploy Workers Production - deploy-workers-production: - runs-on: ubuntu-latest - needs: build-and-publish-workers - environment: production - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - path: functionsapp - name: functionsapp - - - name: "Run Azure Functions Action" - uses: Azure/functions-action@0bd707f87c0b6385742bab336c74e1afc61f6369 # v1.5.3 - id: fa - with: - app-name: ${{ vars.AZURE_FUNCTIONS_NAME }} - slot-name: "production" - package: ${{ github.workspace }}/functionsapp - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # Deploy Web App Production deploy-web-production: runs-on: ubuntu-latest - needs: [build-and-publish-web, deploy-workers-production] + needs: [build-and-publish-web] environment: name: "production" url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8b7966bc9..fce6e925e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -7,9 +7,7 @@ on: workflow_dispatch: env: - AZURE_FUNCTIONAPP_PACKAGE_PATH: "./app/workers" AZURE_WEBAPP_PACKAGE_PATH: "./app/next-client-app" - PYTHON_VERSION: "3.11" NODE_VERSION: 20 app-name: carrot repo-owner: health-informatics-uon @@ -20,43 +18,6 @@ permissions: packages: write # Required to publish packages jobs: - build-and-publish-workers: - runs-on: ubuntu-latest - steps: - - name: "Checkout GitHub Action" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Python ${{ env.PYTHON_VERSION }} Environment - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: "Install UV and Pip" - run: | - python -m pip install --upgrade pip - pip install uv - - - name: "Resolve Project Dependencies Using UV" - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - uv pip compile pyproject.toml --output-file requirements.txt - popd - shell: bash - - - name: "Install Dependencies Using Pip" - shell: bash - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - pip install -r requirements.txt --target=".python_packages/lib/site-packages" - popd - - - name: Upload artifact for deployment job - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: functionsapp - path: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }} - include-hidden-files: true - build-and-publish-web: runs-on: "ubuntu-latest" @@ -193,32 +154,10 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.scheduler_meta.outputs.tags }} - # Deploy Workers Dev - deploy-workers-dev: - runs-on: ubuntu-latest - needs: [build-and-publish-workers, build-and-publish-web] - environment: dev - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - path: functionsapp - name: functionsapp - - - name: "Run Azure Functions Action" - uses: Azure/functions-action@0bd707f87c0b6385742bab336c74e1afc61f6369 # v1.5.3 - id: fa - with: - app-name: ${{ vars.AZURE_FUNCTIONS_NAME }} - slot-name: "production" - package: ${{ github.workspace }}/functionsapp - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # Deploy Web App Dev deploy-web-dev: runs-on: ubuntu-latest - needs: [build-and-publish-web, deploy-workers-dev] + needs: [build-and-publish-web] environment: name: "dev" url: ${{ steps.deploy-web-dev.outputs.webapp-url }} @@ -330,32 +269,10 @@ jobs: publish-profile: ${{ secrets.AIRFLOW_SCHEDULER_PUBLISH_PROFILE }} images: ${{ env.registry }}/${{ env.repo-owner }}/${{ env.app-name }}/airflow-scheduler:${{ github.sha }} - # Deploy Workers Test - deploy-workers-test: - runs-on: ubuntu-latest - needs: deploy-workers-dev - environment: test - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - path: functionsapp - name: functionsapp - - - name: "Run Azure Functions Action" - uses: Azure/functions-action@0bd707f87c0b6385742bab336c74e1afc61f6369 # v1.5.3 - id: fa - with: - app-name: ${{ vars.AZURE_FUNCTIONS_NAME }} - slot-name: "production" - package: ${{ github.workspace }}/functionsapp - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # Deploy Web App Test deploy-web-test: runs-on: ubuntu-latest - needs: [deploy-web-dev, deploy-workers-test] + needs: [deploy-web-dev] environment: name: "test" url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 82880f328..5d22aae0f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,23 +73,3 @@ jobs: push: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.backend_meta.outputs.tags }} - - # Tag and build workers - - name: Extract workers Docker metadata - id: workers_meta - uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 - with: - images: ${{ env.registry }}/${{ env.repo-owner }}/${{ env.app-name }}/workers - tags: | - ${{ github.event.release.tag_name }} - ${{ github.sha }} - ${{ env.RUN_TIMESTAMP }} - - - name: Build and push workers image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ./app - file: ./app/workers/Dockerfile - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.workers_meta.outputs.tags }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22a22a1cd..af2b34597 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,6 @@ name: test +permissions: + contents: read on: [pull_request] diff --git a/docker-compose.yml b/docker-compose.yml index 7a25f24d7..964f9a78b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,7 +98,7 @@ services: - 8000:8000 environment: - FRONTEND_URL=http://frontend:3000 - - ALLOWED_HOSTS=['localhost', '127.0.0.1','api', 'workers'] + - ALLOWED_HOSTS=['localhost', '127.0.0.1','api'] - DB_ENGINE=django.db.backends.postgresql - DB_HOST=db - DB_PORT=5432 @@ -107,12 +107,7 @@ services: - DB_PASSWORD=postgres - DEBUG=True - RULES_QUEUE_NAME=rules-queue - - WORKERS_UPLOAD_NAME=upload-reports-queue - SECRET_KEY=secret - - WORKERS_URL=http://workers:80 - - WORKERS_RULES_NAME=RulesOrchestrator - - WORKERS_RULES_KEY=rules_key - - WORKERS_RULES_EXPORT_NAME=rules-exports-queue - STORAGE_CONN_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1; - SIGNING_KEY=secret - STORAGE_TYPE=${STORAGE_TYPE:-minio} @@ -146,51 +141,6 @@ services: omop-lite: condition: service_completed_successfully - workers: - profiles: ["azure"] - image: carrot-workers - build: - context: app - dockerfile: workers/Dockerfile - ports: - - 8080:80 - - 7071:80 - environment: - # Set Docker to look for secrets locally - - AzureWebJobsSecretStorageType=files - - IsEncrypted=false - - AzureWebJobsStorage=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1; - - FUNCTIONS_WORKER_RUNTIME=python - - STORAGE_CONN_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1; - - APP_URL=http://api:8000/ - - WORKERS_UPLOAD_NAME=upload-reports-queue - - RULES_QUEUE_NAME=rules-queue - - RULES_FILE_QUEUE_NAME=rules-exports-queue - # The address that can be used to reach the function app from outside - - WEBSITE_HOSTNAME=localhost:7071 - # Database setup - - DB_ENGINE=django.db.backends.postgresql - - DB_HOST=db - - DB_PORT=5432 - - DB_NAME=postgres - - DB_USER=postgres - - DB_PASSWORD=postgres - - PYTHONPATH=/home/site/wwwroot:/home/site/wwwroot/shared - - STORAGE_TYPE=azure - - MINIO_ENDPOINT=minio:9000 - - MINIO_ACCESS_KEY=minioadmin - - MINIO_SECRET_KEY=minioadmin - volumes: - - ./app/workers:/home/site/wwwroot - - ./app/shared:/home/site/wwwroot/shared - # Mount the local secret to Docker container - - ./app/workers/Secrets:/azure-functions-host/Secrets/ - depends_on: - azurite: - condition: service_started - api: - condition: service_healthy - azurite: profiles: ["azure"] image: mcr.microsoft.com/azure-storage/azurite diff --git a/sample-env.txt b/sample-env.txt deleted file mode 100644 index c0dbead23..000000000 --- a/sample-env.txt +++ /dev/null @@ -1,14 +0,0 @@ -ALLOWED_HOSTS=['localhost', '127.0.0.1'] -DB_ENGINE=django.db.backends.postgresql -DB_HOST=localhost -DB_NAME=postgres -DB_PASSWORD=postgres -DB_PORT=5432 -DB_USER=postgres -DEBUG=True -WORKERS_UPLOAD_NAME=upload-reports-queue -SECRET_KEY=secret -STORAGE_CONN_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" -CCOM_APP_URL= http://localhost:8000 -SIGNING_KEY=secret -WORKERS_RULES_KEY=rules-key diff --git a/sample-local.settings.json b/sample-local.settings.json deleted file mode 100644 index 53acb76dc..000000000 --- a/sample-local.settings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "python", - "STORAGE_CONN_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;QueueEndpoint=http://localhost:10001/devstoreaccount1;BlobEndpoint=http://localhost:10000/devstoreaccount1;", - "APP_URL": "http://localhost:8000/", - "AZ_FUNCTION_KEY": "", - "WORKERS_UPLOAD_NAME": "upload-reports-queue", - "RULES_QUEUE_NAME": "rules-queue", - "RULES_FILE_QUEUE_NAME": "rules-exports-queue" - } -}