diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index ef9e17435e..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,510 +0,0 @@ -version: 2.1 - -orbs: - aws-cli: circleci/aws-cli@4.1.1 - aws-ecs: circleci/aws-ecs@4.0.0 - opsgenie: opsgenie/opsgenie@1.0.8 - - -jobs: - frontend-code-test: - resource_class: large - docker: - - image: cimg/node:22.13.0 - working_directory: /home/circleci/tasking-manager - steps: - - checkout - - restore_cache: - keys: - - yarn-deps-{{ checksum "frontend/yarn.lock" }} - - run: - name: Install node dependencies - command: | - yarn --version - cd ${CIRCLE_WORKING_DIRECTORY}/frontend - yarn install - - save_cache: - key: yarn-deps-{{ checksum "frontend/yarn.lock" }} - paths: - - frontend/node_modules - - env - - run: - name: Run yarn test - no_output_timeout: 20m - command: | - cd ${CIRCLE_WORKING_DIRECTORY}/frontend/ - CI=true yarn test -w 1 --silent - CI=true GENERATE_SOURCEMAP=false yarn build - - backend-code-check-PEP8: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - run: - name: flake8 tests - command: | - pip install flake8 - flake8 manage.py backend tests migrations - - backend-code-check-Black: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - run: - name: black tests - command: | - pip install 'black==23.12.1' ## TODO: Update to version 24 - black --check manage.py backend tests migrations - - backend-functional-tests: - resource_class: large - docker: - - - image: cimg/python:3.10 - environment: - SQLALCHEMY_DATABASE_URI: postgresql://taskingmanager@localhost/test_tm - POSTGRES_TEST_DB: test_tm - POSTGRES_USER: taskingmanager - POSTGRES_ENDPOINT: localhost - TM_ORG_CODE: "CICode" - TM_ORG_NAME: "CircleCI Test Organisation" - - - image: cimg/postgres:14.9-postgis - environment: - POSTGRES_USER: taskingmanager - POSTGRES_DB: test_tm - - working_directory: /home/circleci/tasking_manager - steps: - - checkout - - run: sudo apt-get update - - run: sudo apt-get -y install libgeos-dev # Required for shapely - - run: sudo apt-get -y install proj-bin libproj-dev - - run: pip install --upgrade pip uv==0.7.2 - - run: uv export --all-groups > requirements.txt - - run: pip install -r requirements.txt - - run: mkdir --mode 766 -p /tmp/logs - - run: mkdir ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results - - run: find ./tests/backend -name "test*.py" -exec chmod -x {} \; - - run: echo "export TM_LOG_DIR=/tmp/logs" >> $BASH_ENV - - run: coverage erase - - run: - name: Run backend functional tests - command: | - coverage run --source ./backend -m pytest \ - --rootdir ./tests/backend \ - --junit-xml ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/unitresults.xml - - run: coverage xml -o ${CIRCLE_WORKING_DIRECTORY}/tests/backend/results/coverage.xml - - store_test_results: - path: tests/backend/results/unitresults.xml - - store_artifacts: - path: tests/backend/results - - database-backup: - resource_class: large - parameters: - stack_name: - description: "Cloudformation stack name" - type: string - docker: - - image: cimg/postgres:15.4-postgis - steps: - - aws-cli/setup: - role_arn: "arn:aws:iam::$ORG_AWS_ACCOUNT_ID:role/CircleCI-OIDC-Connect" - profile_name: "OIDC-Profile" - role_session_name: "database-snapshot" - session_duration: "2700" - - run: - name: Find the instance ID of the database in the stack to backup - command: | - RDS_ID=$(aws rds describe-db-instances \ - --query 'DBInstances[?contains(TagList[].Key, `aws:cloudformation:stack-name`) && contains(TagList[].Value, `tasking-manager-<< parameters.stack_name >>`)].[DBInstanceIdentifier]' \ - --output text) - echo "export RDS_ID=$RDS_ID" >> $BASH_ENV - echo "RDS ID is: $RDS_ID" - - run: - name: Find Snapshot creation timestamp - command: | - # Given instance ID, find the timestamp of the latest snapshot - SNAPSHOT_TIMESTAMP=$(aws rds describe-db-snapshots \ - --db-instance-identifier=${RDS_ID} \ - --query="max_by(DBSnapshots, &SnapshotCreateTime).OriginalSnapshotCreateTime" \ - --output text) - - run: - name: Make Database Backup - command: | - aws rds wait db-instance-available \ - --db-instance-identifier ${RDS_ID} - # create new aws rds snapshot - printf -v time_now '%(%Y-%m-%d-%H-%M)T' -1 - aws rds create-db-snapshot \ - --db-snapshot-identifier tm4-<< parameters.stack_name >>-${RDS_ID}-${time_now} \ - --db-instance-identifier ${RDS_ID} - aws rds wait db-snapshot-completed \ - --db-snapshot-identifier tm4-<< parameters.stack_name >>-${RDS_ID}-${time_now} \ - --db-instance-identifier ${RDS_ID} - if [[ $? -eq 255 ]]; then - echo "Production snapshot creation failed. Exiting with exit-code 125" - exit 125 - fi - - run: - name: Check / validate backup - command: | - echo "TODO: BACKUP VALIDATION NOT IMPLEMENTED" - - backend_deploy: - parameters: - stack_name: - description: "the name of the stack for cfn-config" - type: string - gitsha: - description: "The 40 char hash of the git commit" - type: string - host_ami: - description: "AMI of the host instance" - type: string - pg_version: - description: "Engine version of PostgreSQL database" - type: string - default: "11.19" - pg_param_group: - description: "Parameter group for RDS PostgreSQL server" - type: string - default: "tm3-logging-postgres11" - db_instance_type: - description: "RDS DB Instance class for the backend database" - type: string - default: "db.t3.xlarge" - backend_instance_type: - description: "Backend EC2 Instance type" - type: string - default: "c6a.large" - working_directory: /home/circleci/tasking-manager - resource_class: medium - docker: - - image: cimg/node:22.13.0 - steps: - - checkout - - aws-cli/setup: - role_arn: "arn:aws:iam::$ORG_AWS_ACCOUNT_ID:role/CircleCI-OIDC-Connect" - profile_name: "OIDC-Profile" - role_session_name: "backend-deploy" - session_duration: "2700" - - run: sudo apt-get update - - run: sudo apt-get -y install libgeos-dev jq - - run: sudo yarn global add @mapbox/cfn-config @mapbox/cloudfriend - - run: - name: Download and patch Cloudformation parameter JSON file - command: | - tmpfile=$(mktemp) - aws s3 cp s3://hot-cfn-config/tasking-manager/tasking-manager-<< parameters.stack_name >>-${AWS_REGION}.cfn.json /tmp/tasking-manager.cfn.json - jq --compact-output \ - --arg GITSHA "<< parameters.gitsha >>" \ - --arg AMI "<< parameters.host_ami >>" \ - --arg PGVERSION "<< parameters.pg_version >>" \ - --arg DBTYPE "<< parameters.db_instance_type >>" \ - --arg EC2TYPE "<< parameters.backend_instance_type >>" \ - --arg DBPARAMG "<< parameters.pg_param_group >>" \ - '.GitSha = $GITSHA | .TaskingManagerBackendAMI = $AMI | .DatabaseEngineVersion = $PGVERSION | .DatabaseInstanceType = $DBTYPE | .DatabaseParameterGroupName = $DBPARAMG | .TaskingManagerBackendInstanceType = $EC2TYPE' \ - /tmp/tasking-manager.cfn.json > "$tmpfile" && mv "$tmpfile" $CIRCLE_WORKING_DIRECTORY/cfn-config-<< parameters.stack_name >>.json - - run: - name: Deploy to << parameters.stack_name >> - command: | - export NODE_PATH=/usr/local/share/.config/yarn/global/node_modules/ - validate-template $CIRCLE_WORKING_DIRECTORY/scripts/aws/cloudformation/tasking-manager.template.js - export JSON_CONFIG="$(< $CIRCLE_WORKING_DIRECTORY/cfn-config-<< parameters.stack_name >>.json)" - cfn-config update << parameters.stack_name >> $CIRCLE_WORKING_DIRECTORY/scripts/aws/cloudformation/tasking-manager.template.js -f -c hot-cfn-config -t hot-cfn-config -r $AWS_REGION -p "$JSON_CONFIG" - - backend_deploy_containers: - working_directory: /home/circleci/tasking-manager - docker: - - image: cimg/python:3.10.7 - steps: - - checkout - - aws-cli/setup: - role_arn: "arn:aws:iam::$ORG_AWS_ACCOUNT_ID:role/CircleCI-OIDC-Connect" - profile_name: "OIDC-Profile" - role_session_name: "backend-deploy-containers" - session_duration: "2700" - - run: sudo apt-get update - - run: sudo apt-get -y install curl - - run: echo "Run AWS Fargate" - - frontend_deploy: - working_directory: /home/circleci/tasking-manager - resource_class: large - docker: - - image: cimg/node:22.13.0 - parameters: - stack_name: - description: "the name of the stack for cfn-config" - type: string - steps: - - checkout - - aws-cli/setup: - role_arn: "arn:aws:iam::$ORG_AWS_ACCOUNT_ID:role/CircleCI-OIDC-Connect" - profile_name: "OIDC-Profile" - role_session_name: "frontend-deploy" - session_duration: "1800" - - run: - name: Deploy Frontend to S3 - command: | - cd ${CIRCLE_WORKING_DIRECTORY}/frontend/ - export TM_ENVIRONMENT=<< parameters.stack_name >> - yarn - CI=true GENERATE_SOURCEMAP=true yarn build - aws s3 sync build/ s3://tasking-manager-<< parameters.stack_name >>-react-app --delete --cache-control max-age=31536000 - aws s3 cp s3://tasking-manager-<< parameters.stack_name >>-react-app s3://tasking-manager-<< parameters.stack_name >>-react-app --recursive --exclude "*" --include "*.html" --metadata-directive REPLACE --cache-control no-cache --content-type text/html - export DISTRIBUTION_ID=`aws cloudformation list-exports --output=text --query "Exports[?Name=='tasking-manager-<< parameters.stack_name >>-cloudfront-id-${AWS_REGION}'].Value"` - aws cloudfront create-invalidation --distribution-id $DISTRIBUTION_ID --paths "/*" - -workflows: - version: 2 - - production-all: - when: - and: - - equal: [ deployment/hot-tasking-manager, << pipeline.git.branch >> ] - jobs: - - database-backup: - name: Backup production database - stack_name: "tm4-production" - context: - - org-global - - tasking-manager-tm4-production - - frontend-code-test - - frontend_wait_for_approval: - type: approval - requires: - - frontend-code-test - - backend-functional-tests - - backend-code-check-PEP8 - - backend-code-check-Black - - backend_wait_for_approval: - type: approval - requires: - - Backup production database - - backend-functional-tests - - backend-code-check-PEP8 - - backend-code-check-Black - - backend_deploy: - name: Deploy backend production - gitsha: $CIRCLE_SHA1 - stack_name: "tm4-production" - host_ami: "/aws/service/debian/release/11/20240813-1838/amd64" - backend_instance_type: c6a.large - pg_version: "13.10" - pg_param_group: "default.postgres13" - db_instance_type: "db.t4g.2xlarge" - requires: - - backend_wait_for_approval - context: - - org-global - - tasking-manager-tm4-production - - frontend_deploy: - name: Deploy frontend production - stack_name: "tm4-production" - requires: - - frontend_wait_for_approval - context: - - org-global - - tasking-manager-tm4-production - - production-frontend-only: - when: - and: - - equal: [ deployment/hot-tasking-manager-frontend, << pipeline.git.branch >> ] - jobs: - - frontend-code-test - - frontend_wait_for_approval: - type: approval - requires: - - frontend-code-test - - frontend_deploy: - name: Deploy frontend production - stack_name: "tm4-production" - requires: - - frontend_wait_for_approval - context: - - org-global - - tasking-manager-tm4-production - - production-backend-only: - when: - and: - - equal: [ deployment/hot-tasking-manager-backend, << pipeline.git.branch >> ] - jobs: - - database-backup: - name: Backup production database - stack_name: "tm4-production" - context: - - org-global - - tasking-manager-tm4-production - - backend-functional-tests - - backend-code-check-PEP8 - - backend-code-check-Black - - backend_wait_for_approval: - type: approval - requires: - - Backup production database - - backend-functional-tests - - backend-code-check-PEP8 - - backend-code-check-Black - - backend_deploy: - name: Deploy backend production - gitsha: $CIRCLE_SHA1 - stack_name: "tm4-production" - host_ami: "/aws/service/debian/release/11/20240813-1838/amd64" - backend_instance_type: c6a.large - pg_version: "13.10" - pg_param_group: "default.postgres13" - db_instance_type: "db.t4g.2xlarge" - requires: - - backend_wait_for_approval - context: - - org-global - - tasking-manager-tm4-production - - teachosm-all: - when: - and: - - equal: [ deployment/teachosm-tasking-manager, << pipeline.git.branch >> ] - jobs: - - database-backup: - name: Backup TeachOSM database - stack_name: "teachosm" - context: - - org-global - - tasking-manager-teachosm - - backend-functional-tests - - backend_deploy: - name: Deploy TeachOSM Backend - gitsha: $CIRCLECI_SHA1 - stack_name: "teachosm" - host_ami: "/aws/service/debian/release/11/20240813-1838/amd64" - requires: - - backend-functional-tests - context: tasking-manager-teachosm - - frontend_deploy: - name: Deploy TeachOSM Frontend - stack_name: "teachosm" - requires: - - backend-functional-tests - context: tasking-manager-teachosm - - staging-all: - when: - and: - - not: - matches: - pattern: "^deployment/.*" - value: << pipeline.git.branch >> - - equal: [ main, << pipeline.git.branch >> ] - jobs: - - database-backup: - name: Backup staging database - stack_name: "staging" - requires: - - backend-code-check-PEP8 - - backend-code-check-Black - - backend-functional-tests - context: - - org-global - - tasking-manager-staging - - frontend-code-test - - backend-code-check-PEP8 - - backend-code-check-Black - - backend-functional-tests - - backend_deploy: - name: Deploy staging backend - gitsha: $CIRCLE_SHA1 - stack_name: "staging" - host_ami: "/aws/service/debian/release/11/20240813-1838/amd64" - pg_version: "14.8" - pg_param_group: "default.postgres14" - db_instance_type: "db.t4g.small" - backend_instance_type: "t3.medium" - requires: - - Backup staging database - - backend-code-check-PEP8 - - backend-code-check-Black - - backend-functional-tests - context: - - org-global - - tasking-manager-staging - - frontend_deploy: - name: Deploy staging frontend - stack_name: "staging" - requires: - - frontend-code-test - context: - - org-global - - tasking-manager-staging - - development-all: - when: - and: - - not: - matches: - pattern: "^deployment/.*" - value: << pipeline.git.branch >> - - or: - ## - equal: [ develop, << pipeline.git.branch >> ] # Disabled while we use dev setup for e2e testing - - equal: [ dev-switch-to-sandbox, << pipeline.git.branch >> ] - jobs: - - database-backup: - name: Backup development database - stack_name: "dev" - requires: - - backend-code-check-PEP8 - - backend-code-check-Black - - backend-functional-tests - context: - - org-global - - tasking-manager-dev - - frontend-code-test - - backend-code-check-PEP8 - - backend-code-check-Black - - backend-functional-tests - - backend_deploy: - name: Deploy development backend - gitsha: $CIRCLE_SHA1 - stack_name: "dev" - host_ami: "/aws/service/debian/release/11/20240813-1838/amd64" - pg_version: "14.10" - pg_param_group: "default.postgres14" - db_instance_type: "db.t4g.small" - backend_instance_type: "t3.medium" - requires: - - Backup development database - - backend-code-check-PEP8 - - backend-code-check-Black - - backend-functional-tests - context: - - org-global - - tasking-manager-dev - - frontend_deploy: - name: Deploy development frontend - stack_name: "dev" - requires: - - frontend-code-test - context: - - org-global - - tasking-manager-dev - - build-only-all: - when: - not: - or: # don't run this workflow for deployment branches - - matches: - pattern: "^deployment/.*" - value: << pipeline.git.branch >> - - equal: [ develop, << pipeline.git.branch >> ] - - equal: [ main, << pipeline.git.branch >> ] - jobs: - - frontend-code-test - - backend-code-check-PEP8 - - backend-code-check-Black - - backend-functional-tests diff --git a/.circleci/rdsid.sh b/.circleci/rdsid.sh deleted file mode 100755 index d707b33db8..0000000000 --- a/.circleci/rdsid.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -ARNS=$(aws rds describe-db-instances --query "DBInstances[].DBInstanceArn" --output text --region us-east-1) -for line in $ARNS; do - TAGS=$(aws rds list-tags-for-resource --resource-name "$line" --query "TagList[]" --region us-east-1) - MATCHES=$(echo $TAGS | python -c "import sys, json; tags = json.loads('$1'); remote = {t['Key']: t['Value'] for t in json.load(sys.stdin)}; print('$line'.split(':')[-1]) if len({'$line' for k, v in tags.items() if k in remote and v == remote[k]}) > 0 else ''") - if [[ ! -z $MATCHES ]]; then - echo $MATCHES - fi -done diff --git a/.github/workflows/add-version-to-db.yml b/.github/workflows/add-version-to-db.yml index cce67fc3bb..1526d023cd 100644 --- a/.github/workflows/add-version-to-db.yml +++ b/.github/workflows/add-version-to-db.yml @@ -6,6 +6,6 @@ jobs: add_release: runs-on: ubuntu-latest steps: - - name: Add release version to db - run: | - curl -X POST "https://tasking-manager-tm4-production-api.hotosm.org/api/v2/system/release/" + - name: Add release version to db + run: | + curl -X POST "https://tasking-manager-production-api.hotosm.org/api/v2/system/release/" diff --git a/.github/workflows/backend-build-deploy.yml b/.github/workflows/backend-build-deploy.yml index 6996b0b792..5b8d399ede 100644 --- a/.github/workflows/backend-build-deploy.yml +++ b/.github/workflows/backend-build-deploy.yml @@ -26,51 +26,11 @@ env: jobs: image-build-and-push: - name: Build Container Images - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - outputs: - image_tags: ${{ steps.meta.outputs.tags }} - - steps: - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set container image metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=${{ github.ref_name }} - type=raw,value=${{ github.sha }} - - - name: Build and push container image - id: build-push-image - uses: docker/build-push-action@v5 - with: - context: "{{defaultContext}}" - target: prod - platforms: linux/amd64,linux/arm64 - file: scripts/docker/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: | - type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ github.ref_name }} - type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-main - type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-staging - type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-develop - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ github.ref_name }},mode=max + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1 + with: + image_name: ghcr.io/${{ github.repository }}/backend + build_target: prod + dockerfile: scripts/docker/Dockerfile deploy-service: name: Deploy ${{ matrix.service }} to ECS diff --git a/.github/workflows/backend-build.yml b/.github/workflows/backend-build.yml deleted file mode 100644 index 981e339820..0000000000 --- a/.github/workflows/backend-build.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Build & publish TM backend container image - -on: - pull_request: - branches: - - main - - staging - - develop - -jobs: - backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1 - with: - image_name: ghcr.io/${{ github.repository }}/backend - build_target: prod - dockerfile: scripts/docker/Dockerfile diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml new file mode 100644 index 0000000000..acd2b6efa0 --- /dev/null +++ b/.github/workflows/pr_test_backend.yml @@ -0,0 +1,54 @@ +name: 🧪 PR Test Backend + +on: + pull_request: + branches: + - main + - staging + - develop + paths: + - "backend/**" + workflow_dispatch: + +jobs: + code-check-PEP8: + name: Run PEP8 code style checks + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + pip install black==25.1.0 + + - name: Run PEP8 checks + run: | + flake8 manage.py backend tests migrations + black --check manage.py backend tests migrations + + pytest: + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@3.2.0 + with: + image_name: ghcr.io/${{ github.repository }}/backend + build_context: . + build_dockerfile: ./scripts/docker/Dockerfile + pre_command: | + cp example.env tasking-manager.env + docker compose up -d tm-db tm-backend tm-migration tm-cron-jobs traefik + docker compose exec tm-db bash -c "PGPASSWORD=tm psql -U tm -d tasking-manager < { expect(screen.getByLabelText('Sample avatar')).toBeInTheDocument(); }); - test("when the user isn't logged in", () => { + test("when the user isn't logged in", async () => { act(() => { store.dispatch({ type: 'SET_USER_DETAILS', @@ -142,8 +142,10 @@ describe('Right side action items', () => { }); }); setup(); - expect(screen.getByRole('combobox')).toBeInTheDocument(); - expect(screen.getByText('English')).toBeInTheDocument(); + await waitFor(() => { + expect(screen.getByRole('combobox')).toBeInTheDocument(); + expect(screen.getByText('English')).toBeInTheDocument(); + }); expect( screen.getByRole('button', { name: /log in/i, diff --git a/frontend/src/components/localeSelect.js b/frontend/src/components/localeSelect.js index 5c716723a9..cd4ad153f1 100644 --- a/frontend/src/components/localeSelect.js +++ b/frontend/src/components/localeSelect.js @@ -3,7 +3,7 @@ import { FormattedMessage } from 'react-intl'; import Select from 'react-select'; import messages from './messages'; -import { supportedLocales } from '../utils/internationalization'; +import { useFetch } from '../hooks/UseFetch'; import { setLocale } from '../store/actions/userPreferences'; function LocaleSelect({ @@ -13,22 +13,30 @@ function LocaleSelect({ removeBorder = true, fullWidth = false, }) { + const [errorLanguages, loadingLanguages, languages] = useFetch('system/languages/'); + + const supportedLanguages = + !errorLanguages && !loadingLanguages ? languages.supportedLanguages : []; + const onLocaleSelect = (arr) => { - setLocale(arr[0].value); + setLocale(arr[0].code); }; const getActiveLanguageNames = () => { const locales = [userPreferences.locale, navigator.language, navigator.language.substr(0, 2)]; let supportedLocaleNames = []; locales.forEach((locale) => - supportedLocales - .filter((i) => i.value === locale) + supportedLanguages + .filter((i) => i.code === locale) .forEach((i) => supportedLocaleNames.push(i)), ); - return supportedLocaleNames.length ? supportedLocaleNames[0].value : 'en'; + return supportedLocaleNames.length ? supportedLocaleNames[0].code : 'en'; }; + // wait till supportedLanguages are fetched + if (!supportedLanguages.length) return <>; + return (