Merge pull request #301 from digital-gov-mg/develop

Merge Upgrade to v1.6.3

Author: tokinrandr

.github/workflows/auto-pr-to-release.yml

@@ -18,36 +18,122 @@ on:
         description: 'PR number to process'
         required: true
         default: ''
+      dry_run:
+        description: 'Dry run'
+        required: false
+        default: false
+        type: boolean
 
 jobs:
-  create-pr:
+  resolve-releases:
     if: ${{ github.event_name == 'pull_request' && github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' }}
     runs-on: ubuntu-latest
-
+    outputs:
+      matrix: ${{ steps.resolve-applicable-versions.outputs.matrix }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Get PR details from workflow dispatch
         if: ${{ github.event_name == 'workflow_dispatch' }}
         id: get_pr_details_dispatch
         run: |
           PR_NUMBER=${{ github.event.inputs.pr_number }}
-          PR_DATA=$(gh pr view $PR_NUMBER --json number,headRefName,baseRefName,mergedBy,mergeCommit,author,milestone,title --jq '{number: .number, headRefName: .headRefName, baseRefName: .baseRefName, merger: .mergedBy.login, author: .author.login, milestone: .milestone.title, title: .title}')
-          echo "PR_ID=$(echo $PR_DATA | jq -r '.number')" >> $GITHUB_ENV
-          echo "PR_AUTHOR=$(echo $PR_DATA | jq -r '.author')" >> $GITHUB_ENV
-          echo "PR_MERGER=$(echo $PR_DATA | jq -r '.merger')" >> $GITHUB_ENV
-          echo "MILESTONE=$(echo $PR_DATA | jq -r '.milestone')" >> $GITHUB_ENV
-          echo "BASE_BRANCH=$(echo $PR_DATA | jq -r '.baseRefName')" >> $GITHUB_ENV
-          echo "HEAD_BRANCH=$(echo $PR_DATA | jq -r '.headRefName')" >> $GITHUB_ENV
-          echo "PR_TITLE=$(echo $PR_DATA | jq -r '.title')" >> $GITHUB_ENV
+          PR_DATA=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/opencrvs/opencrvs-countryconfig/pulls/$PR_NUMBER)
+          echo "MILESTONE=$(printf '%s' $PR_DATA | jq -r '.milestone.title')" >> $GITHUB_ENV
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
-          LATEST_COMMIT_SHA=$(gh pr view $PR_NUMBER --json commits --jq '.commits[-1].oid')
-          FIRST_COMMIT_SHA=$(gh pr view $PR_NUMBER --json commits --jq '.commits[0].oid')
-          echo "LATEST_COMMIT_SHA=${LATEST_COMMIT_SHA}" >> $GITHUB_ENV
-          echo "FIRST_COMMIT_SHA=${FIRST_COMMIT_SHA}" >> $GITHUB_ENV
+      - name: Get PR details from event
+        if: ${{ github.event_name == 'pull_request' }}
+        id: get_pr_details_event
+        run: |
+          echo "MILESTONE=${{ github.event.pull_request.milestone.title }}" >> $GITHUB_ENV
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      - name: Check for milestone and if release branch exists
+        continue-on-error: true
+        id: resolve-applicable-versions
+        run: |
+          if [ -z "${{ env.MILESTONE }}" ] || [ "${{ env.MILESTONE }}" = "null" ]; then
+            echo "No milestone set. Exiting."
+            exit 1
+          fi
+
+          filter_versions() {
+              local input_version=$1
+
+              # List remote branches, extract versions, and sort them semantically
+              versions=$(git ls-remote --heads origin 'release-*' | awk -F'release-' '{print $2}' | sort -V)
+
+              # Filter out versions less than the input version
+              filtered_versions=$(echo "$versions" | awk -v input="$input_version" '
+              function compare_versions(v1, v2) {
+                  split(v1, a, /[.v]/);
+                  split(v2, b, /[.v]/);
+                  for (i = 2; i <= 4; i++) {
+                      if (a[i] < b[i]) return -1;
+                      if (a[i] > b[i]) return 1;
+                  }
+                  return 0;
+              }
+              {
+                  if (compare_versions($0, input) >= 0) {
+                      print $0
+                  }
+              }')
+
+              # Keep only the highest patch version for each minor version
+              echo "$filtered_versions" | awk -F. '
+              {
+                  minor = $1 "." $2;
+                  patches[minor] = $0;
+              }
+              END {
+                  for (minor in patches) {
+                      print patches[minor];
+                  }
+              }' | sort -V
+          }
+
+          versions=$(filter_versions "${{ env.MILESTONE }}")
+          json_array=$(echo "$versions" | jq -R -s -c 'split("\n") | map(select(. != ""))')
+          echo "matrix=$json_array" >> $GITHUB_OUTPUT
+
+  create-pr:
+    needs: resolve-releases
+    runs-on: ubuntu-22.04
+    if: ${{ always() && needs.resolve-releases.result == 'success' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{fromJson(needs.resolve-releases.outputs.matrix)}}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get PR details from workflow dispatch
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        id: get_pr_details_dispatch
+        run: |
+          PR_NUMBER=${{ github.event.inputs.pr_number }}
+          PR_DATA=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/opencrvs/opencrvs-countryconfig/pulls/$PR_NUMBER)
+          # printf escapes the newlines in the JSON, so we can use jq to parse output such as:
+          # "body": "![image](https://github.com/user-attachments/assets/8eee5bcf-7692-490f-a19f-576623e09961)\r\n",
+          echo "PR_ID=$(printf '%s' $PR_DATA | jq -r '.number')" >> $GITHUB_ENV
+          echo "PR_AUTHOR=$(printf '%s' $PR_DATA | jq -r '.user.login')" >> $GITHUB_ENV
+          echo "PR_MERGER=$(printf '%s' $PR_DATA | jq -r '.merged_by.login')" >> $GITHUB_ENV
+          echo "BASE_BRANCH=$(printf '%s' $PR_DATA | jq -r '.base.ref')" >> $GITHUB_ENV
+          echo "HEAD_BRANCH=$(printf '%s' $PR_DATA | jq -r '.head.ref')" >> $GITHUB_ENV
+          echo "PR_TITLE=$(printf '%s' $PR_DATA | jq -r '.title')" >> $GITHUB_ENV
+          echo "BASE_SHA=$(printf '%s' $PR_DATA | jq -r '.base.sha')" >> $GITHUB_ENV
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
       - name: Get PR details from event
         if: ${{ github.event_name == 'pull_request' }}
@@ -56,7 +142,6 @@ jobs:
           PR_NUMBER=${{ github.event.pull_request.number }}
           echo "PR_ID=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
           echo "PR_AUTHOR=${{ github.event.pull_request.user.login }}" >> $GITHUB_ENV
-          echo "MILESTONE=${{ github.event.pull_request.milestone.title }}" >> $GITHUB_ENV
           echo "BASE_BRANCH=${{ github.event.pull_request.base.ref }}" >> $GITHUB_ENV
           echo "HEAD_BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
           echo "PR_TITLE=${{ github.event.pull_request.title }}" >> $GITHUB_ENV
@@ -70,52 +155,36 @@ jobs:
           MERGED_BY_LOGIN=$(echo "$PR_DETAILS" | jq -r '.mergedBy.login')
           echo "PR_MERGER=$MERGED_BY_LOGIN" >> $GITHUB_ENV
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Check for milestone and if release branch exists
-        continue-on-error: true
-        id: check_release_branch
-        run: |
-          if [ -z "${{ env.MILESTONE }}" ]; then
-            echo "No milestone set. Exiting."
-            exit 1
-          fi
-
-          RELEASE_BRANCH="release-${{ env.MILESTONE }}"
-
-          # Check if the release branch exists
-          if git ls-remote --heads origin $RELEASE_BRANCH | grep -q "refs/heads/$RELEASE_BRANCH"; then
-            echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV
-          else
-            echo "Release branch $RELEASE_BRANCH does not exist. Exiting."
-            exit 1
-          fi
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
       - name: Create and push the new branch for the PR
-        if: ${{ steps.check_release_branch.outcome == 'success' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
         run: |
           SEMANTIC_PR_TITLE="${{ env.PR_TITLE }}"
+          RELEASE_BRANCH="release-${{ matrix.version }}"
+          MILESTONE="${{ matrix.version }}"
 
           # Check for semantic prefix
           if [[ $SEMANTIC_PR_TITLE =~ ^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert|wip|merge)\: ]]; then
             SEMANTIC_PR_TITLE="${BASH_REMATCH[1]}(${MILESTONE}): ${SEMANTIC_PR_TITLE#*: }"
           else
-            SEMANTIC_PR_TITLE="🍒 Merge changes from PR #${{ env.PR_ID }} to ${{ env.RELEASE_BRANCH }}"
+            SEMANTIC_PR_TITLE="🍒 Merge changes from PR #${{ env.PR_ID }} to $RELEASE_BRANCH"
           fi
 
-          PR_BODY="Automated PR to merge changes from develop to ${{ env.RELEASE_BRANCH }}"
+          PR_BODY="Automated PR to merge changes from develop to $RELEASE_BRANCH"
 
           # Configure git
           git config user.name "github-actions"
           git config user.email "[email protected]"
           git config advice.mergeConflict false
 
           # Fetch and checkout the release branch
-          git fetch --all
-          git checkout ${{ env.RELEASE_BRANCH }}
+          git fetch --all --unshallow
+          git checkout $RELEASE_BRANCH
 
           # Create a new branch for the PR
-          NEW_BRANCH="auto-pr-${{ env.RELEASE_BRANCH }}-${{ env.PR_ID }}-$RANDOM"
+          NEW_BRANCH="auto-pr-$RELEASE_BRANCH-${{ env.PR_ID }}-$RANDOM"
           git checkout -b $NEW_BRANCH
 
           echo "First commit: ${{ env.FIRST_COMMIT_SHA }}"
@@ -167,6 +236,15 @@ jobs:
             "
           }
 
+          if [ "${{ github.event.inputs.dry_run }}" == "true" ]; then
+            echo "This is a dry run."
+            echo "Would have pushed the new branch $NEW_BRANCH"
+            echo "PR title: $SEMANTIC_PR_TITLE"
+            echo "PR body:"
+            echo "$PR_BODY"
+            exit 0
+          fi
+
           # Push the new branch
           git push origin $NEW_BRANCH
 
@@ -180,6 +258,4 @@ jobs:
               AUTHOR=${{ env.PR_MERGER }}
             fi
           fi
-          gh pr create --title "$SEMANTIC_PR_TITLE" --body "$PR_BODY" --head "$NEW_BRANCH" --base "${{ env.RELEASE_BRANCH }}" --assignee "$AUTHOR" --reviewer "$AUTHOR"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          gh pr create --title "$SEMANTIC_PR_TITLE" --body "$PR_BODY" --head "$NEW_BRANCH" --base "$RELEASE_BRANCH" --assignee "$AUTHOR" --reviewer "$AUTHOR"

.github/workflows/deploy-prod.yml

@@ -14,7 +14,7 @@ on:
       core-image-tag:
         description: Core DockerHub image tag
         required: true
-        default: 'v1.6.0'
+        default: 'v1.6.3'
       countryconfig-image-tag:
         description: Your Country Config DockerHub image tag
         required: true

.github/workflows/deploy.yml

@@ -15,7 +15,7 @@ on:
       core-image-tag:
         description: Core DockerHub image tag
         required: true
-        default: 'v1.6.0'
+        default: 'v1.6.3'
       countryconfig-image-tag:
         description: Your Country Config DockerHub image tag
         required: true

CHANGELOG.md

@@ -1,6 +1,22 @@
 # Changelog
 
-## 1.6.0 (TBD)
+## 1.6.3
+
+### Breaking changes
+
+- Add constant.humanName to allow countries to customise the format of the full name in the sytem for `sytem users` and `citizens` e.g `{LastName} {MiddleName} {Firstname}`, in any case where one of the name is not provided e.g no `MiddleName`, we'll simply render e.g `{LastName} {FirstName}` without any extra spaces if that's the order set in `country-config`. [#6830](https://github.com/opencrvs/opencrvs-core/issues/6830)
+
+## 1.6.2
+
+## 1.6.1
+
+### Bug fixes
+
+- We make sure that the automatic cleanup job only runs before deployment (instead of cron schedule cleanup).
+- Previously it was possible MongoDB replica set and users were left randomly uninitialised after a deployment. MongoDB initialisation container now retries on failure.
+- On some machines 'file' utility was not preinstalled causing provision to fail. We now install the utility if it doesn't exist.
+
+## 1.6.0
 
 ### Breaking changes
 
@@ -40,6 +56,7 @@
 - You can now configure the home screen application’s name and icons in your country configuration package as manifest.json and app icon files are moved from core to country config (check `src/client-static` folder)
 - Updated `allowedFileFormats` in signature fields to use MIME types (`image/png`, `image/jpg`, `image/jpeg`, `image/svg`) instead of simple file extensions. If you are already using the `allowedFileFormats` field in your implementation, please ensure to update the format accordingly.
 - Remove unnecessary UI dividers that add in various sections of the declaration forms(e.g the Death, Birth and Marriage forms) [#244](https://github.com/opencrvs/opencrvs-countryconfig/pull/244)
+- The details exists conditionals for the various sections i.e. father, mother, spouse has to use the `values.detailsExist` property instead of accessing it from `draftData.[sectionName].detailsExists`. This is due to the fact that the draftData is not populated until any changes have been made to any of the fields in the current section.
 
 ### Bug fixes
 
@@ -77,6 +94,14 @@ INSERT CSV ROWS IN ENGLISH ONLY
 
 - Github pipeline dedicated for reading secrets and variables from other environments now checks if GH_TOKEN is still valid before attempting other operations
 - Remove unnecessary UI dividers that add in various sections of the declaration forms(e.g the Death, Birth and Marriage forms) [#244](https://github.com/opencrvs/opencrvs-countryconfig/pull/244)
+- Update template transformer for fields `informantType` and `otherInformantType` that fixes the bug of unavailability of these template fields [#5952](https://github.com/opencrvs/opencrvs-countryconfig/pull/5952)
+- Fixed missing InitialValue property to set initial values based on an expression
+
+## 1.5.2 (https://github.com/opencrvs/opencrvs-countryconfig/compare/v1.5.1...v1.5.2)
+
+## Bug fixes
+
+- Broken email alerts from low disk space are now fixed [293](https://github.com/opencrvs/opencrvs-countryconfig/pull/293)
 
 ## 1.5.0 (https://github.com/opencrvs/opencrvs-countryconfig/compare/v1.4.1...v1.5.0)
 

infrastructure/deployment/deploy.sh

@@ -268,6 +268,11 @@ split_and_join() {
    SPLIT=$(echo $text | sed -e "s/$separator_for_splitting/$separator_for_joining/g")
    echo $SPLIT
 }
+cleanup_docker_images()
+{
+   echo "Cleaning up the docker images"
+   configured_ssh "/usr/bin/docker system prune -af | sudo tee -a /var/log/docker-prune.log > /dev/null"
+}
 
 docker_stack_deploy() {
   echo "Deploying this environment: $ENVIRONMENT_COMPOSE"
@@ -407,6 +412,8 @@ configured_ssh "/opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST"
 
 rotate_secrets
 
+cleanup_docker_images
+
 docker_stack_deploy
 
 echo

infrastructure/docker-compose.deploy.yml

@@ -226,8 +226,11 @@ services:
       labels:
         - traefik.enable=false
       replicas: 1
+      # If this doesn't get run successfully, both the replica set in mongo and
+      # the user schema remains uninitialized.
       restart_policy:
-        condition: none
+        condition: on-failure
+        delay: 10s
     environment:
       - REPLICAS=1
       - MONGODB_ADMIN_USER=${MONGODB_ADMIN_USER}

infrastructure/monitoring/elastalert/rules/alert.yaml

@@ -26,13 +26,13 @@ filter:
       should:
         - term:
             rule.name.keyword:
-              value: 'Available disk space in data partition'
+              value: 'Available disk space in root file system'
         - term:
             rule.name.keyword:
               value: 'CPU under heavy load'
         - term:
             rule.name.keyword:
-              value: 'Low on available disk space'
+              value: 'Low on available disk space in data partition'
       minimum_should_match: 1
 
 alert: post2

infrastructure/server-setup/tasks/docker.yml

@@ -45,9 +45,8 @@
   cron:
     user: '{{ crontab_user }}'
     name: 'cleanup docker images'
-    minute: '0'
-    hour: '0'
     job: '/usr/bin/docker system prune -af >> /var/log/docker-prune.log'
+    state: absent
 
 - name: Check if Docker group exists
   command: getent group docker

infrastructure/server-setup/tasks/swap.yml

@@ -14,6 +14,13 @@
   tags:
     - swap.file.permissions
 
+- name: Ensure 'file' utility is installed
+  ansible.builtin.package:
+    name: file
+    state: present
+  tags:
+    - swap.file.mkswap
+
 - name: 'Check swap file type'
   command: file {{ swap_file_path }}
   register: swapfile