feat: Verify passphrase before provision new one

Author: adskyiproger

.github/workflows/k8s-reset-data.yml

@@ -75,7 +75,10 @@ jobs:
     needs: prepare
     env:
       namespace: opencrvs-${{ inputs.environment }}
-    runs-on: [self-hosted, k8s, e2e]
+    runs-on:
+      - self-hosted
+      - k8s
+      - ${{ inputs.environment }}
     strategy:
       max-parallel: 1  # Ensure jobs run one by one
       fail-fast: true  # Stop on first failure

.github/workflows/provision.yml

@@ -23,6 +23,7 @@ on:
           - application
           - tools
           - fail2ban
+          - data-partition
           - decrypt-on-boot
           - checks
           - containerd-setup

infrastructure/server-setup/group_vars/all.yml

@@ -7,14 +7,21 @@
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
 ansible_python_interpreter: /usr/bin/python3
-encrypt_data: False
 
-backup_server_user: 'backup'
-backup_server_user_home: '/home/backup'
 crontab_user: root
 provisioning_user: provision
 kubernetes_version: "v1.33"
 pod_network_cidr: "192.168.0.0/16"
 calico_version: "v3.26.1"
+
+# Kubernetes secret in opencrvs-deps-<env name> namespace
 backup_server_ssh_credentials_secret: "backup-server-ssh-credentials"
-backup_ssh_key_path: /home/backup/.ssh/id_ed25519
+# Path on backup server where to private key is stored
+backup_ssh_key_path: /home/backup/.ssh/id_ed25519
+# User for ssh connection to backup server
+backup_server_user: 'backup'
+# Home directory for user on backup server
+backup_server_user_home: '/home/backup'
+
+# Disk Encryption key location as an example (in production use a hardware security module)
+disk_encryption_key_path: /root/disk-encryption-key.txt

infrastructure/server-setup/k8s.yml

@@ -20,14 +20,23 @@
             - data-partition
       tags:
         - data-partition
-
+    - include_tasks:
+        file: tasks/k8s/validate-data-partition.yml
+        apply:
+          tags:
+            - data-partition
+            - decrypt-on-boot
+      tags:
+        - data-partition
+        - decrypt-on-boot
     - include_tasks:
         file: tasks/k8s/decrypt-on-boot.yml
         apply:
           tags:
             - decrypt-on-boot
       tags:
         - decrypt-on-boot
+
     - name: Include system preparation tasks
       include_tasks: tasks/k8s/system-preparation.yml
       tags:

infrastructure/server-setup/tasks/backups/crontab.yml

@@ -173,20 +173,6 @@
     mode: ugo+rwx
   when: not minio_data_backup.stat.exists
 
-- name: Check wireguard data backup directory
-  stat:
-    path: /data/wireguard
-  register: wireguard_data
-
-- name: 'Create wireguard backup directory'
-  file:
-    path: /data/wireguard
-    state: directory
-    group: 1000
-    owner: 1000
-    mode: ugo+rwx
-  when: not wireguard_data.stat.exists
-
 - name: Check vsexport data directory
   stat:
     path: /data/vsexport
@@ -215,14 +201,6 @@
     mode: ugo+rwx
   when: not vsexport_data_backup.stat.exists
 
-- name: 'Create secrets directory'
-  file:
-    path: /data/secrets
-    state: directory
-    group: 1000
-    owner: 1000
-    mode: g+rwx
-
 - name: 'Create acme file for traefik'
   file:
     path: /data/traefik/acme.json

infrastructure/server-setup/tasks/k8s/data-partition.yml

@@ -4,10 +4,38 @@
     state: present
   when: disk_encryption_key is defined
 
-- name: Save disk encryption key into a file as an example (in production use a hardware security module)
+- name: 'Register encrypted file system'
+  stat:
+    path: /cryptfs_file_sparse.img
+    get_checksum: False
+  register: encryptedFileSystemPostCheck
+  when: disk_encryption_key is defined
+
+- name: Check if Disk encryption key file exists
+  ansible.builtin.stat:
+    path: "{{ disk_encryption_key_path }}"
+  register: encryption_keyfile_stat
+  when: disk_encryption_key is defined
+
+- name: Read existing Disk encryption key
+  ansible.builtin.slurp:
+    src: "{{ disk_encryption_key_path }}"
+  register: existing_disk_encryption_key
+  when:
+    - disk_encryption_key is defined
+    - encryption_keyfile_stat.stat.exists
+
+- name: Fail if key exists and differs
+  ansible.builtin.fail:
+    msg: "Disk encryption key on disk does not match ansible variable! GitHub secret ENCRYPTION_KEY was updated!"
+  when:
+    - encryption_keyfile_stat.stat.exists
+    - existing_disk_encryption_key['content'] | b64decode != ("DISK_ENCRYPTION_KEY=" ~ disk_encryption_key ~ "\n")
+
+- name: Save disk encryption key into a file
   when: disk_encryption_key is defined
   ansible.builtin.copy:
-    dest: /root/disk-encryption-key.txt
+    dest: "{{ disk_encryption_key_path }}"
     group: 1000
     owner: 1000
     mode: g+rwx
@@ -42,7 +70,7 @@
       Description=Mount encrypted dir
 
       [Service]
-      ExecStart=bash /opt/opencrvs/scripts/cryptfs/decrypt.sh -key /root/disk-encryption-key.txt >> /var/log/cryptfs-reboot.log 2>&1
+      ExecStart=bash /opt/opencrvs/scripts/cryptfs/decrypt.sh -key {{ disk_encryption_key_path }} >> /var/log/cryptfs-reboot.log 2>&1
 
       [Install]
       WantedBy=multi-user.target
@@ -51,6 +79,7 @@
    - encryptedFileSystemPostCheck.stat.exists
 
 - name: 'Setup systemd to mount encrypted folder'
+  when: disk_encryption_key is defined
   shell: systemctl daemon-reload && systemctl enable reboot.service
   when:
     - disk_encryption_key is defined

infrastructure/server-setup/tasks/k8s/decrypt-on-boot.yml

@@ -0,0 +1,26 @@
+- name: Verify LUKS disk encryption key
+  when: disk_encryption_key is defined
+  block:
+    - name: Backup LUKS header
+      shell: |
+        loop_device=$(losetup -j /cryptfs_file_sparse.img | cut -d: -f1)
+        rm -vf /tmp/luks-header.img
+        cryptsetup luksHeaderBackup $loop_device --header-backup-file /tmp/luks-header.img
+
+    - name: Test disk encryption key for LUKS header
+      when: disk_encryption_key is defined
+      shell: 'echo "{{ disk_encryption_key }}" | cryptsetup -d - luksOpen /tmp/luks-header.img test --test-passphrase'
+      register: luks_test
+
+    - name: Display success message
+      debug:
+        msg: "✓ LUKS passphrase test successful - the disk encryption key is valid: {{ luks_test.stdout }}"
+  rescue:
+    - name: DO NOT REBOOT the SERVER
+      fail:
+        msg: |
+          {% if luks_test.rc == 2 and 'No key available with this passphrase' in luks_test.stderr -%}
+          LUKS passphrase test failed: Please make sure your key in GitHub secret is correct. Otherwise system will not be able to decrypt /data. MAKE SURE DATA BACKUP IS RESTORABLE BEFORE SERVER REBOOT.
+          {%- else -%}
+          cryptsetup error (RC: {{ luks_test.rc }}) - {{ luks_test.stderr | default('Unknown error') }}
+          {%- endif %}