Merge pull request #7 from merizrizal/ceph-integration

Ceph integration - part 2

Author: merizrizal

README.md

@@ -112,6 +112,18 @@ Run `make -C base_image/ rebuild-base-image-ubuntu`. This will create a new Vagr
 3. Run `make -C controller/ start-provider-network` to create a new interface which will be used by Controller, Compute and Storage node.
 4. Run `make -C controller/ start-vm` to spin up 4 VMs which are 1 Controller, 2 Computes and 1 Storage node.
 
+**Provision Ceph:**
+1. Run `source envrc` then navigate to `./ansible` directory.
+2. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_pre_setup.yml` to install and configure the pre-requisite packages.
+3. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_setup_adm.yml` to install and configure Ceph ADM service.
+4. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_setup_common.yml` to install and configure Ceph common packages to the other hosts.
+5. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_apply_osd.yml` to add the other hosts and apply the OSD.
+6. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_openstack_init.yml` to configure OpenStack Cinder and Nova to be able to integrate with Ceph
+
+or
+
+Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_deploy.yml` to deploy all at once.
+
 **Provision OpenStack:**
 1. Run `source envrc` then navigate to `./ansible` directory.
 2. Run `ansible-playbook -i deploy_openstack/inventories/local/local.yml deploy_openstack/playbook_pre_setup.yml` to install and configure the pre-requisite packages.
@@ -141,19 +153,6 @@ Replace `<host-external-interface>` with the actual NIC connected to the interne
 You can replace `<host-external-interface>` with `$(ip route get 1.1.1.1 | awk '{print $5}')` to dynamically detect your default NIC.
 <br>
 
-
-**Provision Ceph:**
-1. Run `source envrc` then navigate to `./ansible` directory.
-2. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_pre_setup.yml` to install and configure the pre-requisite packages.
-3. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_setup_adm.yml` to install and configure Ceph ADM service.
-4. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_setup_common.yml` to install and configure Ceph common packages to the other hosts.
-5. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_apply_osd.yml` to add the other hosts and apply the OSD.
-6. Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_openstack_init.yml` to configure OpenStack Cinder and Nova to be able to integrate with Ceph
-
-or
-
-Run `ansible-playbook -i deploy_ceph/inventories/local/local.yml deploy_ceph/playbook_deploy.yml` to deploy all at once.
-
 ---
 
 **CI/CD Lab Deployment**

ansible/cicd_in_openstack/inventories/local/group_vars/all/common_gitlab.yml

@@ -7,7 +7,7 @@ gitlab_url: http://{{ hostvars[cicd_list.gitlab01.name].ansible_host }}
 gitlab_api_url: "{{ gitlab_url }}/api/v4"
 
 gitlab_runner_binary: >
-  https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/latest/binaries/gitlab-runner-linux-{{ arch_mapping[ansible_architecture] }}
+  https://s3.dualstack.us-east-1.amazonaws.com/gitlab-runner-downloads/latest/binaries/gitlab-runner-linux-{{ arch_mapping[ansible_facts['architecture']] }}
 
 gitlab_runner_installed: /usr/local/bin/gitlab-runner
 gitlab_runner_token_var: gitlab_runner_token

ansible/deploy_ceph/inventories/local/group_vars/all/common.yml

@@ -7,5 +7,8 @@ ntp_server: time.aws.com
 ceph_pub_key_fetch_path: /tmp/fetch-ceph.pub
 device_name: vdb
 
-mon_count: 3
-mgr_count: 2
+ceph_common_nodes: "{{ groups.ceph_common | default([]) }}"
+is_single_node: "{{ ceph_common_nodes | length == 0 }}"
+
+mon_count: "{{ is_single_node | ternary(1, 3) }}"
+mgr_count: "{{ is_single_node | ternary(1, 2) }}"

ansible/deploy_ceph/inventories/local/local.yml

@@ -8,9 +8,9 @@ lab:
       hosts:
         ceph01:
           ansible_host: "{{ ceph01.mgmtnet_ip_address }}"
-    ceph_common:
-      hosts:
-        ceph02:
-          ansible_host: "{{ ceph02.mgmtnet_ip_address }}"
-        ceph03:
-          ansible_host: "{{ ceph03.mgmtnet_ip_address }}"
+    # ceph_common:
+    #   hosts:
+    #     ceph02:
+    #       ansible_host: "{{ ceph02.mgmtnet_ip_address }}"
+    #     ceph03:
+    #       ansible_host: "{{ ceph03.mgmtnet_ip_address }}"

ansible/deploy_ceph/playbook_deploy.yml

@@ -7,6 +7,7 @@
 
 - name: Setup Ceph common
   ansible.builtin.import_playbook: playbook_setup_common.yml
+  when: not is_single_node
 
 - name: Apply Ceph OSDs
   ansible.builtin.import_playbook: playbook_apply_osd.yml

ansible/deploy_ceph/roles/ceph_adm/tasks/add_hosts.yml

@@ -4,7 +4,7 @@
     cmd: ceph orch host add {{ item }} {{ node_vars.mgmtnet_ip_address }}
   vars:
     node_vars: "{{ lookup('ansible.builtin.vars', item) }}"
-  loop: "{{ groups['ceph_common'] }}"
+  loop: "{{ ceph_common_nodes }}"
   changed_when: false
 
 - name: Wait hosts and its devices to be ready
@@ -15,7 +15,7 @@
         sleep 1
       done
     executable: /bin/bash
-  loop: "{{ groups['ceph_common'] }}"
+  loop: "{{ ceph_common_nodes }}"
   changed_when: false
 
 - name: Place MON

ansible/deploy_ceph/roles/ceph_adm/tasks/add_osds.yml

@@ -3,15 +3,25 @@
   ansible.builtin.command:
     cmd: ceph orch daemon add osd {{ item }}:/dev/{{ device_name }}
   register: apply_osd
-  loop: "{{ groups['ceph_adm'] + groups['ceph_common'] }}"
+  loop: "{{ groups.ceph_adm + ceph_common_nodes }}"
   changed_when: apply_osd.stdout is defined and 'Created osd' in apply_osd.stdout
 
+- name: Set pool size limit to 1 on single_node mode
+  ansible.builtin.shell:
+    cmd: |
+      ceph config set global osd_pool_default_size 1
+      ceph config set global osd_pool_default_min_size 1
+    executable: /bin/bash
+  when: is_single_node
+  changed_when: false
+
 - name: Create OSD pool
   ansible.builtin.shell:
     cmd: |
-      ceph osd pool create {{ item }} 16
+      ceph osd pool create {{ item }} {{ is_single_node | ternary(8, 16) }}
       ceph osd pool application enable {{ item }} rbd
       rbd pool init {{ item }}
+    executable: /bin/bash
   loop:
     - images
     - volumes

ansible/deploy_ceph/roles/openstack/tasks/main.yml

@@ -20,8 +20,8 @@
     cmd: >
       ceph auth get-or-create client.cinder
       mon 'profile rbd'
-      osd 'profile rbd pool=volumes, profile rbd pool=images'
-      mgr 'profile rbd pool=volumes, profile rbd pool=images'
+      osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images'
+      mgr 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images'
       -o /etc/ceph/ceph.client.cinder.keyring
   changed_when: false
 
@@ -31,22 +31,6 @@
     dest: "{{ cinder_keyring_path }}"
     flat: true
 
-- name: Create auth for Nova
-  ansible.builtin.command:
-    cmd: >
-      ceph auth get-or-create client.nova
-      mon 'profile rbd'
-      osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images'
-      mgr 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd pool=images'
-      -o /etc/ceph/ceph.client.nova.keyring
-  changed_when: false
-
-- name: Fetch Nova keyring
-  ansible.builtin.fetch:
-    src: /etc/ceph/ceph.client.nova.keyring
-    dest: "{{ nova_keyring_path }}"
-    flat: true
-
 - name: Fetch Ceph config
   ansible.builtin.fetch:
     src: /etc/ceph/ceph.conf

ansible/deploy_openstack/inventories/local/group_vars/all/common.yml

@@ -9,4 +9,4 @@ controller01_host_addr: "{{ use_hostname | ternary(controller01.hostname, contro
 compute01_host_addr: "{{ use_hostname | ternary(compute01.hostname, compute01.mgmtnet_ip_address) }}"
 storage01_host_addr: "{{ use_hostname | ternary(storage01.hostname, storage01.mgmtnet_ip_address) }}"
 
-ceph_enabled: false
+ceph_enabled: true

ansible/deploy_openstack/roles/ceph/tasks/cinder.yml

@@ -3,7 +3,7 @@
   ansible.builtin.set_fact:
     mgmtnet_ip_address: "{{ node.mgmtnet_ip_address }}"
   vars:
-    node: "{{ lookup('ansible.builtin.vars', ansible_hostname) }}"
+    node: "{{ lookup('ansible.builtin.vars', ansible_facts['hostname']) }}"
 
 - name: Reconfigure cinder.conf - set default - apply ceph
   ansible.builtin.replace: