ci_dcn_site: Add removing and adding a node to a DCN site

Author: krcmarik

playbooks/dcn.yml

@@ -61,7 +61,6 @@
       when:
         - _subnet_network_range != ''
         - _ceph_bootstrap_node != ''
-        - cifmw_ci_dcn_site_scaledown_az is not defined or cifmw_ci_dcn_site_scaledown_az == ""
       ansible.builtin.include_role:
         name: ci_dcn_site
 
@@ -83,6 +82,49 @@
         name: ci_dcn_site
       when: cifmw_ci_dcn_site_scaledown_az is defined and cifmw_ci_dcn_site_scaledown_az != ""
 
+    - name: Remove a compute node from the deployment
+      vars:
+        _node_to_remove: "{{ cifmw_ci_dcn_site_scaledown_node }}"
+        _az: "{{ cifmw_ci_dcn_site_scaledown_node_az | default('az1') }}"
+        _group_name: "{{ az_to_group_map[_az] }}"
+        _group_hosts: "{{ groups[_group_name] }}"
+        _edpm_hosts: "{{ cifmw_baremetal_hosts | dict2items | selectattr('key', 'in', groups[_group_name]) | items2dict }}"
+        _ceph_bootstrap_node: "{{ (_edpm_hosts | dict2items | first).key if _edpm_hosts | length > 0 else '' }}"
+      when:
+        - cifmw_ci_dcn_site_scaledown_node is defined and cifmw_ci_dcn_site_scaledown_node != ""
+      ansible.builtin.include_role:
+        name: ci_dcn_site
+        tasks_from: remove_node.yml
+
+    - name: Add a compute node to a site
+      when:
+        - cifmw_ci_dcn_site_scaleout_node is defined and cifmw_ci_dcn_site_scaleout_node != ""
+      block:
+        # We need to update ci-framework related variable and inventory files to include the new host
+        - name: Update ci-framework variables and inventory files
+          vars:
+            _node_to_add: "{{ cifmw_ci_dcn_site_scaleout_node }}"
+            _az: "{{ cifmw_ci_dcn_site_scaleout_node_az | default('az1') }}"
+            _subnet: "{{ cifmw_ci_dcn_site_scaleout_node_subnet | default('subnet2') }}"
+            _group_name: "{{ az_to_group_map[_az] }}"
+          ansible.builtin.include_role:
+            name: ci_dcn_site
+            tasks_from: update_conf_new_node.yml
+
+        - name: Add a compute node to dataplane
+          vars:
+            _node_to_add: "{{ cifmw_ci_dcn_site_scaleout_node }}"
+            _az: "{{ cifmw_ci_dcn_site_scaleout_node_az | default('az1') }}"
+            _subnet: "{{ cifmw_ci_dcn_site_scaleout_node_subnet | default('subnet2') }}"
+            _subnet_network_range: "{{ _network_ranges[_az[-1] | int] }}"
+            _group_name: "{{ az_to_group_map[_az] }}"
+            _group_hosts: "{{ groups[_group_name] }}"
+            _edpm_hosts: "{{ updated_cifmw_baremetal_hosts | dict2items | selectattr('key', 'in', groups[_group_name]) | items2dict }}"
+            _ceph_bootstrap_node: "{{ (_edpm_hosts | dict2items | first).key if _edpm_hosts | length > 0 else '' }}"
+          ansible.builtin.include_role:
+            name: ci_dcn_site
+            tasks_from: add_node.yml
+
     - name: Find all created CRs
       ansible.builtin.find:
         paths: >-

roles/ci_dcn_site/README.md

@@ -16,6 +16,8 @@ with a collocated Ceph cluster.
 * `_group_name`: The name of the group of nodes to be deployed, e.g. `dcn1-computes`
 * `_subnet`: The name of the subnet the DCN site will use, e.g. `subnet2`
 * `_subnet_network_range`: The range of the subnet the DCN site will use, e.g. `192.168.133.0/24`
+* `_node_to_remove`: The hostname of the node to be removed from the DCN deployment.
+* `_node_to_add`: The hostname of the node to be added to the specified AZ.
 
 ## Examples
 

roles/ci_dcn_site/tasks/add_node.yml

@@ -0,0 +1,115 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# We are scaling out and the ceph cluster already exists we can directly create the Nodeset utilizing the existing
+# "pre-ceph" values and then update it with "post-ceph" values which include additional ceph configuration and then
+# create and apply Deployment CR with full list of dataplane services.
+- name: Create a values.yaml.j2 file with full list of dataplane services
+  ansible.builtin.copy:
+    dest: "{{ ci_dcn_site_cifmw_repo_path }}/roles/ci_dcn_site/templates/deployment/values.yaml.j2"
+    mode: '0644'
+    content: |
+      ---
+      # source: dcn/deployment/values.yaml.j2
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: edpm-deployment-values-post-ceph
+        annotations:
+          config.kubernetes.io/local-config: "true"
+      data:
+        nodeset_name: "{% raw %}{{ _group_name }}{% endraw %}-edpm"
+        deployment:
+          name: "add-node-{% raw %}{{ _group_name }}{% endraw %}"
+        servicesOverride:
+          - bootstrap
+          - configure-network
+          - validate-network
+          - install-os
+          - ceph-hci-pre
+          - configure-os
+          - ssh-known-hosts
+          - run-os
+          - reboot-os
+          - install-certs
+          - ceph-client
+          - ovn
+          - "neutron-metadata-cell{% raw %}{{ _all_azs.index(_az) +1 }}{% endraw %}"
+          - libvirt
+          - nova-custom-ceph-{% raw %}{{ _az }}{% endraw %}
+
+- name: Initialize vars
+  ansible.builtin.set_fact:
+    _ceph_vars_list: []
+    _all_azs: []
+
+- name: Find all ceph .conf and .keyring files
+  register: _ceph_conf_files
+  ansible.builtin.find:
+    paths: "/tmp"
+    patterns: "ceph*.conf,ceph*.keyring,az*.conf,az*.keyring"
+    recurse: false
+
+- name: Load ceph configuration files
+  ansible.builtin.set_fact:
+    _ceph_files: "{{ _ceph_conf_files.files | map(attribute='path') | list }}"
+
+- name: Render the NodeSet values.yaml
+  vars:
+    _edpm_instance_dict: "{{ cifmw_networking_env_definition.instances }}"
+    _edpm_network_dict: "{{ cifmw_networking_env_definition.networks }}"
+    _ssh_authorizedkeys: "{{ lookup('file', '~/.ssh/id_cifw.pub', rstrip=False) }}"
+    _ssh_private_key: "{{ lookup('file', '~/.ssh/id_cifw', rstrip=False) }}"
+    _ssh_public_key: "{{ lookup('file', '~/.ssh/id_cifw.pub', rstrip=False) }}"
+    _migration_priv_key: "{{ lookup('file', '~/ci-framework-data/artifacts/nova_migration_key', rstrip=False) }}"
+    _migration_pub_key: "{{ lookup('file', '~/ci-framework-data/artifacts/nova_migration_key.pub', rstrip=False) }}"
+  ansible.builtin.template:
+    backup: true
+    src: "templates/edpm-pre-ceph/nodeset/values.yaml.j2"
+    dest: "{{ ci_dcn_site_arch_path }}/edpm-pre-ceph/nodeset/values.yaml"
+    mode: "0644"
+
+- name: Kustomize NodeSet
+  ansible.builtin.set_fact:
+    nodeset_cr: >-
+      {{ lookup('kubernetes.core.kustomize',
+      dir=ci_dcn_site_arch_path + '/edpm-pre-ceph/nodeset') }}
+
+- name: Save the NodeSet CR
+  ansible.builtin.copy:
+    mode: "0644"
+    dest: "{{ ci_dcn_site_arch_path }}/dataplane-nodeset-pre-ceph_{{ _az }}.yaml"
+    content: "{{ nodeset_cr }}"
+    backup: true
+
+- name: Render the values with updated ceph configuration, kustomize and apply CR of NodeSet and DataPlaneDeployment
+  ansible.builtin.import_tasks: post-ceph.yml
+
+- name: Set Network related facts
+  ansible.builtin.include_tasks: set_network_facts.yml
+
+- name: Deploy Ceph in DCN context
+  ansible.builtin.include_tasks: ceph.yml
+
+- name: Run Nova cell discovery for new DCN hosts
+  kubernetes.core.k8s_exec:
+    api_key: "{{ _auth_results.openshift_auth.api_key }}"
+    namespace: openstack
+    pod: nova-cell0-conductor-0
+    command: nova-manage cell_v2 discover_hosts --verbose
+
+- name: Add new hosts to AZ
+  ansible.builtin.include_tasks: az.yml

roles/ci_dcn_site/tasks/ceph.yml

@@ -67,6 +67,25 @@
     dest: "~/ci-framework-data/parameters/ceph-{{ _az }}.yml"
     content: "{{ _content | to_nice_yaml }}"
 
+- name: Check if ceph_client file already exists
+  ansible.builtin.stat:
+    path: "/tmp/ceph_client_{{_az}}.yml"
+  register: ceph_file_stat
+
+- name: Load ceph variables if the file exists
+  ansible.builtin.include_vars:
+    file: "/tmp/ceph_client_{{_az}}.yml"
+  when: ceph_file_stat.stat.exists
+
+- name: If It exists Add cifmw_cephadm_keys to ceph variables file
+  ansible.builtin.blockinfile:
+    path: "~/ci-framework-data/parameters/ceph-{{ _az }}.yml"
+    block: |
+      cifmw_cephadm_keys:
+      {{ keys | default({}) | to_nice_yaml }}
+    insertafter: EOF
+  when: ceph_file_stat.stat.exists
+
 - name: Deploy Ceph
   cifmw.general.ci_script:
     output_dir: "/home/zuul/ci-framework-data/artifacts"

roles/ci_dcn_site/tasks/post-ceph.yml

@@ -47,6 +47,10 @@
   ansible.builtin.set_fact:
     ci_dcn_site_glance_map: "{{ ci_dcn_site_glance_map | combine( { item: ['az0', item ] } ) }}"
 
+- name: Get fsid of ceph cluster for currently deployed AZ
+  ansible.builtin.set_fact:
+    cifmw_ceph_client_fsid: "{{ _ceph_vars_list | selectattr('cifmw_ceph_client_cluster', 'equalto', _az) | map(attribute='cifmw_ceph_client_fsid') | first }}"
+
 - name: Render the post-ceph values.yaml
   ansible.builtin.template:
     mode: "0644"