Feature branch sync - staging to pub/ATSSA

.config/requirements.yml

@@ -24,6 +24,3 @@ collections:
     version: 1.16.2
   - name: community.postgresql
     version: 3.10.2
-  - name: https://github.com/kubernetes-sigs/kubespray
-    type: git
-    version: v2.25.0

build_image_aarch64/roles/image_creation/tasks/build_base_image.yml

@@ -58,7 +58,8 @@
   rescue:
     - name: Fail the build if the base image build fails
       ansible.builtin.fail:
-        msg: "{{ base_image_failure_msg }}"
+        msg: |
+          {{ base_image_failure_msg }}
 
   always:
     - name: Remove generated base image vars file

build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml

@@ -49,8 +49,6 @@
     ansible-playbook {{ openchami_clone_path }}/dell/podman-quadlets/image.yaml \
     -i {{ aarch64_inventory_file }} -v \
       --extra-vars '@{{ openchami_dir }}/{{ item.key }}_compute_images.yaml' \
-      -e "minio_s3_username={{ minio_s3_username }}" \
-      -e "minio_s3_password={{ minio_s3_password }}" \
       --tags compute_image -v | \
       /usr/bin/tee '{{ openchami_log_dir }}/{{ item.key }}_compute_image.log'
   async: 3600   # Set async timeout (e.g., 1 hour)
@@ -63,21 +61,65 @@
 
 - name: Wait for all OpenCHAMI jobs to finish and remove generated compute images templates
   block:
+    - name: Display image build jobs status
+      ansible.builtin.debug:
+        msg: "Waiting for image build: {{ item.item.key }} (Job ID: {{ item.ansible_job_id }})"
+      loop: "{{ compute_image_build_job.results }}"
+      loop_control:
+        label: "{{ item.item.key }}"
+
     - name: Wait for all OpenCHAMI jobs to finish
       ansible.builtin.async_status:
         jid: "{{ item.ansible_job_id }}"
       register: job_result
       until: job_result.finished
+      no_log: true
       retries: "{{ job_retry }}"
       delay: "{{ job_delay }}"
       loop: "{{ compute_image_build_job.results }}"
       loop_control:
-        label: "{{ item.item.key }}"
+        label: "Building: {{ item.item.key }}"
 
   rescue:
-    - name: Fail explicitly if job failed
+    - name: Identify failed image builds
+      ansible.builtin.set_fact:
+        failed_images: >
+          {{ job_result.results
+          | selectattr('failed', 'defined')
+          | selectattr('failed', 'equalto', true)
+          | map(attribute='item.item.key')
+          | list }}
+      when: job_result.results is defined
+
+    - name: Build failure message list
+      ansible.builtin.set_fact:
+        failure_msg_list:
+          - "aarch64 compute image build job did not complete successfully."
+          - "Check logs at {{ openchami_log_dir }} for respective functional group for more details."
+          - ""
+          - "Failed images:"
+
+    - name: Add failed image names to message
+      ansible.builtin.set_fact:
+        failure_msg_list: "{{ failure_msg_list + ['  - ' + item] }}"
+      loop: "{{ failed_images | default(['Unknown - check all logs']) }}"
+
+    - name: Add log paths section to message
+      ansible.builtin.set_fact:
+        failure_msg_list: "{{ failure_msg_list + ['', 'Check logs at ' + openchami_log_dir + ' for details:'] }}"
+
+    - name: Add log file paths to message
+      ansible.builtin.set_fact:
+        failure_msg_list: "{{ failure_msg_list + ['  - ' + openchami_log_dir + '/' + item + '_compute_image.log'] }}"
+      loop: "{{ failed_images | default([]) }}"
+
+    - name: Display aarch64 compute image build failure details
+      ansible.builtin.debug:
+        msg: "{{ failure_msg_list }}"
+
+    - name: Failed to build the aarch64 compute image
       ansible.builtin.fail:
-        msg: "{{ compute_image_failure_msg }}"
+        msg: "aarch64 compute image build failed. See details above."
 
   always:
     - name: Remove generated compute images templates

build_image_aarch64/roles/image_creation/templates/compute_images_templates.j2

@@ -9,6 +9,8 @@ group_name: "{{ item.key }}"
 rhel_base_compute_mounts: {{ ochami_compute_mounts | join(' ') }}
 image_build_name: {{ ochami_aarch64_image  | join (' ') }}
 rhel_base_compute_command_options: {{ ochami_base_command | join (' ') }}
+minio_s3_username: "{{ minio_s3_username }}"
+minio_s3_password: "{{ minio_s3_password }}"
 
 rhel_repos:
 {% set rhel_repo = rhel_aarch64_repos %}

build_image_aarch64/roles/image_creation/vars/main.yml

@@ -19,7 +19,7 @@ dir_permissions_644: "0644"
 dir_permissions_755: "0755"
 openchami_dir: "/opt/omnia/openchami"
 openchami_clone_path: /opt/omnia/openchami/deployment-recipes
-job_retry: "60"
+job_retry: "120"
 job_delay: "30"
 openchami_work_dir: "{{ oim_shared_path }}/omnia/openchami/workdir"
 ochami_mounts:
@@ -43,8 +43,12 @@ openchami_aarch64_base_image_log_path: "{{ openchami_log_dir }}/aarch64_base_ima
 openchami_base_image_vars_template: "{{ role_path }}/templates/base_image_template.j2"
 openchami_aarch64_base_image_vars_path: "/opt/omnia/openchami/aarch64_base_image_template.yaml"
 aarch64_inventory_file: "/tmp/temp_ochami_inventory.ini"
-base_image_failure_msg: "Base aarch64 image build job failed or timed out. Check logs at path {{ openchami_aarch64_base_image_log_path }} for details."
-compute_image_failure_msg: "Compute aarch64 image build job did not completed successfully."
+base_image_failure_msg: |
+  Base aarch64 image build job failed or timed out.
+  Check logs at path {{ openchami_aarch64_base_image_log_path }} for details.
+compute_image_failure_msg: |
+  aarch64 compute image build job did not complete successfully.
+  Check logs at {{ openchami_log_dir }} for respective functional group for more details.
 
 # Usage: build_compute_image.yml
 openchami_compute_image_vars_template: "{{ role_path }}/templates/compute_images_templates.j2"

build_image_x86_64/roles/image_creation/tasks/build_base_image.yml

@@ -50,7 +50,8 @@
   rescue:
     - name: Fail the build if the base image build fails
       ansible.builtin.fail:
-        msg: "{{ base_image_failure_msg }}"
+        msg: |
+          {{ base_image_failure_msg }}
 
   always:
     - name: Remove generated base image vars file

build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml

@@ -41,8 +41,6 @@
     ansible-playbook {{ openchami_clone_path }}/dell/podman-quadlets/image.yaml \
       -i {{ openchami_clone_path }}/dell/podman-quadlets/inventory -v \
       --extra-vars '@{{ openchami_dir }}/{{ item.key }}_compute_images.yaml' \
-      -e "minio_s3_username={{ minio_s3_username }}" \
-      -e "minio_s3_password={{ minio_s3_password }}" \
       --tags compute_image -v | \
       /usr/bin/tee '{{ openchami_log_dir }}/{{ item.key }}_compute_image.log'
   async: 3600   # Set async timeout (e.g., 1 hour)
@@ -55,21 +53,65 @@
 
 - name: Wait for all OpenCHAMI jobs to finish and remove generated compute images templates
   block:
+    - name: Display image build jobs status
+      ansible.builtin.debug:
+        msg: "Waiting for image build: {{ item.item.key }} (Job ID: {{ item.ansible_job_id }})"
+      loop: "{{ compute_image_build_job.results }}"
+      loop_control:
+        label: "{{ item.item.key }}"
+
     - name: Wait for all OpenCHAMI jobs to finish
       ansible.builtin.async_status:
         jid: "{{ item.ansible_job_id }}"
       register: job_result
       until: job_result.finished
+      no_log: true
       retries: "{{ job_retry }}"
       delay: "{{ job_delay }}"
       loop: "{{ compute_image_build_job.results }}"
       loop_control:
-        label: "{{ item.item.key }}"
+        label: "Building: {{ item.item.key }}"
 
   rescue:
-    - name: Fail explicitly if job failed
+    - name: Identify failed image builds
+      ansible.builtin.set_fact:
+        failed_images: >
+          {{ job_result.results
+          | selectattr('failed', 'defined')
+          | selectattr('failed', 'equalto', true)
+          | map(attribute='item.item.key')
+          | list }}
+      when: job_result.results is defined
+
+    - name: Build failure message list
+      ansible.builtin.set_fact:
+        failure_msg_list:
+          - "x86_64 compute image build job did not complete successfully."
+          - "Check logs at {{ openchami_log_dir }} for respective functional group for more details."
+          - ""
+          - "Failed images:"
+
+    - name: Add failed image names to message
+      ansible.builtin.set_fact:
+        failure_msg_list: "{{ failure_msg_list + ['  - ' + item] }}"
+      loop: "{{ failed_images | default(['Unknown - check all logs']) }}"
+
+    - name: Add log paths section to message
+      ansible.builtin.set_fact:
+        failure_msg_list: "{{ failure_msg_list + ['', 'Check logs at ' + openchami_log_dir + ' for details:'] }}"
+
+    - name: Add log file paths to message
+      ansible.builtin.set_fact:
+        failure_msg_list: "{{ failure_msg_list + ['  - ' + openchami_log_dir + '/' + item + '_compute_image.log'] }}"
+      loop: "{{ failed_images | default([]) }}"
+
+    - name: Display x86_64 compute image build failure details
+      ansible.builtin.debug:
+        msg: "{{ failure_msg_list }}"
+
+    - name: Failed to build the x86_64 compute image
       ansible.builtin.fail:
-        msg: "{{ compute_image_failure_msg }}"
+        msg: "x86_64 compute image build failed. See details above."
 
   always:
     - name: Remove generated compute images templates

build_image_x86_64/roles/image_creation/templates/compute_images_templates.j2

@@ -8,6 +8,8 @@ group_name: "{{ item.key }}"
 rhel_base_compute_mounts: {{ ochami_compute_mounts | join(' ') }}
 image_build_name: {{ ochami_x86_64_image | join (' ') }}
 rhel_base_compute_command_options: {{ ochami_base_command | join (' ') }}
+minio_s3_username: "{{ minio_s3_username }}"
+minio_s3_password: "{{ minio_s3_password }}"
 
 rhel_repos:
 {% set rhel_repo = rhel_x86_64_repos %}

build_image_x86_64/roles/image_creation/vars/main.yml

@@ -19,7 +19,7 @@ dir_permissions_644: "0644"
 dir_permissions_755: "0755"
 openchami_dir: "/opt/omnia/openchami"
 openchami_clone_path: /opt/omnia/openchami/deployment-recipes
-job_retry: "60"
+job_retry: "120"
 job_delay: "30"
 openchami_work_dir: "{{ oim_shared_path }}/omnia/openchami/workdir"
 ochami_mounts:
@@ -42,8 +42,12 @@ openchami_log_dir: /opt/omnia/log/openchami
 openchami_x86_64_base_image_log_path: "{{ openchami_log_dir }}/x86_64_base_image.log"
 openchami_base_image_vars_template: "{{ role_path }}/templates/base_image_template.j2"
 openchami_x86_64_base_image_vars_path: "/opt/omnia/openchami/x86_64_base_image_template.yaml"
-base_image_failure_msg: "Base x86_64 image build job failed or timed out. Check logs at path {{ openchami_x86_64_base_image_log_path }} for details."
-compute_image_failure_msg: "Compute x86_64 image build job did not completed successfully."
+base_image_failure_msg: |
+  Base x86_64 image build job failed or timed out.
+  Check logs at path {{ openchami_x86_64_base_image_log_path }} for details.
+compute_image_failure_msg: |
+  x86_64 compute image build job did not complete successfully.
+  Check logs at {{ openchami_log_dir }} for respective functional group for more details.
 
 # build_compute_image.yml
 openchami_compute_image_vars_template: "{{ role_path }}/templates/compute_images_templates.j2"

common/library/module_utils/input_validation/common_utils/config.py

@@ -39,24 +39,23 @@
     "software_config": "software_config.json",
     "storage_config": "storage_config.yml",
     "telemetry_config": "telemetry_config.yml",
-    #"functional_groups_config": "functional_groups_config.yml",
     "high_availability_config": "high_availability_config.yml"
     # "additional_software": "additional_software.json"
 }
 
 # Tags and the files that will be run based off of it
 input_file_inventory = {
+    "build_image": [files["provision_config"]],
     "scheduler": [
         files["software_config"],
-        #files['functional_groups_config'],
+
         files["omnia_config"]
         # files["high_availability_config"]
     ],
     "provision": [
         files["provision_config"],
         files["network_spec"],
         files["software_config"],
-       # files["functional_groups_config"]
         # files["high_availability_config"]
     ],
     "security": [
@@ -73,12 +72,10 @@
         files["omnia_config"],
         files["storage_config"],
         files["high_availability_config"],
-        #files["functional_groups_config"]
     ],
     "storage": [files["storage_config"]],
     "prepare_oim": [
         files["network_spec"],
-        #files["functional_groups_config"]
     ],
     # "high_availability": [files["high_availability_config"]],
     # "additional_software": [files["additional_software"]],
@@ -92,7 +89,6 @@
         files["software_config"],
         files["storage_config"],
         files["high_availability_config"],
-        #files["functional_groups_config"]
     ],
 }
 
@@ -102,10 +98,11 @@
     "ofed": "24.10-1.1.4.0",
     "beegfs": "7.4.5",
     "intel_benchmarks": "2024.1.0",
-    "ucx": "1.15.0",
-    "openmpi": "4.1.6",
-    "csi_driver_powerscale": "v2.11.0",
-    "rocm": "6.3.1"
+    "ucx": "1.19.0",
+    "openmpi": "5.0.8",
+    "csi_driver_powerscale": "v2.15.0",
+    "rocm": "6.3.1",
+    "service_k8s": "1.34.1"
 }
 
 # All of the passwords fields
@@ -156,6 +153,19 @@
 
 supported_telemetry_collection_type = ["victoria","kafka"]
 
+FUNCTIONAL_GROUP_LAYER_MAP = {
+    "service_kube_control_plane_first_x86_64": "management",
+    "service_kube_control_plane_x86_64": "management",
+    "service_kube_node_x86_64": "management",
+    "login_node_x86_64": "management",
+    "login_node_aarch64": "management",
+    "login_compiler_node_x86_64": "management",
+    "login_compiler_node_aarch64": "management",
+    "slurm_control_node_x86_64": "management",
+    "slurm_node_x86_64": "compute",
+    "slurm_node_aarch64": "compute"
+}
+
 # used for security_config.yml validation
 supported_ldap_connection_type = ["TLS","SLS"]
 EMAIL_MAX_LENGTH = 320