Feat/ansible perf (#1917)

This PR addresses issue #1831.

Improvements:
- Removed unnecessary fact gathering.
- Replaced loops with blockinfile where applicable.
- Introduced `async` with `poll: 0` to run non-blocking tasks, combined
with later state checks using `async_status`.
- Reorganized tasks and play structure for more efficient execution.

overall speedup of execution is ~20%

The WireGuard playbook remains the slowest part of the workflow. It
could be parallelized further, but doing so would require synchronizing
host facts, particularly the public key values used when rendering the
WireGuard template in this task:

```
- name: Template wireguard config file to a node
  ansible.builtin.template:
    src: "{{ 'wg-static.conf.j2' if 'static' in group_names else 'wg-dynamic.conf.j2' }}"
    dest: "{{ wg_conf_dir.path }}/wg0.conf"
```

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Performance Improvements**
* More background package operations and reduced fact gathering to speed
playbook runs.

* **Stability & Reliability**
* Consolidated kernel, sysctl and systemd limits with a single reload
for consistent node limits.
* Centralized proxy environment management via block updates and
standardized drop-in dirs.
* Safer WireGuard key handling and idempotent interface/IP management;
unattended-upgrades now masked.

* **Refactor**
* Merged and simplified related tasks for cleaner, more consistent
playbooks.

* **Chores**
  * Updated ansibler image tag.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Matus Brandys <matus.brandys@berops.com>
Co-authored-by: CI/CD pipeline <CI/CD-pipeline@users.noreply.github.com>

Author: 3 people

manifests/claudie/kustomization.yaml

@@ -56,7 +56,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: ghcr.io/berops/claudie/ansibler
-  newTag: 7430672-3800
+  newTag: 24bd046-3775
 - name: ghcr.io/berops/claudie/autoscaler-adapter
   newTag: 193717f-3802
 - name: ghcr.io/berops/claudie/builder

services/ansibler/server/ansible-playbooks/longhorn-req.yml

@@ -1,6 +1,6 @@
 ---
 - hosts: all
-  gather_facts: no
+  gather_facts: false
   become: yes
   vars:
     # for node-limits tasks
@@ -12,33 +12,18 @@
     max_queued_events: 65535
   # Install longhorn  dependencies
   tasks:
-    - name: install open-iscsi
+    - name: install open-iscsi and nfs-common
       ansible.builtin.apt:
-        name: open-iscsi
+        name: 
+          - open-iscsi
+          - nfs-common
         state: present
         update_cache: true
       retries: 2
       delay: 10
-      register: res
-      until: res is not failed
-      # open-iscsi packages are few hundred Kibs in size
-      # having a timeout of 8 mins with 2 retries is generous.
+      register: async_job
       async: 480
-      poll: 5
-
-    - name: install nfs-common
-      ansible.builtin.apt:
-        name: nfs-common
-        state: present
-        update_cache: true
-      retries: 2
-      delay: 10
-      register: res
-      until: res is not failed
-      # nfs-common packages are few hundred Kibs in size
-      # having a timeout of 8 mins with 2 retries generous.
-      async: 480
-      poll: 5
+      poll: 0
 
     - name: Update /etc/multipath.conf with blacklist configuration
       blockinfile:
@@ -61,3 +46,11 @@
 
     - name: update node limits
       include_tasks: ./node-limits.yml
+
+    - name: Check async task result
+      async_status:
+        jid: "{{ async_job.ansible_job_id }}"
+      register: result
+      until: result.finished
+      retries: 60
+      delay: 5

services/ansibler/server/ansible-playbooks/node-limits.yml

@@ -10,76 +10,44 @@
       root hard nofile {{ nofile_hard }}
     state: present
 
-- name: Ensure PAM limits are enabled
-  lineinfile:
-    path: /etc/pam.d/common-session
-    line: "session required pam_limits.so"
-    state: present
-
-- name: Ensure PAM limits are enabled for non-interactive sessions
-  lineinfile:
-    path: /etc/pam.d/common-session-noninteractive
+- name: Enable PAM limits
+  ansible.builtin.lineinfile:
+    path: "{{ item }}"
     line: "session required pam_limits.so"
     state: present
+  loop:
+    - /etc/pam.d/common-session
+    - /etc/pam.d/common-session-noninteractive
 
 - name: Configure systemd system limits
   ansible.builtin.lineinfile:
-    path: /etc/systemd/system.conf
-    regexp: "^#?DefaultLimitNOFILE="
-    line: "DefaultLimitNOFILE={{ nofile_hard }}"
-
-- name: Configure systemd user limits
-  ansible.builtin.lineinfile:
-    path: /etc/systemd/user.conf
+    path: "{{ item }}"
     regexp: "^#?DefaultLimitNOFILE="
     line: "DefaultLimitNOFILE={{ nofile_hard }}"
+  loop:
+    - /etc/systemd/system.conf
+    - /etc/systemd/user.conf
 
 - name: Reload systemd configuration
   ansible.builtin.systemd:
     daemon_reload: yes
 
-- name: Set kernel file handle maximum
-  ansible.builtin.sysctl:
-    name: fs.file-max
-    value: "{{ file_max }}"
-    state: present
-
-- name: Set kernel user queued events maximum
-  ansible.builtin.sysctl:
-    name: fs.inotify.max_queued_events
-    value: "{{ max_queued_events }}"
-    state: present
-
-- name: Set kernel user watches maximum
-  ansible.builtin.sysctl:
-    name: fs.inotify.max_user_watches
-    value: "{{ max_user_watches }}"
-    state: present
-
-- name: Set kernel user instances maximum
-  ansible.builtin.sysctl:
-    name: fs.inotify.max_user_instances
-    value: "{{ max_user_instances }}"
-    state: present
-    reload: yes
-
-- name: Set Ephemeral port range
-  ansible.builtin.sysctl:
-    name: net.ipv4.ip_local_port_range
-    value: "2048 65535"
-    state: present
-    reload: yes
-
-- name: Set somaxconn to 65535
-  ansible.builtin.sysctl:
-    name: net.core.somaxconn
-    value: "65535"
-    state: present
-    reload: yes
-
-- name: Set TCP fin timeout to 30s
-  ansible.builtin.sysctl:
-    name: net.ipv4.tcp_fin_timeout
-    value: "30"
-    state: present
-    reload: yes
+- name: Configure kernel parameters
+  ansible.builtin.blockinfile:
+    path: /etc/sysctl.d/99-claudie-custom.conf
+    create: yes
+    owner: root
+    group: root
+    mode: '0644'
+    block: |
+      fs.file-max = {{ file_max }}
+      fs.inotify.max_queued_events = {{ max_queued_events }}
+      fs.inotify.max_user_watches = {{ max_user_watches }}
+      fs.inotify.max_user_instances = {{ max_user_instances }}
+      net.ipv4.ip_local_port_range = 2048 65535
+      net.core.somaxconn = 65535
+      net.ipv4.tcp_fin_timeout = 30
+
+  
+- name: Reload sysctl at once with new settings
+  ansible.builtin.command: sysctl --system

services/ansibler/server/ansible-playbooks/proxy/commit-proxy-envs-changes.yml

@@ -44,7 +44,7 @@
         crictl pods | grep kube-scheduler | awk '{print $1}' | xargs -I {} sh -c 'crictl stopp {} && crictl rmp {}'
 
 - hosts: all
-  gather_facts: true
+  gather_facts: false
   become: true
   tasks:
     # This will reload the systemd unit files present under /etc/systemd/system/.
@@ -69,7 +69,7 @@
       loop: "{{ service_status.results }}"
 
 - hosts: control
-  gather_facts: true
+  gather_facts: false
   become: true
   tasks:
     - name: Try 10 times to check kube-apiserver health

services/ansibler/server/ansible-playbooks/proxy/populate-proxy-envs.yml

@@ -1,25 +1,25 @@
 - hosts: all
+  strategy: free
   gather_facts: false
   become: true
   tasks:
-    - name: Verifying if /etc/apt/apt.conf.d/ exists
+    - name: Ensure apt and systemd drop-in directories exist
       ansible.builtin.file:
-        path: /etc/apt/apt.conf.d/
-        state: directory
-    - name: Verifying if directories in /etc/systemd/system for containerd,kubelet,docker exists
-      ansible.builtin.file:
-        path: "/etc/systemd/system/{{ item }}.service.d/"
+        path: "{{ item }}"
         state: directory
+        mode: "0755"
       loop:
-        - docker
-        - containerd
-        - kubelet
+        - /etc/systemd/system/docker.service.d
+        - /etc/systemd/system/containerd.service.d
+        - /etc/systemd/system/kubelet.service.d
+
     - name: Setting HTTP,HTTPS proxy settings in /etc/apt/apt.conf.d/proxy.conf
       ansible.builtin.copy:
         content: |
           Acquire::https::Proxy "{{ http_proxy_url }}";
           Acquire::http::Proxy "{{ http_proxy_url }}";
         dest: "/etc/apt/apt.conf.d/proxy.conf"
+
     - name: Setting HTTP,HTTPS proxy settings in /etc/systemd/system/
       ansible.builtin.copy:
         content: |
@@ -31,28 +31,15 @@
         - containerd
         - docker
 
-    - name: Delete HTTP,HTTPS,NO_PROXY from /etc/environmennt
-      ansible.builtin.lineinfile:
+    - name: Ensure proxy variables in /etc/environment
+      ansible.builtin.blockinfile:
         path: /etc/environment
-        regexp: "^{{ item.key }}=.*$"
-        state: absent
-      loop:
-        - { key: "HTTP_PROXY" }
-        - { key: "http_proxy" }
-        - { key: "HTTPS_PROXY" }
-        - { key: "https_proxy" }
-        - { key: "NO_PROXY" }
-        - { key: "no_proxy" }
-
-    - name: Add HTTP,HTTPS,NO_PROXY into /etc/environmennt
-      ansible.builtin.lineinfile:
-        path: /etc/environment
-        regexp: "^{{ item.key }}=.*$"
-        line: "{{ item.key }}={{ item.value }}"
-      loop:
-        - { key: "HTTP_PROXY", value: "{{ http_proxy_url }}" }
-        - { key: "http_proxy", value: "{{ http_proxy_url }}" }
-        - { key: "HTTPS_PROXY", value: "{{ http_proxy_url }}" }
-        - { key: "https_proxy", value: "{{ http_proxy_url }}" }
-        - { key: "NO_PROXY", value: "{{ no_proxy_list }}" }
-        - { key: "no_proxy", value: "{{ no_proxy_list }}" }
+        create: yes
+        marker: "# {mark} ANSIBLE MANAGED PROXY SETTINGS"
+        block: |
+          HTTP_PROXY={{ http_proxy_url }}
+          http_proxy={{ http_proxy_url }}
+          HTTPS_PROXY={{ http_proxy_url }}
+          https_proxy={{ http_proxy_url }}
+          NO_PROXY={{ no_proxy_list }}
+          no_proxy={{ no_proxy_list }}

services/ansibler/server/ansible-playbooks/proxy/remove-proxy-envs.yml

@@ -14,15 +14,9 @@
         - docker
         - containerd
         - kubelet
-    - name: Delete HTTP,HTTPS,NO_PROXY from /etc/environmennt
-      ansible.builtin.lineinfile:
+
+    - name: Remove proxy variables block from /etc/environment
+      ansible.builtin.blockinfile:
         path: /etc/environment
-        regexp: "^{{ item.key }}=.*$"
+        marker: "# {mark} ANSIBLE MANAGED PROXY SETTINGS"
         state: absent
-      loop:
-        - { key: "HTTP_PROXY" }
-        - { key: "http_proxy" }
-        - { key: "HTTPS_PROXY" }
-        - { key: "https_proxy" }
-        - { key: "NO_PROXY" }
-        - { key: "no_proxy" }

services/ansibler/server/ansible-playbooks/wireguard.yml

@@ -8,10 +8,6 @@
       wait_for_connection:
         timeout: 100
 
-    # Gather facts manually after we made sure, VMs are accessible
-    - name: Gather facts from nodes
-      ansible.builtin.setup:
-
   # abort playbook on any fatal error, the golang code will trigger a retry
   any_errors_fatal: true
   roles:

services/ansibler/server/ansible-playbooks/wireguard/tasks/configure.yml

@@ -5,32 +5,42 @@
     state: directory
   register: wg_conf_dir
 
-- name: Generate wireguard keypair
-  ansible.builtin.shell: umask 077 && wg genkey | tee {{ wg_conf_dir.path }}/privatekey | wg pubkey > {{ wg_conf_dir.path }}/publickey
+- name: Generate WireGuard keypair if not existing
+  ansible.builtin.shell: |
+    umask 077 && wg genkey | tee {{ wg_conf_dir.path }}/privatekey | wg pubkey > {{ wg_conf_dir.path }}/publickey
+  args:
+    creates: "{{ wg_conf_dir.path }}/privatekey"
 
-- name: Save privatekey
-  ansible.builtin.shell: cat {{ wg_conf_dir.path }}/privatekey
+- name: Read private key
+  ansible.builtin.slurp:
+    src: "{{ wg_conf_dir.path }}/privatekey"
   register: privatekey
 
-- name: Save publickey
-  ansible.builtin.shell: cat {{ wg_conf_dir.path }}/publickey
+- name: Read public key
+  ansible.builtin.slurp:
+    src: "{{ wg_conf_dir.path }}/publickey"
   register: publickey
 
 - name: Check if wireguard interface exists
-  ansible.builtin.shell: ip a
-  register: ipa_contents
+  ansible.builtin.stat:
+    path: /sys/class/net/wg0
+  register: wg0_stat
 
 - name: Create wireguard interface
-  ansible.builtin.shell: ip link add dev wg0 type wireguard
-  when: ipa_contents.stdout.find('wg0') == -1
+  ansible.builtin.command:
+    cmd: ip link add dev wg0 type wireguard
+  when: not wg0_stat.stat.exists
 
-- name: Check if wireguard interface has IP address
-  ansible.builtin.shell: ip a
-  register: ipa_contents
+- name: Check if interface already has the IP
+  ansible.builtin.command:
+    cmd: ip -o addr show dev wg0
+  register: wg0_ip_info
+  changed_when: false 
 
-- name: Configure an IP address for an interface
-  ansible.builtin.shell: "ip address add dev wg0 {{ private_ip }}/{{ netmask }}"
-  when: "ipa_contents.stdout.find( private_ip ) == -1"
+- name: Assign IP address
+  ansible.builtin.command:
+    cmd: ip address add dev wg0 {{ private_ip }}/{{ netmask }}
+  when: private_ip not in wg0_ip_info.stdout
 
 - name: Template wireguard config file to a node
   ansible.builtin.template:
@@ -46,18 +56,13 @@
   ansible.builtin.shell: ip link set up dev wg0
 
 - name: Set MTU size for wg0
-  ansible.builtin.shell: ifconfig {{ wg_interface_name }} mtu {{ mtu_size }}
-
-- name: Ensure file /etc/dhcp/dhclient.conf exists
-  ansible.builtin.copy:
-    content: ""
-    dest: /etc/dhcp/dhclient.conf
-    force: false
+  ansible.builtin.shell: ip link set mtu {{ mtu_size }} dev {{ wg_interface_name }}
 
-- name: Make MTU change persistent across reboots
+- name: Ensure file /etc/dhcp/dhclient.conf exists and make MTU change persistent across reboots
   ansible.builtin.blockinfile:
     path: /etc/dhcp/dhclient.conf
-    insertbefore: "request"
+    create: yes
+    insertbefore: "^request"
     block: |
       interface "{{ wg_interface_name }}" {
          default interface-mtu {{ mtu_size }};