ansible-scylla-node: increase API port wait timeout to reduce CI flakiness

ansible-scylla-manager/tasks/add_cluster.yml

@@ -8,6 +8,16 @@
     echo $(sctool status|grep -q "Cluster: {{ item.cluster_name }} " && echo "present")
   register: cluster_already_added
 
+- name: Fetch alternator salted hash for "{{ item.username }}"
+  shell: |
+    cqlsh {{ item.host }} -u '{{ item.username }}' -p '{{ item.password }}' \
+      -e "SELECT salted_hash FROM system.roles WHERE role = '{{ item.username }}';" \
+      | grep -E '^\s+\$' | tr -d ' '
+  register: alternator_salted_hash
+  when:
+    - item.username is defined
+    - item.password is defined
+
 - name: run sctool cluster add for "{{ item.cluster_name }}"
   shell: |
     sctool cluster add \
@@ -20,6 +30,12 @@
       --username {{ item.username }} \
       --password {{ item.password }} \
       {% endif %}
+      {% if alternator_salted_hash is defined
+         and alternator_salted_hash.stdout is defined
+         and alternator_salted_hash.stdout != '' %}
+      --alternator-access-key-id {{ item.username }} \
+      --alternator-secret-access-key '{{ alternator_salted_hash.stdout }}' \
+      {% endif %}
   when: cluster_already_added is not defined or cluster_already_added.stdout != "present"
 
 - name: Enforce configuration for existing cluster "{{ item.cluster_name }}"
@@ -33,5 +49,11 @@
       --username {{ item.username }} \
       --password {{ item.password }} \
       {% endif %}
+      {% if alternator_salted_hash is defined
+         and alternator_salted_hash.stdout is defined
+         and alternator_salted_hash.stdout != '' %}
+      --alternator-access-key-id {{ item.username }} \
+      --alternator-secret-access-key '{{ alternator_salted_hash.stdout }}' \
+      {% endif %}
   when: cluster_already_added is defined and cluster_already_added.stdout == "present"
 

ansible-scylla-monitoring/defaults/main.yml

@@ -24,6 +24,9 @@ scylla_monitoring_archive_url: 'https://github.com/scylladb/scylla-monitoring/ar
 scylla_monitoring_role_config_path: "{{ inventory_dir }}"
 
 scylla_monitoring_deploy_path: /opt/scylla-monitoring
+scylla_monitoring_start_script_path: "{{ scylla_monitoring_deploy_path }}/start-monitoring.sh"
+scylla_monitoring_stop_script_path: "{{ scylla_monitoring_deploy_path }}/stop-monitoring.sh"
+scylla_monitoring_start_env_path: "{{ scylla_monitoring_deploy_path }}/start-monitoring.env"
 scylla_monitoring_data_path: "{{ scylla_monitoring_deploy_path }}/data"
 scylla_monitoring_config_path:  "{{ scylla_monitoring_deploy_path }}/config"
 scylla_monitoring_prom_rules_path:  "{{ scylla_monitoring_config_path }}/prom_rules"

ansible-scylla-monitoring/tasks/docker.yml

@@ -31,6 +31,28 @@
   when: "'cql_credentials' in groups and item.split('=')[0] == scylla_monitoring_cql_default_user | default('scylla_cql_monitor')"
   loop: "{{ groups['cql_credentials'] }}"
 
+- name: "docker.yml: Write stop monitoring wrapper script"
+  template:
+    src: stop-monitoring.sh.j2
+    dest: "{{ scylla_monitoring_stop_script_path }}"
+    mode: '0750'
+    owner: "{{ ansible_user_id }}"
+
+- name: "docker.yml: Write start monitoring env file"
+  template:
+    src: start-monitoring.env.j2
+    dest: "{{ scylla_monitoring_start_env_path }}"
+    mode: '0600'
+    owner: "{{ ansible_user_id }}"
+  when: monitoring_cql_username is defined and monitoring_cql_password is defined
+
+- name: "docker.yml: Write start monitoring wrapper script"
+  template:
+    src: start-monitoring.sh.j2
+    dest: "{{ scylla_monitoring_start_script_path }}"
+    mode: '0750'
+    owner: "{{ ansible_user_id }}"
+
 - name: "docker.yml: restart the docker daemon again"
   service:
     name: docker
@@ -49,15 +71,10 @@
   when: ansible_facts.services["docker.service"] is defined
 
 - name: "docker.yml: Stop monitoring"
-  shell: |
-    cd {{ base_dir }}
-    ./kill-all.sh
-    sudo ./kill-all.sh
+  command: "{{ scylla_monitoring_stop_script_path }}"
 
 - name: "docker.yml: start scylla-monitoring"
-  shell: "{{ start_command }}"
-  args:
-    chdir: "{{ base_dir }}"
+  command: "{{ scylla_monitoring_start_script_path }}"
   become: "{{ 'true' if run_docker_with_sudo is defined and run_docker_with_sudo|bool == True else 'false' }}"
   environment:
     SCYLLA_USER: "{% if monitoring_cql_username is defined and monitoring_cql_password is defined %}{{ monitoring_cql_username }}{% endif %}"

ansible-scylla-monitoring/templates/start-monitoring.env.j2

@@ -0,0 +1,4 @@
+{% if monitoring_cql_username is defined and monitoring_cql_password is defined %}
+export SCYLLA_USER={{ monitoring_cql_username | quote }}
+export SCYLLA_PSSWD={{ monitoring_cql_password | quote }}
+{% endif %}

ansible-scylla-monitoring/templates/start-monitoring.sh.j2

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd {{ base_dir | quote }}
+ENV_FILE={{ scylla_monitoring_start_env_path | quote }}
+if [ -f "$ENV_FILE" ]; then
+  # shellcheck disable=SC1090
+  . "$ENV_FILE"
+fi
+{{ start_command }}

ansible-scylla-monitoring/templates/stop-monitoring.sh.j2

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd {{ base_dir | quote }}
+./kill-all.sh || sudo ./kill-all.sh || true

ansible-scylla-node/defaults/main.yml

@@ -295,9 +295,14 @@ io_conf: 'SEASTAR_IO="--io-properties-file /etc/scylla.d/io_properties.yaml"'
 #
 # scylla_args_value: "--log-to-syslog 1 --log-to-stdout 0 --default-log-level info --network-stack posix"
 
-# Seeds node list
-scylla_seeds:
-  - "{{ groups['scylla'][0] }}"
+# Seeds node list (mandatory parameter)
+# A seed should be either a DNS resolvable address (e.g. hostname) that is resolvable by all nodes in the cluster,
+# or an IP address that is accessible by all nodes in the cluster.
+# If the hostname is not resolvable by all nodes, then such an IP address can be set, for example, explicitly or via ansible_host.
+# Do not use the below default example if ansible_host is not specified and the hostname is not resolvable by all nodes.
+#
+#scylla_seeds:
+#  - "{{ hostvars[groups['scylla'][0]].ansible_host if 'ansible_host' in hostvars[groups['scylla'][0]] else groups['scylla'][0] }}"
 
 scylla_listen_address: "{{ vars['ansible_'~scylla_nic].ipv4.address }}"
 scylla_rpc_address: "{{ vars['ansible_'~scylla_nic].ipv4.address }}"

ansible-scylla-node/tasks/common.yml

@@ -1,4 +1,15 @@
 ---
+
+# Sanity check: scylla_seeds has no default and is mandatory since there is no
+# hostname/IP that can be safely assumed reachable by all nodes in the cluster.
+- name: "Sanity check: scylla_seeds must be explicitly defined"
+  fail:
+    msg: >-
+      scylla_seeds is not defined or empty. Set it explicitly, e.g. to a DNS resolvable
+      hostname reachable by all nodes, or an IP address (possibly via ansible_host).
+      See defaults/main.yml for an example.
+  when: scylla_seeds is not defined or scylla_seeds | length == 0
+
 - name: General, NIC and CPU settings related tweaking
   block:
     - name: configure Scylla
@@ -341,16 +352,31 @@
       wait_for:
         port: "{{ scylla_api_port }}"
         host: "{{ scylla_api_address }}"
-        timeout: 300
+        timeout: 600
 
     - name: wait for the cluster to become healthy
-      shell: |
-        nodetool status|grep -E '^UN|^UJ|^DN'|wc -l
+      shell:
+        executable: /bin/bash
+        cmd: |
+          state=$(systemctl is-active scylla-server)
+
+          if [[ "$state" != "active" && "$state" != "activating" ]]; then
+            echo "scylla-server is not running (state: $state)"
+            exit 1
+          fi
+
+          count=$(nodetool status | grep -E '^UN|^UJ|^DN' | wc -l)
+          if [[ "$count" == "{{ ansible_play_batch|length }}" ]]; then
+            exit 0
+          fi
+
+          exit 2
       register: node_count
-      until: node_count.stdout|int == ansible_play_batch|length
+      until: node_count.rc != 2
       retries: "{{ scylla_bootstrap_wait_time_sec|int }}"
       delay: 1
       when: full_inventory|bool
+      become: true
 
     - name: check API access
       uri:

ansible-scylla-node/tasks/start_one_node.yml

@@ -13,8 +13,8 @@
     cmd: |
       state=$(systemctl is-active scylla-server)
 
-      if [[ "$state" == "failed" || "$state" == "inactive" ]]; then
-        echo "scylla-server has unexpectedly stopped"
+      if [[ "$state" != "active" && "$state" != "activating" ]]; then
+        echo "scylla-server is not running (state: $state)"
         exit 1
       fi
 

example-playbooks/replace_node/replace_node.yml

@@ -350,12 +350,29 @@
     - vars/main.yml
   tasks:
     - name: Wait for the added node to become healthy
-      shell: |
-        nodetool status|grep -E '^UN'|grep -w "{{ hostvars[new_node]['broadcast_address'] }}"| wc -l
+      shell:
+        executable: /bin/bash
+        cmd: |
+          state=$(systemctl is-active scylla-server)
+
+          if [[ "$state" != "active" && "$state" != "activating" ]]; then
+            echo "scylla-server is not running (state: $state)"
+            exit 1
+          fi
+
+          count=$(nodetool status | grep -E '^UN' | grep -w "{{ hostvars[new_node]['broadcast_address'] }}" | wc -l)
+          if [[ "$count" == "1" ]]; then
+            exit 0
+          fi
+
+          exit 2
       register: node_count
-      until: node_count.stdout|int == 1
+      delegate_to: "{{ new_node }}"
+      run_once: true
+      until: node_count.rc != 2
       retries: "{{ scylla_bootstrap_wait_time_sec|int }}"
       delay: 1
+      become: true
 
     - name: start and enable the Manager agent service
       service: