Skip to content

Enhance timeout settings and validation for HANA cluster operations #232

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
devanshjainms merged 2 commits into from
Apr 10, 2026
Merged

Enhance timeout settings and validation for HANA cluster operations #232

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
706f899
Enhance timeout settings and validation checks for HANA cluster opera…
devanshjainms Apr 8, 2026
fb58919
Update src/roles/ha_db_hana/tasks/secondary-block-network.yml
devanshjainms Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions src/roles/ha_db_hana/tasks/includes/scaleout-worker-operation.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
ignore_errors: "{{ hana_test_action_ignore_errors | default(false) }}"
register: action_result
changed_when: >-
{{ hana_test_action_ignore_errors | default(false)
or action_result.rc == 0 }}
hana_test_action_ignore_errors | default(false)
or action_result.rc == 0
failed_when: >-
{{ not (hana_test_action_ignore_errors | default(false))
and action_result.rc != 0 }}
not (hana_test_action_ignore_errors | default(false))
and action_result.rc != 0

- name: "Test Execution: Validate worker operation cluster status"
block:
Expand Down Expand Up @@ -63,11 +63,13 @@
(
(hana_test_target_site_label == "primary" and
cluster_status_post.primary_node in cluster_status_pre.secondary_site_nodes and
cluster_status_post.secondary_node in cluster_status_pre.primary_site_nodes)
cluster_status_post.secondary_node in cluster_status_pre.primary_site_nodes and
hana_test_target_worker_node in cluster_status_post.secondary_site_nodes)
or
(hana_test_target_site_label == "secondary" and
cluster_status_post.primary_node in cluster_status_pre.primary_site_nodes and
cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes)
cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes and
hana_test_target_worker_node in cluster_status_post.secondary_site_nodes)
)

- name: "Test Execution: Determine test execution status"
Expand Down
1 change: 1 addition & 0 deletions src/roles/ha_db_hana/tasks/primary-node-kill.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
when:
- node_tier == "hana"
- pre_validations_status == "PASSED"
- saphanasr_provider | default('SAPHanaSR') == "SAPHanaSR-angi"
block:
- name: "Test Execution: Kill the primary node."
when: ansible_hostname == cluster_status_pre.primary_node
Expand Down
89 changes: 46 additions & 43 deletions src/roles/ha_db_hana/tasks/secondary-block-network.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,46 +1,53 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

# /*---------------------------------------------------------------------------
# | Block Network Communication From Secondary Site |
# +--------------------------------------------------------------------------*/
- name: "Test Setup Tasks"
ansible.builtin.include_tasks: "roles/misc/tasks/test-case-setup.yml"
run_once: true

# /*---------------------------------------------------------------------------
# | Pre Validations |
# +--------------------------------------------------------------------------*/
- name: "Pre Validations: HANA DB Nodes"
ansible.builtin.include_tasks: "roles/misc/tasks/pre-validations-db.yml"

# /*---------------------------------------------------------------------------
# | Test Execution |
# +--------------------------------------------------------------------------*/
- name: "Test Execution: Block Network Communication From Secondary Master"
become: true
when:
- node_tier == "hana"
- pre_validations_status == "PASSED"
- hana_topology == 'scale_out_hsr'
- cluster_status_pre.stonith_action == "reboot"
- ansible_hostname == primary_master_node
block:
- name: "Test Execution: Start timer on primary master"
when: ansible_hostname == primary_master_node
ansible.builtin.set_fact:
test_execution_start: "{{ now(utc=true, fmt='%Y-%m-%d %H:%M:%S') }}"
test_execution_hostname: "{{ hostvars[secondary_master_node].ansible_hostname }}"

- name: "Test Execution: Prepare network partition facts"
when: ansible_hostname == secondary_master_node
ansible.builtin.set_fact:
secondary_node_ip: "{{ hostvars[secondary_master_node].ansible_host }}"
blocked_ips: "{{ cluster_status_pre.primary_site_nodes | map('extract', hostvars, 'ansible_host') | list }}"

- name: "Test Execution: Create firewall on secondary master"
when: ansible_hostname == secondary_master_node
ansible.builtin.shell: |
{% for ip in blocked_ips %}
iptables -A INPUT -s {{ ip }} -j DROP;
iptables -A OUTPUT -d {{ ip }} -j DROP;
{% endfor %}
- name: "Test Execution: Block all network communication on secondary site"
become: true
ansible.builtin.shell: "sleep 3; iptables -P INPUT DROP; iptables -P OUTPUT DROP"
async: 15
poll: 0
register: firewall_rule_created
changed_when: firewall_rule_created.rc == 0
failed_when: firewall_rule_created.rc != 0
delegate_to: "{{ secondary_node }}"
loop: "{{ cluster_status_pre.secondary_site_nodes }}"
loop_control:
loop_var: secondary_node
ignore_unreachable: true

- name: "Test Execution: Wait for the cluster to be in a stable state"
ansible.builtin.wait_for:
timeout: "{{ default_timeout }}"

- name: "Test Execution: Validate HANA DB cluster status during partition"
when: ansible_hostname == primary_master_node
get_cluster_status_db:
db_instance_number: "{{ db_instance_number }}"
operation_step: "test_execution"
Expand All @@ -55,31 +62,13 @@
until: >
cluster_status_test_execution.primary_node != "" and
cluster_status_test_execution.primary_node in cluster_status_pre.primary_site_nodes and
(
cluster_status_test_execution.secondary_node == "" or
cluster_status_test_execution.secondary_node in cluster_status_pre.secondary_site_nodes
)

- name: "Test Execution: Remove firewall rule on secondary master"
when: ansible_hostname == secondary_master_node
ansible.builtin.shell: |
{% for ip in blocked_ips %}
iptables -D INPUT -s {{ ip }} -j DROP 2>/dev/null;
iptables -D OUTPUT -d {{ ip }} -j DROP 2>/dev/null;
{% endfor %}
true
register: firewall_rule_deleted
changed_when: true
failed_when: false
ignore_unreachable: true
cluster_status_test_execution.secondary_node == ""

- name: "Test Execution: Wait for the cluster to be in a stable state"
when: ansible_hostname == primary_master_node
ansible.builtin.wait_for:
timeout: "{{ default_timeout }}"

- name: "Test Execution: Validate HANA DB cluster status 2"
when: ansible_hostname == primary_master_node
- name: "Test Execution: Validate HANA DB cluster status (post recovery)"
get_cluster_status_db:
db_instance_number: "{{ db_instance_number }}"
operation_step: "post_failover"
Expand All @@ -97,17 +86,18 @@
cluster_status_post.primary_node in cluster_status_pre.primary_site_nodes and
cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes

- name: "Test Execution: Stop timer"
ansible.builtin.set_fact:
test_execution_end: "{{ now(utc=true, fmt='%Y-%m-%d %H:%M:%S') }}"

- name: "Test Execution: Determine test execution status"
when: ansible_hostname == primary_master_node
ansible.builtin.set_fact:
test_execution_status: "{{ 'PASSED' if (
cluster_status_post.primary_node in cluster_status_pre.primary_site_nodes and
cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes
) else 'FAILED' }}"
test_execution_end: "{{ now(utc=true, fmt='%Y-%m-%d %H:%M:%S') }}"

- name: "Set test case message and details"
when: ansible_hostname == primary_master_node
ansible.builtin.set_fact:
test_case_message_from_test_case: |
Source node: {{ secondary_master_node }}
Expand All @@ -117,16 +107,29 @@
"Pre Validations: Remove any location_constraints": "{{ location_constraints_results }}",
"Pre Validations: Validate HANA DB cluster status": "{{ cluster_status_pre }}",
"Pre Validations: CleanUp any failed resource": "{{ cleanup_failed_resource_pre }}",
"Test Execution: Firewall Rule Created": "{{ hostvars[secondary_master_node].firewall_rule_created | default('N/A') }}",
"Test Execution: Network blocked on secondary site": "{{ firewall_rule_created | default('iptables -P DROP applied') }}",
"Test Execution: Cluster Status during partition": "{{ cluster_status_test_execution }}",
"Test Execution: Firewall Rule Removed": "{{ hostvars[secondary_master_node].firewall_rule_deleted | default('Node rebooted') }}",
"Test Execution: Firewall removed by": "Secondary site nodes rebooted by fencing",
"Post Validations: Validate HANA DB cluster status": "{{ cluster_status_post }}",
}
}

# /*---------------------------------------------------------------------------
# | Post Validations |
# +--------------------------------------------------------------------------*/
- name: "Post Validations Tasks"
ansible.builtin.include_tasks: "roles/misc/tasks/post-validations.yml"

rescue:
- name: "Test Execution Failure: Reset iptables policies on secondary site nodes"
become: true
ansible.builtin.shell: "iptables -P INPUT ACCEPT; iptables -P OUTPUT ACCEPT"
register: firewall_rule_deleted
changed_when: firewall_rule_deleted.rc == 0
failed_when: false
ignore_unreachable: true
loop: "{{ cluster_status_pre.secondary_site_nodes }}"
delegate_to: "{{ item }}"

- name: "Rescue operation"
ansible.builtin.include_tasks: "roles/misc/tasks/rescue.yml"

Expand Down
2 changes: 1 addition & 1 deletion src/vars/input-api.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ sap_port_to_ping: "1128"
# Default values for retries, delay, timeout
default_retries: 75
default_delay: 10
default_timeout: 60
default_timeout: 90
ascs_stonith_timeout: 120

# Default values for Azure Backup test cases
Expand Down
Loading