diff --git a/src/roles/ha_db_hana/tasks/includes/scaleout-worker-operation.yml b/src/roles/ha_db_hana/tasks/includes/scaleout-worker-operation.yml index 06acf053..4ebfbcad 100644 --- a/src/roles/ha_db_hana/tasks/includes/scaleout-worker-operation.yml +++ b/src/roles/ha_db_hana/tasks/includes/scaleout-worker-operation.yml @@ -29,11 +29,11 @@ ignore_errors: "{{ hana_test_action_ignore_errors | default(false) }}" register: action_result changed_when: >- - {{ hana_test_action_ignore_errors | default(false) - or action_result.rc == 0 }} + hana_test_action_ignore_errors | default(false) + or action_result.rc == 0 failed_when: >- - {{ not (hana_test_action_ignore_errors | default(false)) - and action_result.rc != 0 }} + not (hana_test_action_ignore_errors | default(false)) + and action_result.rc != 0 - name: "Test Execution: Validate worker operation cluster status" block: @@ -63,11 +63,13 @@ ( (hana_test_target_site_label == "primary" and cluster_status_post.primary_node in cluster_status_pre.secondary_site_nodes and - cluster_status_post.secondary_node in cluster_status_pre.primary_site_nodes) + cluster_status_post.secondary_node in cluster_status_pre.primary_site_nodes and + hana_test_target_worker_node in cluster_status_post.secondary_site_nodes) or (hana_test_target_site_label == "secondary" and cluster_status_post.primary_node in cluster_status_pre.primary_site_nodes and - cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes) + cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes and + hana_test_target_worker_node in cluster_status_post.secondary_site_nodes) ) - name: "Test Execution: Determine test execution status" diff --git a/src/roles/ha_db_hana/tasks/primary-node-kill.yml b/src/roles/ha_db_hana/tasks/primary-node-kill.yml index 2183513f..4fc2cc5b 100644 --- a/src/roles/ha_db_hana/tasks/primary-node-kill.yml +++ b/src/roles/ha_db_hana/tasks/primary-node-kill.yml @@ -22,6 +22,7 @@ when: - node_tier == "hana" - pre_validations_status == "PASSED" + - saphanasr_provider | default('SAPHanaSR') == "SAPHanaSR-angi" block: - name: "Test Execution: Kill the primary node." when: ansible_hostname == cluster_status_pre.primary_node diff --git a/src/roles/ha_db_hana/tasks/secondary-block-network.yml b/src/roles/ha_db_hana/tasks/secondary-block-network.yml index 4b51ec32..7213c526 100644 --- a/src/roles/ha_db_hana/tasks/secondary-block-network.yml +++ b/src/roles/ha_db_hana/tasks/secondary-block-network.yml @@ -1,13 +1,22 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +# /*--------------------------------------------------------------------------- +# | Block Network Communication From Secondary Site | +# +--------------------------------------------------------------------------*/ - name: "Test Setup Tasks" ansible.builtin.include_tasks: "roles/misc/tasks/test-case-setup.yml" run_once: true +# /*--------------------------------------------------------------------------- +# | Pre Validations | +# +--------------------------------------------------------------------------*/ - name: "Pre Validations: HANA DB Nodes" ansible.builtin.include_tasks: "roles/misc/tasks/pre-validations-db.yml" +# /*--------------------------------------------------------------------------- +# | Test Execution | +# +--------------------------------------------------------------------------*/ - name: "Test Execution: Block Network Communication From Secondary Master" become: true when: @@ -15,32 +24,30 @@ - pre_validations_status == "PASSED" - hana_topology == 'scale_out_hsr' - cluster_status_pre.stonith_action == "reboot" + - ansible_hostname == primary_master_node block: - name: "Test Execution: Start timer on primary master" - when: ansible_hostname == primary_master_node ansible.builtin.set_fact: test_execution_start: "{{ now(utc=true, fmt='%Y-%m-%d %H:%M:%S') }}" test_execution_hostname: "{{ hostvars[secondary_master_node].ansible_hostname }}" - - name: "Test Execution: Prepare network partition facts" - when: ansible_hostname == secondary_master_node - ansible.builtin.set_fact: - secondary_node_ip: "{{ hostvars[secondary_master_node].ansible_host }}" - blocked_ips: "{{ cluster_status_pre.primary_site_nodes | map('extract', hostvars, 'ansible_host') | list }}" - - - name: "Test Execution: Create firewall on secondary master" - when: ansible_hostname == secondary_master_node - ansible.builtin.shell: | - {% for ip in blocked_ips %} - iptables -A INPUT -s {{ ip }} -j DROP; - iptables -A OUTPUT -d {{ ip }} -j DROP; - {% endfor %} + - name: "Test Execution: Block all network communication on secondary site" + become: true + ansible.builtin.shell: "sleep 3; iptables -P INPUT DROP; iptables -P OUTPUT DROP" + async: 15 + poll: 0 register: firewall_rule_created - changed_when: firewall_rule_created.rc == 0 - failed_when: firewall_rule_created.rc != 0 + delegate_to: "{{ secondary_node }}" + loop: "{{ cluster_status_pre.secondary_site_nodes }}" + loop_control: + loop_var: secondary_node + ignore_unreachable: true + + - name: "Test Execution: Wait for the cluster to be in a stable state" + ansible.builtin.wait_for: + timeout: "{{ default_timeout }}" - name: "Test Execution: Validate HANA DB cluster status during partition" - when: ansible_hostname == primary_master_node get_cluster_status_db: db_instance_number: "{{ db_instance_number }}" operation_step: "test_execution" @@ -55,31 +62,13 @@ until: > cluster_status_test_execution.primary_node != "" and cluster_status_test_execution.primary_node in cluster_status_pre.primary_site_nodes and - ( - cluster_status_test_execution.secondary_node == "" or - cluster_status_test_execution.secondary_node in cluster_status_pre.secondary_site_nodes - ) - - - name: "Test Execution: Remove firewall rule on secondary master" - when: ansible_hostname == secondary_master_node - ansible.builtin.shell: | - {% for ip in blocked_ips %} - iptables -D INPUT -s {{ ip }} -j DROP 2>/dev/null; - iptables -D OUTPUT -d {{ ip }} -j DROP 2>/dev/null; - {% endfor %} - true - register: firewall_rule_deleted - changed_when: true - failed_when: false - ignore_unreachable: true + cluster_status_test_execution.secondary_node == "" - name: "Test Execution: Wait for the cluster to be in a stable state" - when: ansible_hostname == primary_master_node ansible.builtin.wait_for: timeout: "{{ default_timeout }}" - - name: "Test Execution: Validate HANA DB cluster status 2" - when: ansible_hostname == primary_master_node + - name: "Test Execution: Validate HANA DB cluster status (post recovery)" get_cluster_status_db: db_instance_number: "{{ db_instance_number }}" operation_step: "post_failover" @@ -97,17 +86,18 @@ cluster_status_post.primary_node in cluster_status_pre.primary_site_nodes and cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes + - name: "Test Execution: Stop timer" + ansible.builtin.set_fact: + test_execution_end: "{{ now(utc=true, fmt='%Y-%m-%d %H:%M:%S') }}" + - name: "Test Execution: Determine test execution status" - when: ansible_hostname == primary_master_node ansible.builtin.set_fact: test_execution_status: "{{ 'PASSED' if ( cluster_status_post.primary_node in cluster_status_pre.primary_site_nodes and cluster_status_post.secondary_node in cluster_status_pre.secondary_site_nodes ) else 'FAILED' }}" - test_execution_end: "{{ now(utc=true, fmt='%Y-%m-%d %H:%M:%S') }}" - name: "Set test case message and details" - when: ansible_hostname == primary_master_node ansible.builtin.set_fact: test_case_message_from_test_case: | Source node: {{ secondary_master_node }} @@ -117,16 +107,29 @@ "Pre Validations: Remove any location_constraints": "{{ location_constraints_results }}", "Pre Validations: Validate HANA DB cluster status": "{{ cluster_status_pre }}", "Pre Validations: CleanUp any failed resource": "{{ cleanup_failed_resource_pre }}", - "Test Execution: Firewall Rule Created": "{{ hostvars[secondary_master_node].firewall_rule_created | default('N/A') }}", + "Test Execution: Network blocked on secondary site": "{{ firewall_rule_created | default('iptables -P DROP applied') }}", "Test Execution: Cluster Status during partition": "{{ cluster_status_test_execution }}", - "Test Execution: Firewall Rule Removed": "{{ hostvars[secondary_master_node].firewall_rule_deleted | default('Node rebooted') }}", + "Test Execution: Firewall removed by": "Secondary site nodes rebooted by fencing", "Post Validations: Validate HANA DB cluster status": "{{ cluster_status_post }}", - } + } + # /*--------------------------------------------------------------------------- + # | Post Validations | + # +--------------------------------------------------------------------------*/ - name: "Post Validations Tasks" ansible.builtin.include_tasks: "roles/misc/tasks/post-validations.yml" rescue: + - name: "Test Execution Failure: Reset iptables policies on secondary site nodes" + become: true + ansible.builtin.shell: "iptables -P INPUT ACCEPT; iptables -P OUTPUT ACCEPT" + register: firewall_rule_deleted + changed_when: firewall_rule_deleted.rc == 0 + failed_when: false + ignore_unreachable: true + loop: "{{ cluster_status_pre.secondary_site_nodes }}" + delegate_to: "{{ item }}" + - name: "Rescue operation" ansible.builtin.include_tasks: "roles/misc/tasks/rescue.yml" diff --git a/src/vars/input-api.yaml b/src/vars/input-api.yaml index 22ac89c9..7cb17553 100644 --- a/src/vars/input-api.yaml +++ b/src/vars/input-api.yaml @@ -444,7 +444,7 @@ sap_port_to_ping: "1128" # Default values for retries, delay, timeout default_retries: 75 default_delay: 10 -default_timeout: 60 +default_timeout: 90 ascs_stonith_timeout: 120 # Default values for Azure Backup test cases