Robustness improvements for rolling upgrade. Also fixes issue #33 (#46)

bczoma · PhilippeKhalife · commit ec1e726a58cd · 2019-01-16T13:03:07.000-05:00
* Additional checks for readiness, increase config-sync-check timeout, enable local debug.log
diff --git a/.travis.yml b/.travis.yml
@@ -43,6 +43,7 @@ script:
   - kubectl get statefulset,svc,pods,pvc,pv --show-labels
   - echo "Waiting for cluster to become active"
   - "travis_wait 30 sleep 1800 &"
+  - sleep 40; kubectl describe nodes # This shall show resource availability
   - until kubectl get pods --show-labels | grep solace-0 | grep -m 1 -E 'active=true'; do sleep 10; done
   - until kubectl get pods --show-labels | grep solace-1 | grep -m 1 -E '1/1'; do sleep 10; done
   - until kubectl get pods --show-labels | grep solace-2 | grep -m 1 -E '1/1'; do sleep 10; done
diff --git a/solace/templates/solaceConfigMap.yaml b/solace/templates/solaceConfigMap.yaml
@@ -1,4 +1,3 @@
----
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -17,7 +16,7 @@ data:
       export service_webtransport_port='60080'
       export service_webtransport_tlsport='60443'
       export service_semp_tlsport='60943'
-      export logging_debug_output=stdout
+      export logging_debug_output=all
 {{- if eq .Values.solace.size "dev100" }}
       export system_scaling_maxconnectioncount="100"
 {{- else if eq .Values.solace.size "prod100" }}
@@ -83,7 +82,7 @@ data:
       IFS='-' read -ra host_array <<< $(hostname)
       node_ordinal=${host_array[-1]}
       password=`cat {{ .Values.filepaths.secrets }}/username_admin_password`
-      loop_guard=30
+      loop_guard=60
       pause=10
       count=0
       resync_step=""
@@ -93,7 +92,7 @@ data:
         echo "`date` INFO: ${APP}-Determine if Active or Backup role to know which Virtual router to check"
         while [ ${count} -lt ${loop_guard} ]; do 
           role_results=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-                -q "<rpc semp-version='soltr/8_5VMR'><show><redundancy><detail/></redundancy></show></rpc>" \
+                -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
                 -v "/rpc-reply/rpc/show/redundancy/active-standby-role[text()]"`
           case "`echo ${role_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`" in
             "Primary")
@@ -118,7 +117,7 @@ data:
         echo "`date` INFO: ${APP}-Active or Backup role is ${role}"
         while [ ${count} -lt ${loop_guard} ]; do 
           online_results=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-              -q "<rpc semp-version='soltr/8_5VMR'><show><redundancy><detail/></redundancy></show></rpc>" \
+              -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
               -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${role}/status/activity[text()]"`
           local_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
           echo "`date` INFO: ${APP}-Local activity state is: ${local_activity}"
@@ -155,7 +154,7 @@ data:
           echo "`date` INFO: ${APP}-Wait for mate to be 'Standby'"
           while [ ${count} -lt ${loop_guard} ]; do 
             online_results=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-                -q "<rpc semp-version='soltr/8_5VMR'><show><redundancy><detail/></redundancy></show></rpc>" \
+                -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
                 -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${role}/status/detail/priority-reported-by-mate/summary[text()]"`
             mate_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
             echo "`date` INFO: ${APP}-Mate activity state is: ${mate_activity}"
@@ -177,9 +176,9 @@ data:
         fi # if assert-master
         # If only needs to do is resync master, can issue cammand and exit.
         {{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-              -q "<rpc semp-version='soltr/8_5VMR'><admin><config-sync><${resync_step}><router/></${resync_step}></config-sync></admin></rpc>"
+              -q "<rpc><admin><config-sync><${resync_step}><router/></${resync_step}></config-sync></admin></rpc>"
         {{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-              -q "<rpc semp-version='soltr/8_5VMR'><admin><config-sync><${resync_step}><vpn-name>default</vpn-name></${resync_step}></config-sync></admin></rpc>"
+              -q "<rpc><admin><config-sync><${resync_step}><vpn-name>default</vpn-name></${resync_step}></config-sync></admin></rpc>"
         echo "`date` INFO: ${APP}-Solace VMR bringup complete"
       fi # if not monitor
 {{- end }}
@@ -220,13 +219,71 @@ data:
     node_ordinal=${host_array[-1]}
     echo "`date` INFO: ${APP}-node ordinal: ${node_ordinal}"
     password=`cat {{ .Values.filepaths.secrets }}/username_admin_password`
+
+    # For upgrade purposes, ensure redundancy is up only when the pod is started
+    redundacycheck_file=/tmp/redundacycheck
+    if [ ! -f ${redundacycheck_file} ]; then
+      # First check all nodes are online
+      results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+              -q "<rpc><show><redundancy><group/></redundancy></show></rpc>" \
+              -c "/rpc-reply/rpc/show/redundancy/group-node/status[text() = \"Online\"]"`
+      nr_node_results=`echo ${role_results} |  xmllint -xpath "string(returnInfo/countSearchResult)" -`
+      if [ $nr_node_results -ne 3 ]; then
+        echo "`date` INFO: ${APP}-Not all nodes are online. Query results: ${nr_node_results}"
+        exit 1
+      fi
+      # Then for each node determine the ip address and check redundancy. Note: id starts here from 1 and not 0.
+      for id in 1 2 3; do
+        results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+                -q "<rpc><show><redundancy><group/></redundancy></show></rpc>" \
+                -v "//ip-address[$id]"`
+        node_ip_address=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
+        results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://$node_ip_address:8080/SEMP \
+                -q "<rpc><show><redundancy/></show></rpc>" \
+                -v "/rpc-reply/rpc/show/redundancy/redundancy-status"`
+        redundancystatus_results=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
+        if [ "${redundancystatus_results}" != "Up" ]; then
+          echo "`date` INFO: ${APP}-Redundancy state is not yet up. Query results: ${redundancystatus_results}"
+          exit 1
+        fi
+      done
+      # Additionally check config-sync status for non-monitoring nodes
+      if [ "${node_ordinal}" != "2" ]; then
+        results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+                -q "<rpc><show><config-sync></config-sync></show></rpc>" \
+                -v "/rpc-reply/rpc/show/config-sync/status/oper-status"`
+        confsyncstatus_results=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
+        if [ "${confsyncstatus_results}" != "Up" ]; then
+          echo "`date` INFO: ${APP}-Config-sync state is not yet up. Query results: ${confsyncstatus_results}"
+          exit 1
+        fi
+      fi
+      # Then for each node check that they report 3 Consul voters.
+      for id in 1 2 3; do
+        results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+                -q "<rpc><show><redundancy><group/></redundancy></show></rpc>" \
+                -v "//ip-address[$id]"`
+        node_ip_address=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
+        nr_voter_results=`curl --unix-socket /var/run/consul -s http://$node_ip_address:8500/v1/operator/raft/configuration  | python -m json.tool | grep Voter | grep true | wc -l`
+        if [ $nr_voter_results -ne 3 ]; then
+          echo "`date` INFO: ${APP}-Still waiting for all 3 Consul voters to be present for node $node_ip_address. Query results: ${nr_voter_results}"
+          exit 1
+        fi
+      done
+      echo "Creating redundacycheck_file"
+      echo "true" > ${redundacycheck_file}
+      # wait at first startup for stability
+      #sleep 20
+    fi
+    
+
     if [ "${node_ordinal}" = "2" ]; then
       echo "`date` INFO: ${APP}-Monitor node ready check"
       # Note that when dealing with Monitor, only need to be concerned and readiness response.
       # active label will always be "false"
       echo "`date` INFO: ${APP}-For monitor node just check for 3 online nodes in group"
       role_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-              -q "<rpc semp-version='soltr/8_5VMR'><show><redundancy><group/></redundancy></show></rpc>" \
+              -q "<rpc><show><redundancy><group/></redundancy></show></rpc>" \
               -c "/rpc-reply/rpc/show/redundancy/group-node/status[text() = \"Online\"]"`
       if [ `echo ${role_results} |  xmllint -xpath "string(returnInfo/countSearchResult)" -` -eq 3 ]; then
         echo "`date` INFO: ${APP}-Monitor node is redundancy ready"
@@ -270,7 +327,7 @@ data:
         ;;
     esac
     online_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-            -q "<rpc semp-version='soltr/8_5VMR'><show><redundancy><detail/></redundancy></show></rpc>" \
+            -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
             -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${config_role}/status/activity[text()]"`
     local_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
     echo "`date` INFO: ${APP}-Local activity state is: ${local_activity}"
@@ -385,4 +442,4 @@ data:
           echo -e "`date` INFO: ${APP}-${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2
           echo  "<returnInfo><errorInfo></errorInfo><countSearchResult>${count_result}</countSearchResult></returnInfo>"
           exit 0
-      fi
+      fi
