Zookeeper migration patches (#10)

* Make zookeeper ids start from an offset

* Bootstrap zookeeper cluster from an external SEED_NODE

* Use public ip for internal quorum election

* Delay zookeeper start to allow for LB registration

* Fix a bug when a node is teared down

* Implement review

Author: dobrerazvan

docker/bin/zookeeperFunctions.sh

@@ -13,7 +13,12 @@
 set -ex
 
 function zkConfig() {
-  echo "$HOST.$DOMAIN:$QUORUM_PORT:$LEADER_PORT:$ROLE;$CLIENT_PORT"
+  if [ -n "$1" ]; then
+    FQDN="$1"
+  else
+    FQDN="$HOST.$DOMAIN"
+  fi
+  echo "$FQDN:$QUORUM_PORT:$LEADER_PORT:$ROLE;$CLIENT_PORT"
 }
 
 function zkConnectionString() {
@@ -27,4 +32,4 @@ function zkConnectionString() {
     set -e
     echo "${CLIENT_HOST}:${CLIENT_PORT}"
   fi
-}
+}

docker/bin/zookeeperReady.sh

@@ -20,6 +20,14 @@ MYID_FILE=$DATA_DIR/myid
 LOG4J_CONF=/conf/log4j-quiet.properties
 STATIC_CONFIG=/data/conf/zoo.cfg
 
+# used when zkid starts from value grater then 1, default 1
+OFFSET=${OFFSET:-1}
+
+# use SEED_NODE to bootstrap the current zookeeper cluster, else default to local cluster
+# CLIENT_HOST is used in zkConnectionString function already to create zkURL
+CLIENT_HOST=${SEED_NODE:-$CLIENT_HOST}
+
+
 OK=$(echo ruok | nc 127.0.0.1 $CLIENT_PORT)
 
 # Check to see if zookeeper service answers
@@ -44,8 +52,16 @@ if [[ "$OK" == "imok" ]]; then
         echo Failed to parse name and ordinal of Pod
         exit 1
     fi
-    MYID=$((ORD+1))
+    MYID=$(($ORD+$OFFSET))
     ONDISK_CONFIG=false
+    
+    # use FQDN_TEMPLATE to create an OUTSIDE_NAME that is going to be used to establish quorum election
+    # this should be used along with the quorumListenOnAllIPs set to true
+    # % from the FQDN_TEMPLATE will be replaced with pod index+1
+    if [ -n "$FQDN_TEMPLATE" ]; then
+      OUTSIDE_NAME=$(echo ${FQDN_TEMPLATE} | sed "s/%/$(($ORD+1))/g")
+    fi
+    
     if [ -f $MYID_FILE ]; then
       EXISTING_ID="`cat $DATA_DIR/myid`"
       if [[ "$EXISTING_ID" == "$MYID" && -f $STATIC_CONFIG ]]; then
@@ -74,7 +90,7 @@ if [[ "$OK" == "imok" ]]; then
       echo "Zookeeper service is ready to be upgraded from observer to participant."
       ROLE=participant
       ZKURL=$(zkConnectionString)
-      ZKCONFIG=$(zkConfig)
+      ZKCONFIG=$(zkConfig $OUTSIDE_NAME)
       java -Dlog4j.configuration=file:"$LOG4J_CONF" -jar /root/zu.jar remove $ZKURL $MYID
       sleep 1
       java -Dlog4j.configuration=file:"$LOG4J_CONF" -jar /root/zu.jar add $ZKURL $MYID $ZKCONFIG

docker/bin/zookeeperStart.sh

@@ -14,13 +14,19 @@ set -ex
 source /conf/env.sh
 source /usr/local/bin/zookeeperFunctions.sh
 
+START_DELAY=${START_DELAY:-0}
+
+
 HOST=`hostname -s`
 DATA_DIR=/data
 MYID_FILE=$DATA_DIR/myid
 LOG4J_CONF=/conf/log4j-quiet.properties
 DYNCONFIG=$DATA_DIR/zoo.cfg.dynamic
 STATIC_CONFIG=/data/conf/zoo.cfg
 
+# used when zkid starts from value grater then 1, default 1
+OFFSET=${OFFSET:-1}
+
 # Extract resource name and this members ordinal value from pod hostname
 if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
     NAME=${BASH_REMATCH[1]}
@@ -30,7 +36,27 @@ else
     exit 1
 fi
 
-MYID=$((ORD+1))
+MYID=$(($ORD+$OFFSET))
+
+# use SEED_NODE to bootstrap the current zookeeper cluster, else default to local cluster
+# CLIENT_HOST is used in zkConnectionString function already to create zkURL
+CLIENT_HOST=${SEED_NODE:-$CLIENT_HOST}
+
+
+# use FQDN_TEMPLATE to create an OUTSIDE_NAME that is going to be used to establish quorum election
+# this should be used along with the quorumListenOnAllIPs set to true
+# % from the FQDN_TEMPLATE will be replaced with pod index+1
+if [ -n "$FQDN_TEMPLATE" ]; then
+  OUTSIDE_NAME=$(echo ${FQDN_TEMPLATE} | sed "s/%/$(($ORD+1))/g")
+fi
+
+# domain should be the OUTSIDE_NAME for when it's set
+DOMAIN=${SEED_NODE:-$DOMAIN}
+
+# wait for loadbalancer registration and skip the first one
+if [ $MYID -gt $OFFSET ]; then
+  sleep $START_DELAY
+fi
 
 # Values for first startup
 WRITE_CONFIGURATION=true
@@ -114,10 +140,10 @@ fi
 if [[ "$WRITE_CONFIGURATION" == true ]]; then
   echo "Writing myid: $MYID to: $MYID_FILE."
   echo $MYID > $MYID_FILE
-  if [[ $MYID -eq 1 ]]; then
+  if [[ $MYID -eq $OFFSET && -z "$SEED_NODE" ]]; then
     ROLE=participant
     echo Initial initialization of ordinal 0 pod, creating new config.
-    ZKCONFIG=$(zkConfig)
+    ZKCONFIG=$(zkConfig $OUTSIDE_NAME)
     echo Writing bootstrap configuration with the following config:
     echo $ZKCONFIG
     echo $MYID > $MYID_FILE
@@ -128,7 +154,7 @@ fi
 if [[ "$REGISTER_NODE" == true ]]; then
     ROLE=observer
     ZKURL=$(zkConnectionString)
-    ZKCONFIG=$(zkConfig)
+    ZKCONFIG=$(zkConfig $OUTSIDE_NAME)
     set -e
     echo Registering node and writing local configuration to disk.
     java -Dlog4j.configuration=file:"$LOG4J_CONF" -jar /root/zu.jar add $ZKURL $MYID  $ZKCONFIG $DYNCONFIG

docker/bin/zookeeperTeardown.sh

@@ -18,6 +18,13 @@ DATA_DIR=/data
 MYID_FILE=$DATA_DIR/myid
 LOG4J_CONF=/conf/log4j-quiet.properties
 
+# use SEED_NODE to bootstrap the current zookeeper cluster, else default to local cluster
+# CLIENT_HOST is used in zkConnectionString function already to create zkURL
+CLIENT_HOST=${SEED_NODE:-$CLIENT_HOST}
+
+# used when zkid starts from value grater then 1, default 1
+OFFSET=${OFFSET:-1}
+
 # Wait for client connections to drain. Kubernetes will wait until the confiugred
 # "terminationGracePeriodSeconds" before focibly killing the container
 CONN_COUNT=`echo cons | nc localhost 2181 | grep -v "^$" |grep -v "/127.0.0.1:" | wc -l`
@@ -40,7 +47,7 @@ MYID=`cat $MYID_FILE`
 ZNODE_PATH="/zookeeper-operator/$CLUSTER_NAME"
 CLUSTERSIZE=`java -Dlog4j.configuration=file:"$LOG4J_CONF" -jar /root/zu.jar sync $ZKURL $ZNODE_PATH`
 echo "CLUSTER_SIZE=$CLUSTERSIZE, MyId=$MYID"
-if [[ -n "$CLUSTERSIZE" && "$CLUSTERSIZE" -lt "$MYID" ]]; then
+if [[ -n "$CLUSTERSIZE" && "$(($CLUSTERSIZE+$OFFSET-1))" -lt "$MYID" ]]; then
   # If ClusterSize < MyId, this server is being permanantly removed.
   java -Dlog4j.configuration=file:"$LOG4J_CONF" -jar /root/zu.jar remove $ZKURL $MYID
   echo $?