Ensure etcd cluster is updated serially

Without this, when upgrading the cluster (eg for a stemcell version
change), the first node will be upgraded, then the other 2 will be
upgraded in parallel, resulting in the cluster losing quorum. This then
requires manual intervention to recover from.

Making etcd_z2 serial as well ensures that one node gets done at a time.

Author: alext

manifests/cf-manifest/deployments/030-cf-jobs.yml

@@ -247,21 +247,6 @@ jobs:
           services:
             etcd: {}
 
-  - name: consul_z3
-    templates: (( grab meta.consul_templates ))
-    instances: 1
-    persistent_disk: 1024
-    resource_pool: small_z3
-    networks:
-      - name: cf3
-        static_ips:
-    properties:
-      consul:
-        agent:
-          mode: server
-      metron_agent:
-        zone: z3
-
   - name: etcd_z2
     templates: (( grab meta.etcd_templates ))
     instances: 1
@@ -270,6 +255,8 @@ jobs:
     networks:
       - name: cf2
         static_ips:
+    update:
+      serial: true
     properties:
       metron_agent:
         zone: z2
@@ -278,6 +265,21 @@ jobs:
           services:
             etcd: {}
 
+  - name: consul_z3
+    templates: (( grab meta.consul_templates ))
+    instances: 1
+    persistent_disk: 1024
+    resource_pool: small_z3
+    networks:
+      - name: cf3
+        static_ips:
+    properties:
+      consul:
+        agent:
+          mode: server
+      metron_agent:
+        zone: z3
+
   - name: etcd_z3
     templates: (( grab meta.etcd_templates ))
     instances: 1

manifests/cf-manifest/spec/jobs_spec.rb

@@ -43,17 +43,21 @@ def ordered(job1_name, job2_name)
     end
   end
 
-  describe "in order to start one etcd master for consensus" do
+  describe "in order to start/upgrade etcd cluster while maintaining consensus" do
     it "has etcd_z1 serial" do
       expect(is_serial("etcd_z1")).to be true
     end
 
+    it "has etcd_z2 serial" do
+      expect(is_serial("etcd_z2")).to be true
+    end
+
     it "has etcd_z1 before etcd_z2" do
       expect(ordered("etcd_z1", "etcd_z2")).to be true
     end
 
-    it "has etcd_z1 before etcd_z3" do
-      expect(ordered("etcd_z1", "etcd_z3")).to be true
+    it "has etcd_z2 before etcd_z3" do
+      expect(ordered("etcd_z2", "etcd_z3")).to be true
     end
   end