Skip to content

Commit 3acc894

Browse files
nvnmandadhinvnmandadhi
committed
add a recipe for cross region internal alb
1 parent 07a2345 commit 3acc894

File tree

1 file changed

+326
-0
lines changed

1 file changed

+326
-0
lines changed

‎cross-region/internal-alb/readme.md

+326
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,326 @@
1+
## Cross Region Internal Application Load Balancer
2+
3+
This recipe provides a walkthrough to set up a cross region internal application load balancer with failover and health checks. For information about the load balancer refer: [Cross Region Internal Application Load Balancer](https://cloud.google.com/load-balancing/docs/l7-internal/setting-up-l7-cross-reg-hybrid).
4+
5+
Initial project configuration
6+
7+
```shell
8+
gcloud config configurations create cross-region-alb
9+
gcloud auth login --update-adc
10+
11+
export PROJECT_ID=cross-region-lb-sandbox
12+
gcloud auth application-default set-quota-project cross-region-lb-sandbox
13+
gcloud config set project cross-region-lb-sandbox
14+
```
15+
16+
Enable required APIs
17+
18+
```shell
19+
gcloud services enable \
20+
cloudresourcemanager.googleapis.com
21+
compute.googleapis.com
22+
container.googleapis.com
23+
```
24+
25+
VPC Network setup
26+
27+
```shell
28+
gcloud compute networks create mc-net --subnet-mode=custom --mtu=1450\
29+
--bgp-routing-mode=global
30+
31+
gcloud compute networks subnets create snet1 --range=192.168.0.0/20 \
32+
--stack-type=IPV4_ONLY --network=mc-net --region=us-central1 \
33+
--secondary-range=snet1-range1=6.0.0.0/20,snet1-range2=6.0.16.0/20 \
34+
--enable-private-ip-google-access
35+
36+
gcloud compute networks subnets create snet2 --range=192.168.16.0/20 \
37+
--stack-type=IPV4_ONLY --network=mc-net --region=us-east1 \
38+
--secondary-range=snet2-range1=6.0.32.0/20,snet2-range2=6.0.48.0/20 \
39+
--enable-private-ip-google-access
40+
41+
gcloud compute firewall-rules create iap-fw --direction=INGRESS \
42+
--priority=1000 --network=mc-net --action=ALLOW --rules=tcp:22 \
43+
--source-ranges=35.235.240.0/20
44+
```
45+
46+
Create GKE clusters
47+
48+
```shell
49+
gcloud iam service-accounts create gke-ap-sa
50+
export GSA_EMAIL=gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com
51+
52+
gcloud projects add-iam-policy-binding ${PROJECT_ID} \
53+
--member serviceAccount:${GSA_EMAIL} \
54+
--role roles/container.admin
55+
56+
gcloud container clusters create-auto "mc1" --region "us-central1" --release-channel "rapid" \
57+
--enable-private-nodes --enable-private-endpoint --master-ipv4-cidr "6.0.128.0/28" \
58+
--enable-master-authorized-networks --enable-master-global-access \
59+
--enable-master-global-access --master-authorized-networks 192.168.0.0/16 \
60+
--network "projects/${PROJECT_ID}/global/networks/mc-net" \
61+
--subnetwork "projects/${PROJECT_ID}/regions/us-central1/subnetworks/snet1" \
62+
--cluster-secondary-range-name "snet1-range1" --services-secondary-range-name "snet1-range2" \
63+
--service-account gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com --scopes=cloud-platform --async
64+
65+
gcloud container clusters create-auto "mc2" --region "us-east1" --release-channel "rapid" \
66+
--enable-private-nodes --enable-private-endpoint --master-ipv4-cidr "6.0.128.16/28" \
67+
--enable-master-authorized-networks --enable-master-global-access --master-authorized-networks 192.168.0.0/16 \
68+
--network "projects/${PROJECT_ID}/global/networks/mc-net" \
69+
--subnetwork "projects/${PROJECT_ID}/regions/us-east1/subnetworks/snet2" --cluster-secondary-range-name "snet2-range1" \
70+
--services-secondary-range-name "snet2-range2" \
71+
--service-account gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com --scopes=cloud-platform --async
72+
```
73+
74+
Create a test instance in central region
75+
76+
```shell
77+
gcloud compute instances create mc-test \
78+
--zone=us-central1-a \
79+
--machine-type=e2-small \
80+
--network-interface=stack-type=IPV4_ONLY,subnet=snet1,no-address \
81+
--maintenance-policy=MIGRATE \
82+
--provisioning-model=STANDARD \
83+
--service-account=gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com \
84+
--scopes=https://www.googleapis.com/auth/cloud-platform \
85+
--create-disk=auto-delete=yes,boot=yes,device-name=mc-test,image=projects/debian-cloud/global/images/debian-11-bullseye-v20231004,mode=rw,size=10,type=projects/${PROJECT_ID}/zones/us-central1-a/diskTypes/pd-balanced \
86+
--shielded-secure-boot \
87+
--shielded-vtpm \
88+
--shielded-integrity-monitoring \
89+
--labels=goog-ec-src=vm_add-gcloud \
90+
--reservation-affinity=any
91+
```
92+
93+
SSH into the test instance and install tools
94+
95+
```shell
96+
gcloud compute ssh --zone "us-central1-a" "mc-test" --tunnel-through-iap
97+
sudo apt-get install kubectl google-cloud-sdk-gke-gcloud-auth-plugin
98+
```
99+
100+
Get cluster credentials
101+
102+
```shell
103+
gcloud container clusters get-credentials mc1 --region us-central1
104+
gcloud container clusters get-credentials mc2 --region us-east1
105+
106+
kubectl config rename-context gke_cross-region-lb-sandbox_us-central1_mc1 mc1
107+
kubectl config rename-context gke_cross-region-lb-sandbox_us-east1_mc2 mc2
108+
```
109+
110+
Deploy sample workloads
111+
112+
```shell
113+
for i in 1 2; do kubectl --context mc${i} apply -f - <<EOF
114+
apiVersion: apps/v1
115+
kind: Deployment
116+
metadata:
117+
name: whereami
118+
labels:
119+
app: whereami
120+
spec:
121+
replicas: 3
122+
selector:
123+
matchLabels:
124+
app: whereami
125+
template:
126+
metadata:
127+
labels:
128+
app: whereami
129+
spec:
130+
topologySpreadConstraints:
131+
- maxSkew: 1
132+
topologyKey: topology.kubernetes.io/zone
133+
whenUnsatisfiable: DoNotSchedule
134+
containers:
135+
- name: frontend
136+
image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.20
137+
ports:
138+
- containerPort: 8080
139+
EOF
140+
done
141+
```
142+
143+
Create the services
144+
145+
```shell
146+
for i in 1 2; do kubectl --context mc${i} apply -f - <<EOF
147+
apiVersion: v1
148+
kind: Service
149+
metadata:
150+
labels:
151+
app: whereami
152+
annotations:
153+
cloud.google.com/neg: '{"exposed_ports": {"80":{"name": "whereami"}}}'
154+
name: whereami
155+
spec:
156+
ports:
157+
- port: 80
158+
protocol: TCP
159+
targetPort: 8080
160+
selector:
161+
app: whereami
162+
EOF
163+
done
164+
```
165+
166+
Create proxy subnets
167+
168+
```shell
169+
gcloud beta compute networks subnets create proxy-snet1 \
170+
--purpose=GLOBAL_MANAGED_PROXY \
171+
--role=ACTIVE \
172+
--region=us-central1 \
173+
--network=mc-net \
174+
--range=6.0.144.0/24
175+
176+
gcloud beta compute networks subnets create proxy-snet2 \
177+
--purpose=GLOBAL_MANAGED_PROXY \
178+
--role=ACTIVE \
179+
--region=us-east1 \
180+
--network=mc-net \
181+
--range=6.0.145.0/24
182+
```
183+
184+
Create firewall rules
185+
186+
```shell
187+
gcloud compute firewall-rules create fw-allow-health-check \
188+
--network=mc-net \
189+
--action=allow \
190+
--direction=ingress \
191+
--source-ranges=130.211.0.0/22,35.191.0.0/16 \
192+
--rules=tcp:8080
193+
194+
gcloud compute firewall-rules create fw-allow-proxy-only-subnet \
195+
--network=mc-net \
196+
--action=allow \
197+
--direction=ingress \
198+
--source-ranges=6.0.144.0/24,6.0.145.0/24 \
199+
--rules=tcp:8080
200+
```
201+
202+
Set up the load balancer
203+
204+
```shell
205+
gcloud compute health-checks create http gil7-basic-check \
206+
--use-serving-port \
207+
--global
208+
209+
gcloud compute backend-services create whereami \
210+
--load-balancing-scheme=INTERNAL_MANAGED \
211+
--protocol=HTTP \
212+
--enable-logging \
213+
--logging-sample-rate=1.0 \
214+
--health-checks=gil7-basic-check \
215+
--global-health-checks \
216+
--global
217+
218+
for zone in $(gcloud container clusters describe mc1 --region us-central1 --format json | \
219+
jq -r .locations[]); do gcloud compute backend-services add-backend whereami \
220+
--global --balancing-mode=RATE --max-rate-per-endpoint=1000 \
221+
--network-endpoint-group=whereami --network-endpoint-group-zone=${zone}; done
222+
223+
for zone in $(gcloud container clusters describe mc2 --region us-east1 --format json | \
224+
jq -r .locations[]); do gcloud compute backend-services add-backend whereami \
225+
--global --balancing-mode=RATE --max-rate-per-endpoint=1000 \
226+
--network-endpoint-group=whereami --network-endpoint-group-zone=${zone}; done
227+
228+
gcloud compute url-maps create gil7-lb \
229+
--default-service=whereami \
230+
--global
231+
232+
gcloud compute target-http-proxies create gil7-http-proxy \
233+
--url-map=gil7-lb \
234+
--global
235+
236+
gcloud compute addresses create ip-central --region=us-central1 --subnet=snet1 --purpose=GCE_ENDPOINT
237+
238+
gcloud compute addresses create ip-east --region=us-east1 --subnet=snet2 --purpose=GCE_ENDPOINT
239+
240+
export IP_CENTRAL=$(gcloud compute addresses describe ip-central --region us-central1 --format json | jq -r .address)
241+
242+
export IP_EAST=$(gcloud compute addresses describe ip-east --region us-east1 --format json | jq -r .address)
243+
244+
gcloud compute forwarding-rules create gil7-fw-rule-central --load-balancing-scheme=INTERNAL_MANAGED \
245+
--network=mc-net --subnet=snet1 --subnet-region=us-central1 \
246+
--address=${IP_CENTRAL} --ports=80 --target-http-proxy=gil7-http-proxy --global
247+
248+
gcloud compute forwarding-rules create gil7-fw-rule-east --load-balancing-scheme=INTERNAL_MANAGED \
249+
--network=mc-net --subnet=snet2 --subnet-region=us-east1 \
250+
--address=${IP_EAST} --ports=80 --target-http-proxy=gil7-http-proxy --global
251+
```
252+
253+
Configure DNS zone, this also enables health checking the backends for failover
254+
255+
```shell
256+
gcloud dns managed-zones create cross-region-lb-internal --description="" \
257+
--dns-name="cross-region-lb.apps.internal." --visibility="private" \
258+
--networks="https://www.googleapis.com/compute/v1/projects/${PROJECT_ID}/global/networks/mc-net"
259+
260+
gcloud dns record-sets create whereami.cross-region-lb.apps.internal --ttl="30" \
261+
--type="A" --zone="cross-region-lb-internal" --routing-policy-type="GEO" \
262+
--routing-policy-data="us-central1=gil7-fw-rule-central@global;us-east1=gil7-fw-rule-east@global" \
263+
--enable-health-checking
264+
```
265+
266+
Create a test instance in us-east1
267+
268+
```shell
269+
gcloud compute instances create mc-test --zone=us-east1-d --machine-type=e2-small \
270+
--network-interface=stack-type=IPV4_ONLY,subnet=snet2,no-address \
271+
--maintenance-policy=MIGRATE --provisioning-model=STANDARD \
272+
--service-account=gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com \
273+
--scopes=https://www.googleapis.com/auth/cloud-platform \
274+
--create-disk=auto-delete=yes,boot=yes,device-name=mc-test,image=projects/debian-cloud/global/images/debian-11-bullseye-v20231004,mode=rw,size=10,type=projects/${PROJECT_ID}/zones/us-east1-d/diskTypes/pd-balanced \
275+
--shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring \
276+
--labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any
277+
```
278+
279+
Test GEO routing
280+
281+
Cloud DNS routes the requests to backends in respective regions
282+
283+
```shell
284+
for zone in us-central1-a us-east1-d; do
285+
gcloud compute ssh mc-test --zone ${zone} -- curl whereami.cross-region-lb.apps.internal;
286+
done
287+
```
288+
289+
Test failover
290+
291+
The requests would failover to the central region since there are no healthy endpoints in east region
292+
293+
```shell
294+
kubectl scale --replicas 0 deploy/whereami --context mc2
295+
```
296+
297+
Clean up resources
298+
299+
```shell
300+
gcloud container clusters delete mc1 --region us-central1
301+
gcloud container clusters delete mc2 --region us-east1
302+
for zone in us-central1-a us-east1-d; do
303+
gcloud compute instances delete mc-test --zone ${zone};
304+
done
305+
306+
gcloud dns record-sets delete whereami.cross-region-lb.apps.internal \
307+
--type="A" --zone="cross-region-lb-internal"
308+
gcloud dns managed-zones delete cross-region-lb-internal
309+
310+
gcloud compute forwarding-rules delete gil7-fw-rule-central --global
311+
gcloud compute forwarding-rules delete gil7-fw-rule-east --global
312+
gcloud compute target-http-proxies delete gil7-http-proxy --global
313+
gcloud compute url-maps delete gil7-lb --global
314+
315+
for zone in $(gcloud container clusters describe mc1 --region us-central1 --format json | \
316+
jq -r .locations[]); do gcloud compute backend-services remove-backend whereami \
317+
--global --network-endpoint-group=whereami --network-endpoint-group-zone=${zone}; done
318+
319+
for zone in $(gcloud container clusters describe mc2 --region us-east1 --format json | \
320+
jq -r .locations[]); do gcloud compute backend-services remove-backend whereami \
321+
--global --network-endpoint-group=whereami --network-endpoint-group-zone=${zone}; done
322+
323+
gcloud compute backend-services delete whereami --global
324+
```
325+
326+

0 commit comments

Comments
 (0)