Skip to content

Commit bfb5f10

Browse files
nvnmandadhinvnmandadhi
committed
add a recipe for cross region internal alb
1 parent 07a2345 commit bfb5f10

File tree

1 file changed

+302
-0
lines changed

1 file changed

+302
-0
lines changed

‎cross-region/internal-alb/readme.md

+302
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,302 @@
1+
## Cross Region Internal Application Load Balancer
2+
3+
This recipe provides a walkthrough to set up a cross region internal application load balancer with failover and health checks. For information about the load balancer refer: [Cross Region Internal Application Load Balancer](https://cloud.google.com/load-balancing/docs/l7-internal/setting-up-l7-cross-reg-hybrid).
4+
5+
Initial project configuration
6+
7+
```shell
8+
gcloud config configurations create cross-region-alb
9+
gcloud auth login --update-adc
10+
11+
export PROJECT_ID=cross-region-lb-sandbox
12+
gcloud auth application-default set-quota-project cross-region-lb-sandbox
13+
gcloud config set project cross-region-lb-sandbox
14+
```
15+
16+
Enable required APIs
17+
18+
```shell
19+
gcloud services enable cloudresourcemanager.googleapis.com
20+
compute.googleapis.com
21+
container.googleapis.com
22+
```
23+
24+
VPC Network setup
25+
26+
```shell
27+
gcloud compute networks create mc-net --subnet-mode=custom --mtu=1450\
28+
--bgp-routing-mode=global
29+
30+
gcloud compute networks subnets create snet1 --range=192.168.0.0/20 \
31+
--stack-type=IPV4_ONLY --network=mc-net --region=us-central1 \
32+
--secondary-range=snet1-range1=6.0.0.0/20,snet1-range2=6.0.16.0/20 \
33+
--enable-private-ip-google-access
34+
35+
gcloud compute networks subnets create snet2 --range=192.168.16.0/20 \
36+
--stack-type=IPV4_ONLY --network=mc-net --region=us-east1 \
37+
--secondary-range=snet2-range1=6.0.32.0/20,snet2-range2=6.0.48.0/20 \
38+
--enable-private-ip-google-access
39+
40+
gcloud compute firewall-rules create iap-fw --direction=INGRESS \
41+
--priority=1000 --network=mc-net --action=ALLOW --rules=tcp:22 \
42+
--source-ranges=35.235.240.0/20
43+
```
44+
45+
Create GKE clusters
46+
47+
```shell
48+
gcloud iam service-accounts create gke-ap-sa
49+
export GSA_EMAIL=gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com
50+
51+
gcloud projects add-iam-policy-binding ${PROJECT_ID} \
52+
--member serviceAccount:${GSA_EMAIL} \
53+
--role roles/container.admin
54+
55+
gcloud container clusters create-auto "mc1" --region "us-central1" --release-channel "rapid" \
56+
--enable-private-nodes --enable-private-endpoint --master-ipv4-cidr "6.0.128.0/28" \
57+
--enable-master-authorized-networks --enable-master-global-access \
58+
--enable-master-global-access --master-authorized-networks 192.168.0.0/16 \
59+
--network "projects/${PROJECT_ID}/global/networks/mc-net" \
60+
--subnetwork "projects/${PROJECT_ID}/regions/us-central1/subnetworks/snet1" \
61+
--cluster-secondary-range-name "snet1-range1" --services-secondary-range-name "snet1-range2" \
62+
--service-account gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com --scopes=cloud-platform --async
63+
64+
gcloud container clusters create-auto "mc2" --region "us-east1" --release-channel "rapid" \
65+
--enable-private-nodes --enable-private-endpoint --master-ipv4-cidr "6.0.128.16/28" \
66+
--enable-master-authorized-networks --enable-master-global-access --master-authorized-networks 192.168.0.0/16 \
67+
--network "projects/${PROJECT_ID}/global/networks/mc-net" \
68+
--subnetwork "projects/${PROJECT_ID}/regions/us-east1/subnetworks/snet2" --cluster-secondary-range-name "snet2-range1" \
69+
--services-secondary-range-name "snet2-range2" \
70+
--service-account gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com --scopes=cloud-platform --async
71+
```
72+
73+
Create a test instance in central region
74+
75+
```shell
76+
gcloud compute instances create mc-test \
77+
--zone=us-central1-a \
78+
--machine-type=e2-small \
79+
--network-interface=stack-type=IPV4_ONLY,subnet=snet1,no-address \
80+
--maintenance-policy=MIGRATE \
81+
--provisioning-model=STANDARD \
82+
--service-account=gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com \
83+
--scopes=https://www.googleapis.com/auth/cloud-platform \
84+
--create-disk=auto-delete=yes,boot=yes,device-name=mc-test,image=projects/debian-cloud/global/images/debian-11-bullseye-v20231004,mode=rw,size=10,type=projects/${PROJECT_ID}/zones/us-central1-a/diskTypes/pd-balanced \
85+
--shielded-secure-boot \
86+
--shielded-vtpm \
87+
--shielded-integrity-monitoring \
88+
--labels=goog-ec-src=vm_add-gcloud \
89+
--reservation-affinity=any
90+
```
91+
92+
SSH into the test instance and install tools
93+
94+
```shell
95+
gcloud compute ssh --zone "us-central1-a" "mc-test" --tunnel-through-iap
96+
sudo apt-get install kubectl google-cloud-sdk-gke-gcloud-auth-plugin
97+
```
98+
99+
Get cluster credentials
100+
101+
```shell
102+
gcloud container clusters get-credentials mc1 --region us-central1
103+
gcloud container clusters get-credentials mc2 --region us-east1
104+
105+
kubectl config rename-context gke_cross-region-lb-sandbox_us-central1_mc1 mc1
106+
kubectl config rename-context gke_cross-region-lb-sandbox_us-east1_mc2 mc2
107+
```
108+
109+
Deploy sample workloads
110+
111+
```shell
112+
for i in 1 2; do kubectl --context mc${i} apply -f - <<EOF
113+
apiVersion: apps/v1
114+
kind: Deployment
115+
metadata:
116+
name: whereami
117+
labels:
118+
app: whereami
119+
spec:
120+
replicas: 3
121+
selector:
122+
matchLabels:
123+
app: whereami
124+
template:
125+
metadata:
126+
labels:
127+
app: whereami
128+
spec:
129+
topologySpreadConstraints:
130+
- maxSkew: 1
131+
topologyKey: topology.kubernetes.io/zone
132+
whenUnsatisfiable: DoNotSchedule
133+
containers:
134+
- name: frontend
135+
image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.20
136+
ports:
137+
- containerPort: 8080
138+
EOF
139+
done
140+
```
141+
142+
Create the services
143+
144+
```shell
145+
for i in 1 2; do kubectl --context mc${i} apply -f - <<EOF
146+
apiVersion: v1
147+
kind: Service
148+
metadata:
149+
labels:
150+
app: whereami
151+
annotations:
152+
cloud.google.com/neg: '{"exposed_ports": {"80":{"name": "whereami"}}}'
153+
name: whereami
154+
spec:
155+
ports:
156+
- port: 80
157+
protocol: TCP
158+
targetPort: 8080
159+
selector:
160+
app: whereami
161+
EOF
162+
done
163+
```
164+
165+
Create proxy subnets
166+
167+
```shell
168+
gcloud beta compute networks subnets create proxy-snet1 \
169+
--purpose=GLOBAL_MANAGED_PROXY \
170+
--role=ACTIVE \
171+
--region=us-central1 \
172+
--network=mc-net \
173+
--range=6.0.144.0/24
174+
175+
gcloud beta compute networks subnets create proxy-snet2 \
176+
--purpose=GLOBAL_MANAGED_PROXY \
177+
--role=ACTIVE \
178+
--region=us-east1 \
179+
--network=mc-net \
180+
--range=6.0.145.0/24
181+
```
182+
183+
Create firewall rules
184+
185+
```shell
186+
gcloud compute firewall-rules create fw-allow-health-check \
187+
--network=mc-net \
188+
--action=allow \
189+
--direction=ingress \
190+
--source-ranges=130.211.0.0/22,35.191.0.0/16 \
191+
--rules=tcp:8080
192+
193+
gcloud compute firewall-rules create fw-allow-proxy-only-subnet \
194+
--network=mc-net \
195+
--action=allow \
196+
--direction=ingress \
197+
--source-ranges=6.0.144.0/24,6.0.145.0/24 \
198+
--rules=tcp:8080
199+
```
200+
201+
Set up the load balancer
202+
203+
```shell
204+
gcloud compute health-checks create http gil7-basic-check \
205+
--use-serving-port \
206+
--global
207+
208+
gcloud compute backend-services create whereami \
209+
--load-balancing-scheme=INTERNAL_MANAGED \
210+
--protocol=HTTP \
211+
--enable-logging \
212+
--logging-sample-rate=1.0 \
213+
--health-checks=gil7-basic-check \
214+
--global-health-checks \
215+
--global
216+
217+
for zone in $(gcloud container clusters describe mc1 --region us-central1 --format json | \
218+
jq -r .locations[]); do gcloud compute backend-services add-backend whereami \
219+
--global --balancing-mode=RATE --max-rate-per-endpoint=1000 \
220+
--network-endpoint-group=whereami --network-endpoint-group-zone=${zone}; done
221+
222+
for zone in $(gcloud container clusters describe mc2 --region us-east1 --format json | \
223+
jq -r .locations[]); do gcloud compute backend-services add-backend whereami \
224+
--global --balancing-mode=RATE --max-rate-per-endpoint=1000 \
225+
--network-endpoint-group=whereami --network-endpoint-group-zone=${zone}; done
226+
227+
gcloud compute url-maps create gil7-lb \
228+
--default-service=whereami \
229+
--global
230+
231+
gcloud compute target-http-proxies create gil7-http-proxy \
232+
--url-map=gil7-lb \
233+
--global
234+
235+
gcloud compute addresses create ip-central --region=us-central1 --subnet=snet1 --purpose=GCE_ENDPOINT
236+
237+
gcloud compute addresses create ip-east --region=us-east1 --subnet=snet2 --purpose=GCE_ENDPOINT
238+
239+
export IP_CENTRAL=$(gcloud compute addresses describe ip-central --region us-central1 --format json | jq -r .address)
240+
241+
export IP_EAST=$(gcloud compute addresses describe ip-east --region us-east1 --format json | jq -r .address)
242+
243+
gcloud compute forwarding-rules create gil7-fw-rule-central --load-balancing-scheme=INTERNAL_MANAGED \
244+
--network=mc-net --subnet=snet1 --subnet-region=us-central1 \
245+
--address=${IP_CENTRAL} --ports=80 --target-http-proxy=gil7-http-proxy --global
246+
247+
gcloud compute forwarding-rules create gil7-fw-rule-central --load-balancing-scheme=INTERNAL_MANAGED \
248+
--network=mc-net --subnet=snet2 --subnet-region=us-east1 \
249+
--address=${IP_EAST} --ports=80 --target-http-proxy=gil7-http-proxy --global
250+
```
251+
252+
Configure DNS zone, this also enables health checking the backends for failover
253+
254+
```shell
255+
gcloud dns managed-zones create cross-region-lb-internal --description="" \
256+
--dns-name="cross-region-lb.apps.internal." --visibility="private" \
257+
--networks="https://www.googleapis.com/compute/v1/projects/${PROJECT_ID}/global/networks/mc-net"
258+
259+
gcloud dns record-sets create whereami.cross-region-lb.apps.internal --ttl="30" \
260+
--type="A" --zone="cross-region-lb-internal" --routing-policy-type="GEO" \
261+
--routing-policy-data="us-central1=gil7-fw-rule-central@global;us-east1=gil7-fw-rule-east@global" \
262+
--enable-health-checking
263+
```
264+
265+
Create a test instance in us-east1
266+
267+
```shell
268+
gcloud compute instances create mc-test --zone=us-east1-d --machine-type=e2-small \
269+
--network-interface=stack-type=IPV4_ONLY,subnet=snet2,no-address \
270+
--maintenance-policy=MIGRATE --provisioning-model=STANDARD \
271+
--service-account=gke-ap-sa@${PROJECT_ID}.iam.gserviceaccount.com \
272+
--scopes=https://www.googleapis.com/auth/cloud-platform \
273+
--create-disk=auto-delete=yes,boot=yes,device-name=mc-test,image=projects/debian-cloud/global/images/debian-11-bullseye-v20231004,mode=rw,size=10,type=projects/${PROJECT_ID}/zones/us-east1-d/diskTypes/pd-balanced \
274+
--shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring \
275+
--labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any
276+
```
277+
278+
Test GEO routing
279+
280+
Cloud DNS routes the requests to backends in respective regions
281+
282+
```shell
283+
for zone in us-central1-a us-east1-d; do gcloud compute ssh mc-test --zone ${zone} -- curl whereami.cross-region-lb.apps.internal; done
284+
```
285+
286+
Test failover
287+
288+
The requests would failover to the central region since there are no healthy endpoints in east region
289+
290+
```shell
291+
kubectl scale --replicas 0 deploy/whereami --context mc2
292+
```
293+
294+
Clean up resources
295+
296+
```shell
297+
gcloud container clusters delete mc1 --region us-central1
298+
gcloud container clusters delete mc2 --region us-east1
299+
for zone in us-central1-a us-east1-d; do gcloud compute instances delete mc-test --zone ${zone}; done
300+
```
301+
302+

0 commit comments

Comments
 (0)