Skip to content

Commit b727cfc

Browse files
kvapskvaps
authored
Merge pull request #4 from cozystack/fix/mesh-route-src
mesh: set preferred source for WireGuard routes
2 parents 062f897 + a9fedec commit b727cfc

File tree

3 files changed

+257
-1
lines changed

3 files changed

+257
-1
lines changed

‎manifests/kilo-azure-route-sync-deployment.yaml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: kilo-azure-route-sync
5+
namespace: cozy-cluster-autoscaler-azure
6+
---
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
kind: ClusterRole
9+
metadata:
10+
name: kilo-azure-route-sync
11+
rules:
12+
- apiGroups: [""]
13+
resources: ["nodes"]
14+
verbs: ["get", "list", "watch"]
15+
---
16+
apiVersion: rbac.authorization.k8s.io/v1
17+
kind: ClusterRoleBinding
18+
metadata:
19+
name: kilo-azure-route-sync
20+
roleRef:
21+
apiGroup: rbac.authorization.k8s.io
22+
kind: ClusterRole
23+
name: kilo-azure-route-sync
24+
subjects:
25+
- kind: ServiceAccount
26+
name: kilo-azure-route-sync
27+
namespace: cozy-cluster-autoscaler-azure
28+
---
29+
apiVersion: apps/v1
30+
kind: Deployment
31+
metadata:
32+
name: kilo-azure-route-sync
33+
namespace: cozy-cluster-autoscaler-azure
34+
spec:
35+
replicas: 1
36+
selector:
37+
matchLabels:
38+
app: kilo-azure-route-sync
39+
template:
40+
metadata:
41+
labels:
42+
app: kilo-azure-route-sync
43+
spec:
44+
serviceAccountName: kilo-azure-route-sync
45+
containers:
46+
- name: sync
47+
image: mcr.microsoft.com/azure-cli:2.67.0
48+
imagePullPolicy: IfNotPresent
49+
env:
50+
- name: AZURE_CLIENT_ID
51+
valueFrom:
52+
secretKeyRef:
53+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
54+
key: ClientID
55+
- name: AZURE_CLIENT_SECRET
56+
valueFrom:
57+
secretKeyRef:
58+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
59+
key: ClientSecret
60+
- name: AZURE_TENANT_ID
61+
valueFrom:
62+
secretKeyRef:
63+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
64+
key: TenantID
65+
- name: AZURE_SUBSCRIPTION_ID
66+
valueFrom:
67+
secretKeyRef:
68+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
69+
key: SubscriptionID
70+
- name: AZURE_RESOURCE_GROUP
71+
valueFrom:
72+
secretKeyRef:
73+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
74+
key: ResourceGroup
75+
- name: AZURE_ROUTE_TABLE
76+
value: kilo-routes-workers-serverscom
77+
- name: AZURE_VNET_NAME
78+
value: cozystack-vnet
79+
- name: AZURE_SUBNET_NAME
80+
value: workers-serverscom
81+
- name: AZURE_ROUTES
82+
value: to-serverscom=192.168.102.0/23
83+
command: ["/bin/sh","-ceu"]
84+
args:
85+
- |
86+
az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID" >/dev/null
87+
az account set --subscription "$AZURE_SUBSCRIPTION_ID"
88+
89+
az aks install-cli --install-location /usr/local/bin/kubectl >/dev/null
90+
91+
sync_route() {
92+
route_name="$1"
93+
route_prefix="$2"
94+
leader_ip="$3"
95+
az network route-table route create -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \
96+
-n "$route_name" --address-prefix "$route_prefix" \
97+
--next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null || true
98+
az network route-table route update -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \
99+
-n "$route_name" --address-prefix "$route_prefix" \
100+
--next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null
101+
}
102+
103+
sync_all_routes() {
104+
leader_ip="$1"
105+
IFS=','
106+
for entry in $AZURE_ROUTES; do
107+
route_name="${entry%%=*}"
108+
route_prefix="${entry#*=}"
109+
[ -n "$route_name" ] && [ -n "$route_prefix" ] || continue
110+
sync_route "$route_name" "$route_prefix" "$leader_ip"
111+
done
112+
unset IFS
113+
}
114+
115+
kubectl get node -w -l topology.kubernetes.io/zone=azure --no-headers \
116+
-o 'custom-columns=NAME:.metadata.name,LEADER:.metadata.annotations.kilo\.squat\.ai/leader,IP:.status.addresses[?(@.type=="InternalIP")].address' \
117+
| while read -r n leader ip; do
118+
echo "$(date -Iseconds) event node=${n} leader=${leader} ip=${ip}"
119+
[ "$leader" = "true" ] || continue
120+
az network vnet subnet update \
121+
-g "$AZURE_RESOURCE_GROUP" \
122+
--vnet-name "$AZURE_VNET_NAME" \
123+
-n "$AZURE_SUBNET_NAME" \
124+
--route-table "$AZURE_ROUTE_TABLE" >/dev/null
125+
126+
sync_all_routes "$ip"
127+
128+
echo "$(date -Iseconds) synced routes to leader ${n} (${ip})"
129+
done

‎pkg/mesh/routes.go‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
7676
Flags: int(netlink.FLAG_ONLINK),
7777
Gw: segment.privateIPs[i],
7878
LinkIndex: tunlIface,
79+
Src: t.privateIP.IP,
7980
Protocol: unix.RTPROT_STATIC,
8081
Table: kiloTableIndex,
8182
})
@@ -135,6 +136,14 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
135136
}
136137
return routes, rules
137138
}
139+
// Compute the preferred source address for routes through the WireGuard interface.
140+
// Without this, the kernel picks the WireGuard overlay IP (e.g. 100.66.0.x) as the
141+
// source, which can cause issues in environments like Azure SDN where the overlay
142+
// IP is unknown to the network fabric and reply packets cannot be routed back.
143+
var src net.IP
144+
if t.privateIP != nil {
145+
src = t.privateIP.IP
146+
}
138147
for _, segment := range t.segments {
139148
// Add routes for the current segment if local is true.
140149
if segment.location == t.location {
@@ -161,6 +170,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
161170
Flags: int(netlink.FLAG_ONLINK),
162171
Gw: segment.privateIPs[i],
163172
LinkIndex: tunlIface,
173+
Src: t.privateIP.IP,
164174
Protocol: unix.RTPROT_STATIC,
165175
Table: kiloTableIndex,
166176
})
@@ -190,6 +200,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
190200
Flags: int(netlink.FLAG_ONLINK),
191201
Gw: segment.wireGuardIP,
192202
LinkIndex: kiloIface,
203+
Src: src,
193204
Protocol: unix.RTPROT_STATIC,
194205
})
195206
// Don't add routes through Kilo if the private IP
@@ -207,6 +218,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
207218
Flags: int(netlink.FLAG_ONLINK),
208219
Gw: segment.wireGuardIP,
209220
LinkIndex: kiloIface,
221+
Src: src,
210222
Protocol: unix.RTPROT_STATIC,
211223
})
212224
}
@@ -218,6 +230,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
218230
Flags: int(netlink.FLAG_ONLINK),
219231
Gw: segment.wireGuardIP,
220232
LinkIndex: kiloIface,
233+
Src: src,
221234
Protocol: unix.RTPROT_STATIC,
222235
})
223236
}
@@ -228,6 +241,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
228241
routes = append(routes, &netlink.Route{
229242
Dst: &peer.AllowedIPs[i],
230243
LinkIndex: kiloIface,
244+
Src: src,
231245
Protocol: unix.RTPROT_STATIC,
232246
})
233247
}
@@ -304,8 +318,11 @@ func (t *Topology) PeerRoutes(name string, kiloIface int, additionalAllowedIPs [
304318
}
305319

306320
func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route {
307-
if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) {
321+
if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && subnet != nil && !subnet.Contains(route.Gw)) {
308322
route.LinkIndex = tunlIface
323+
if subnet != nil && route.Src == nil {
324+
route.Src = subnet.IP
325+
}
309326
}
310327
return route
311328
}

0 commit comments

Comments
 (0)