Skip to content

Commit f9155ac

Browse files
zbb88888zbb88888
committed
allow from snat eip to fip eip
Signed-off-by: zbb88888 <jmdxjsjgcxy@gmail.com>
1 parent 830362a commit f9155ac

File tree

2 files changed

+250
-9
lines changed

2 files changed

+250
-9
lines changed

‎dist/images/vpcnatgateway/nat-gateway.sh‎

Lines changed: 123 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ function show_help() {
7575
echo " dnat-del - Delete DNAT rule"
7676
echo " snat-add - Add SNAT rule"
7777
echo " snat-del - Delete SNAT rule"
78+
echo " hairpin-snat-add - Add hairpin SNAT rule for internal FIP access"
79+
echo " hairpin-snat-del - Delete hairpin SNAT rule"
7880
echo " qos-add - Add QoS rule"
7981
echo " qos-del - Delete QoS rule"
8082
echo " eip-ingress-qos-add - Add EIP ingress QoS"
@@ -162,6 +164,7 @@ function init() {
162164
$iptables_cmd -t nat -N EXCLUSIVE_SNAT # floatingIp SNAT
163165
$iptables_cmd -t nat -N SHARED_DNAT
164166
$iptables_cmd -t nat -N SHARED_SNAT
167+
$iptables_cmd -t nat -N HAIRPIN_SNAT
165168

166169
$iptables_cmd -t nat -A PREROUTING -j DNAT_FILTER
167170
$iptables_cmd -t nat -A DNAT_FILTER -j EXCLUSIVE_DNAT
@@ -170,6 +173,7 @@ function init() {
170173
$iptables_cmd -t nat -A POSTROUTING -j SNAT_FILTER
171174
$iptables_cmd -t nat -A SNAT_FILTER -j EXCLUSIVE_SNAT
172175
$iptables_cmd -t nat -A SNAT_FILTER -j SHARED_SNAT
176+
$iptables_cmd -t nat -A SNAT_FILTER -j HAIRPIN_SNAT
173177

174178
# Load IFB kernel module for ingress QoS traffic shaping
175179
# IFB (Intermediate Functional Block) is required for ingress rate limiting using HTB
@@ -280,6 +284,24 @@ function del_eip() {
280284
done
281285
}
282286

287+
# Check if the given CIDR exists in VPC_INTERFACE's routes (indicates it's an internal CIDR)
288+
# This is used to determine if hairpin SNAT is needed for a given SNAT rule
289+
# Args: $1 - CIDR to check (e.g., "10.0.1.0/24")
290+
# Returns: 0 if the CIDR is found in VPC_INTERFACE routes, 1 otherwise
291+
function is_internal_cidr() {
292+
local cidr="$1"
293+
if [ -z "$cidr" ]; then
294+
return 1
295+
fi
296+
# Escape '.' in CIDR for grep regex to avoid matching unintended characters
297+
# e.g., "10.0.1.0/24" -> "^10\.0\.1\.0/24 " matches exactly, not "10X0Y1Z0/24"
298+
local cidr_pattern="^${cidr//./\\.} "
299+
if ip -4 route show dev "$VPC_INTERFACE" | grep -q "$cidr_pattern"; then
300+
return 0
301+
fi
302+
return 1
303+
}
304+
283305
function add_floating_ip() {
284306
# make sure inited
285307
check_inited
@@ -316,33 +338,118 @@ function del_floating_ip() {
316338
function add_snat() {
317339
# make sure inited
318340
check_inited
319-
# iptables -t nat -F SHARED_SNAT
341+
local all_shared_snat_rules
342+
all_shared_snat_rules=$($iptables_save_cmd | grep SHARED_SNAT)
320343
for rule in $@
321344
do
322345
arr=(${rule//,/ })
323346
eip=(${arr[0]//\// })
324347
internalCIDR=${arr[1]}
325348
randomFullyOption=${arr[2]}
326-
# check if already exist
327-
$iptables_save_cmd | grep SHARED_SNAT | grep "\-s $internalCIDR" | grep "source $eip" && exit 0
328-
exec_cmd "$iptables_cmd -t nat -A SHARED_SNAT -o $EXTERNAL_INTERFACE -s $internalCIDR -j SNAT --to-source $eip $randomFullyOption"
349+
# check if already exist, skip adding if exists (idempotent)
350+
ruleMatch=$(echo "$all_shared_snat_rules" | grep -w -- "-s $internalCIDR" | grep -E -- "--to-source $eip(\$| )")
351+
if [ -z "$ruleMatch" ]; then
352+
exec_cmd "$iptables_cmd -t nat -A SHARED_SNAT -o $EXTERNAL_INTERFACE -s $internalCIDR -j SNAT --to-source $eip $randomFullyOption"
353+
fi
354+
# Add hairpin SNAT when internalCIDR is routed via VPC_INTERFACE
355+
# This enables internal VMs to access other internal VMs via FIP
356+
if is_internal_cidr "$internalCIDR"; then
357+
echo "SNAT cidr $internalCIDR is internal, adding hairpin SNAT with EIP $eip"
358+
add_hairpin_snat "$eip,$internalCIDR"
359+
fi
329360
done
330361
}
331362
function del_snat() {
332363
# make sure inited
333364
check_inited
334-
# iptables -t nat -F SHARED_SNAT
365+
local all_shared_snat_rules
366+
all_shared_snat_rules=$($iptables_save_cmd | grep SHARED_SNAT)
335367
for rule in $@
336368
do
337369
arr=(${rule//,/ })
338370
eip=(${arr[0]//\// })
339371
internalCIDR=${arr[1]}
340372
# check if already exist
341-
ruleMatch=$($iptables_save_cmd | grep SHARED_SNAT | grep "\-s $internalCIDR" | grep "source $eip")
342-
if [ "$?" -eq 0 ];then
343-
ruleMatch=$(echo $ruleMatch | sed 's/-A //')
373+
ruleMatch=$(echo "$all_shared_snat_rules" | grep -w -- "-s $internalCIDR" | grep -E -- "--to-source $eip(\$| )")
374+
if [ -n "$ruleMatch" ]; then
375+
ruleMatch=$(echo "$ruleMatch" | sed 's/-A //')
344376
exec_cmd "$iptables_cmd -t nat -D $ruleMatch"
345377
fi
378+
# also remove hairpin SNAT if it exists
379+
if is_internal_cidr "$internalCIDR"; then
380+
echo "SNAT cidr $internalCIDR is internal, deleting hairpin SNAT with EIP $eip"
381+
del_hairpin_snat "$eip,$internalCIDR"
382+
fi
383+
done
384+
}
385+
386+
# Hairpin SNAT: Enables internal VM to access another internal VM's FIP
387+
# Packet flow when VM A accesses VM B's EIP:
388+
# 1. VM A (10.0.1.6) -> EIP (10.1.69.216) arrives at NAT GW
389+
# 2. DNAT translates destination to VM B's internal IP (10.0.1.11)
390+
# 3. Without hairpin SNAT, reply from VM B goes directly to VM A (same subnet),
391+
# but VM A expects reply from EIP, causing asymmetric routing failure
392+
# 4. Hairpin SNAT translates source to EIP, ensuring symmetric return path via NAT GW
393+
#
394+
# RECOMMENDED: NAT-GW binds to a single VPC internal subnet. In this case,
395+
# only one hairpin SNAT rule is needed (matching the VPC's directly connected route).
396+
#
397+
# Multi-subnet scenarios are supported but NOT recommended. For multiple subnets,
398+
# create separate NAT gateways for each subnet to achieve more direct forwarding paths.
399+
# Each CIDR can only have one hairpin rule to avoid conflicting SNAT sources.
400+
#
401+
# Rule format: eip,internalCIDR
402+
# Example: 10.1.69.219,10.0.1.0/24
403+
# Creates: iptables -t nat -A HAIRPIN_SNAT -s 10.0.1.0/24 -d 10.0.1.0/24 -j SNAT --to-source 10.1.69.219
404+
function add_hairpin_snat() {
405+
# make sure inited
406+
check_inited
407+
local all_hairpin_rules
408+
all_hairpin_rules=$($iptables_save_cmd -t nat | grep HAIRPIN_SNAT)
409+
for rule in $@
410+
do
411+
arr=(${rule//,/ })
412+
eip=(${arr[0]//\// })
413+
internalCIDR=${arr[1]}
414+
415+
# Filter from cached rules for this specific CIDR
416+
local existing_rules
417+
existing_rules=$(echo "$all_hairpin_rules" | grep -w -- "-s $internalCIDR" | grep -w -- "-d $internalCIDR")
418+
419+
# Check if this exact rule already exists (idempotent)
420+
if echo "$existing_rules" | grep -qE -- "--to-source $eip(\$| )"; then
421+
echo "Hairpin SNAT rule for $internalCIDR with EIP $eip already exists, skipping"
422+
continue
423+
fi
424+
425+
# Check if this CIDR already has a hairpin rule with a different EIP
426+
if [ -n "$existing_rules" ]; then
427+
echo "WARNING: Hairpin SNAT rule for $internalCIDR already exists with different EIP. Skipping."
428+
continue
429+
fi
430+
431+
exec_cmd "$iptables_cmd -t nat -A HAIRPIN_SNAT -s $internalCIDR -d $internalCIDR -j SNAT --to-source $eip"
432+
echo "Hairpin SNAT rule added: $internalCIDR -> $eip"
433+
done
434+
}
435+
436+
# Delete a hairpin SNAT rule.
437+
# Args: eip,internalCIDR (comma-separated)
438+
function del_hairpin_snat() {
439+
# make sure inited
440+
check_inited
441+
local all_hairpin_rules
442+
all_hairpin_rules=$($iptables_save_cmd -t nat | grep HAIRPIN_SNAT)
443+
for rule in $@
444+
do
445+
arr=(${rule//,/ })
446+
eip=(${arr[0]//\// })
447+
internalCIDR=${arr[1]}
448+
# check if rule exists (idempotent - skip if not found)
449+
if echo "$all_hairpin_rules" | grep -w -- "-s $internalCIDR" | grep -w -- "-d $internalCIDR" | grep -qE -- "--to-source $eip(\$| )"; then
450+
exec_cmd "$iptables_cmd -t nat -D HAIRPIN_SNAT -s $internalCIDR -d $internalCIDR -j SNAT --to-source $eip"
451+
echo "Hairpin SNAT rule deleted: $internalCIDR -> $eip"
452+
fi
346453
done
347454
}
348455

@@ -1435,6 +1542,14 @@ case $opt in
14351542
echo "snat-del $rules"
14361543
del_snat $rules
14371544
;;
1545+
hairpin-snat-add)
1546+
echo "hairpin-snat-add $rules"
1547+
add_hairpin_snat $rules
1548+
;;
1549+
hairpin-snat-del)
1550+
echo "hairpin-snat-del $rules"
1551+
del_hairpin_snat $rules
1552+
;;
14381553
floating-ip-add)
14391554
echo "floating-ip-add $rules"
14401555
add_floating_ip $rules

‎test/e2e/iptables-vpc-nat-gw/e2e_test.go‎

Lines changed: 127 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,33 @@ func verifySubnetStatusAfterEIPOperation(subnetClient *framework.SubnetClient, s
249249
}
250250
}
251251

252+
// hairpinSnatChainExists checks if the HAIRPIN_SNAT chain exists in the NAT gateway pod.
253+
// Returns false on older versions that don't support this feature.
254+
func hairpinSnatChainExists(natGwPodName string) bool {
255+
cmd := []string{"iptables-save", "-t", "nat"}
256+
stdout, _, err := framework.KubectlExec(framework.KubeOvnNamespace, natGwPodName, cmd...)
257+
framework.ExpectNoError(err, "failed to exec iptables-save in NAT gateway pod %s", natGwPodName)
258+
259+
output := string(stdout)
260+
return strings.Contains(output, ":HAIRPIN_SNAT") || strings.Contains(output, "-N HAIRPIN_SNAT")
261+
}
262+
263+
// hairpinSnatRuleExists checks if hairpin SNAT rule exists in the NAT gateway pod.
264+
// Returns true if rule exists, false otherwise (including when HAIRPIN_SNAT chain doesn't exist).
265+
func hairpinSnatRuleExists(natGwPodName, cidr, eip string) bool {
266+
cmd := []string{"iptables-save", "-t", "nat"}
267+
stdout, _, err := framework.KubectlExec(framework.KubeOvnNamespace, natGwPodName, cmd...)
268+
framework.ExpectNoError(err, "failed to exec iptables-save in NAT gateway pod %s", natGwPodName)
269+
270+
output := string(stdout)
271+
if !strings.Contains(output, ":HAIRPIN_SNAT") && !strings.Contains(output, "-N HAIRPIN_SNAT") {
272+
return false
273+
}
274+
275+
hairpinRulePattern := fmt.Sprintf("-A HAIRPIN_SNAT -s %s -d %s -j SNAT --to-source %s", cidr, cidr, eip)
276+
return strings.Contains(output, hairpinRulePattern)
277+
}
278+
252279
var _ = framework.OrderedDescribe("[group:iptables-vpc-nat-gw]", func() {
253280
f := framework.NewDefaultFramework("iptables-vpc-nat-gw")
254281

@@ -264,6 +291,7 @@ var _ = framework.OrderedDescribe("[group:iptables-vpc-nat-gw]", func() {
264291
var iptablesFIPClient *framework.IptablesFIPClient
265292
var iptablesSnatRuleClient *framework.IptablesSnatClient
266293
var iptablesDnatRuleClient *framework.IptablesDnatClient
294+
var podClient *framework.PodClient
267295

268296
var dockerExtNet1Network *dockernetwork.Inspect
269297
var net1NicName string
@@ -421,6 +449,7 @@ var _ = framework.OrderedDescribe("[group:iptables-vpc-nat-gw]", func() {
421449
vpcName = "vpc-" + randomSuffix
422450
vpcNatGwName = "gw-" + randomSuffix
423451
overlaySubnetName = "overlay-subnet-" + randomSuffix
452+
podClient = f.PodClient()
424453
})
425454

426455
framework.ConformanceIt("[1] change gateway image and custom annotations", func() {
@@ -543,6 +572,81 @@ var _ = framework.OrderedDescribe("[group:iptables-vpc-nat-gw]", func() {
543572
iptablesSnatRuleClient.DeleteSync(snatName)
544573
})
545574

575+
// Verify hairpin SNAT rule is automatically created for internal CIDR
576+
ginkgo.By("Verifying hairpin SNAT rule exists in NAT gateway pod")
577+
vpcNatGwPodName := util.GenNatGwPodName(vpcNatGwName)
578+
snatEip = iptablesEIPClient.Get(snatEipName)
579+
if !hairpinSnatChainExists(vpcNatGwPodName) {
580+
framework.Logf("HAIRPIN_SNAT chain not found, skipping hairpin SNAT verification (feature requires v1.15+)")
581+
} else {
582+
gomega.Eventually(func() bool {
583+
return hairpinSnatRuleExists(vpcNatGwPodName, overlaySubnetV4Cidr, snatEip.Status.IP)
584+
}, 30*time.Second, 2*time.Second).Should(gomega.BeTrue(),
585+
"Hairpin SNAT rule should be created after SNAT creation")
586+
587+
// Verify real data-path: internal pod accessing another internal pod via FIP EIP
588+
// Packet flow: client -> NAT GW (DNAT to serverIP + hairpin SNAT to EIP) -> server -> NAT GW (un-SNAT/DNAT) -> client
589+
ginkgo.By("Verifying hairpin SNAT connectivity: internal pod (SNAT EIP) accessing another internal pod (FIP EIP)")
590+
serverPodName := "server-" + randomSuffix
591+
clientPodName := "client-" + randomSuffix
592+
hairpinFipEipName := "hairpin-fip-eip-" + randomSuffix
593+
hairpinFipName := "hairpin-fip-" + randomSuffix
594+
595+
// Create server pod in overlay subnet with auto-assigned IP
596+
serverAnnotations := map[string]string{
597+
util.LogicalSwitchAnnotation: overlaySubnetName,
598+
}
599+
serverPort := "8080"
600+
serverArgs := []string{"netexec", "--http-port", serverPort}
601+
serverPod := framework.MakePod(f.Namespace.Name, serverPodName, nil, serverAnnotations, framework.AgnhostImage, nil, serverArgs)
602+
_ = podClient.CreateSync(serverPod)
603+
ginkgo.DeferCleanup(func() {
604+
ginkgo.By("Cleaning up server pod " + serverPodName)
605+
podClient.DeleteSync(serverPodName)
606+
})
607+
608+
// Get server pod's auto-assigned IP for FIP binding
609+
createdServerPod := podClient.GetPod(serverPodName)
610+
serverPodIP := createdServerPod.Annotations[util.IPAddressAnnotation]
611+
framework.ExpectNotEmpty(serverPodIP, "server pod should have an IP assigned")
612+
framework.Logf("Server pod %s has IP %s", serverPodName, serverPodIP)
613+
614+
// Create a dedicated EIP and FIP to map FIP EIP -> server pod IP
615+
hairpinFipEip := framework.MakeIptablesEIP(hairpinFipEipName, "", "", "", vpcNatGwName, "", "")
616+
_ = iptablesEIPClient.CreateSync(hairpinFipEip)
617+
ginkgo.DeferCleanup(func() {
618+
ginkgo.By("Cleaning up hairpin FIP EIP " + hairpinFipEipName)
619+
iptablesEIPClient.DeleteSync(hairpinFipEipName)
620+
})
621+
hairpinFipEip = iptablesEIPClient.Get(hairpinFipEipName)
622+
framework.ExpectNotEmpty(hairpinFipEip.Status.IP, "hairpin FIP EIP should have an IP assigned")
623+
624+
hairpinFip := framework.MakeIptablesFIPRule(hairpinFipName, hairpinFipEipName, serverPodIP)
625+
_ = iptablesFIPClient.CreateSync(hairpinFip)
626+
ginkgo.DeferCleanup(func() {
627+
ginkgo.By("Cleaning up hairpin FIP " + hairpinFipName)
628+
iptablesFIPClient.DeleteSync(hairpinFipName)
629+
})
630+
631+
// Create client pod in same subnet (uses SNAT EIP for outbound traffic)
632+
clientAnnotations := map[string]string{
633+
util.LogicalSwitchAnnotation: overlaySubnetName,
634+
}
635+
clientPod := framework.MakePod(f.Namespace.Name, clientPodName, nil, clientAnnotations, framework.AgnhostImage, nil, []string{"pause"})
636+
_ = podClient.CreateSync(clientPod)
637+
ginkgo.DeferCleanup(func() {
638+
ginkgo.By("Cleaning up client pod " + clientPodName)
639+
podClient.DeleteSync(clientPodName)
640+
})
641+
642+
// Test connectivity: client pod (SNAT EIP) -> server pod (FIP EIP)
643+
ginkgo.By("Checking connectivity from client pod (SNAT EIP) to server pod (FIP EIP) " + hairpinFipEip.Status.IP)
644+
cmd := []string{"curl", "-m", "10", fmt.Sprintf("http://%s:%s/clientip", hairpinFipEip.Status.IP, serverPort)}
645+
output, _, err := framework.KubectlExec(f.Namespace.Name, clientPodName, cmd...)
646+
framework.ExpectNoError(err, "Client pod (SNAT EIP) should reach server pod via FIP EIP through hairpin SNAT")
647+
framework.Logf("Hairpin SNAT connectivity verified, output: %s", string(output))
648+
}
649+
546650
ginkgo.By("Creating iptables vip for dnat")
547651
dnatVip := framework.MakeVip(f.Namespace.Name, dnatVipName, overlaySubnetName, "", "", "")
548652
_ = vipClient.CreateSync(dnatVip)
@@ -618,8 +722,18 @@ var _ = framework.OrderedDescribe("[group:iptables-vpc-nat-gw]", func() {
618722
iptablesSnatRuleClient.DeleteSync(sharedEipSnatName)
619723
})
620724

621-
ginkgo.By("Get share eip")
725+
// Verify hairpin SNAT rule is created for shared SNAT as well
726+
ginkgo.By("Getting share eip")
622727
shareEip = iptablesEIPClient.Get(sharedEipName)
728+
framework.ExpectNotEmpty(shareEip.Status.IP, "shareEip.Status.IP should not be empty")
729+
if hairpinSnatChainExists(vpcNatGwPodName) {
730+
ginkgo.By("Verifying hairpin SNAT rule exists for shared snat")
731+
gomega.Eventually(func() bool {
732+
return hairpinSnatRuleExists(vpcNatGwPodName, overlaySubnetV4Cidr, shareEip.Status.IP)
733+
}, 30*time.Second, 2*time.Second).Should(gomega.BeTrue(),
734+
"Hairpin SNAT rule should be created after shared SNAT creation")
735+
}
736+
623737
ginkgo.By("Get share dnat")
624738
shareDnat = iptablesDnatRuleClient.Get(sharedEipDnatName)
625739
ginkgo.By("Get share snat")
@@ -643,6 +757,18 @@ var _ = framework.OrderedDescribe("[group:iptables-vpc-nat-gw]", func() {
643757
// make sure eip is shared
644758
nats := []string{util.DnatUsingEip, util.FipUsingEip, util.SnatUsingEip}
645759
framework.ExpectEqual(shareEip.Status.Nat, strings.Join(nats, ","))
760+
761+
// Verify hairpin SNAT rule cleanup when SNAT is deleted
762+
if hairpinSnatChainExists(vpcNatGwPodName) {
763+
ginkgo.By("Deleting snat to verify hairpin SNAT rule cleanup")
764+
iptablesSnatRuleClient.DeleteSync(snatName)
765+
ginkgo.By("Verifying hairpin SNAT rule is deleted after snat deletion")
766+
gomega.Eventually(func() bool {
767+
return hairpinSnatRuleExists(vpcNatGwPodName, overlaySubnetV4Cidr, snatEip.Status.IP)
768+
}, 30*time.Second, 2*time.Second).Should(gomega.BeFalse(),
769+
"Hairpin SNAT rule should be deleted after SNAT deletion")
770+
}
771+
646772
// All cleanup is handled by DeferCleanup above, no need for manual cleanup
647773
})
648774

0 commit comments

Comments
 (0)