From cf9f1e6e3efc731ad4cc4879332c35f75b95b515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kuli=C5=84ski?= Date: Tue, 21 Oct 2025 13:41:42 +0000 Subject: [PATCH 1/6] Add --create-nat flag --- kubetest2-gke/deployer/down.go | 15 +++- kubetest2-gke/deployer/nat.go | 98 +++++++++++++++++++++++ kubetest2-gke/deployer/options/network.go | 1 + kubetest2-gke/deployer/up.go | 3 + 4 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 kubetest2-gke/deployer/nat.go diff --git a/kubetest2-gke/deployer/down.go b/kubetest2-gke/deployer/down.go index bd08ea9..7461507 100644 --- a/kubetest2-gke/deployer/down.go +++ b/kubetest2-gke/deployer/down.go @@ -55,11 +55,18 @@ func (d *Deployer) Down() error { klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules) } - if err := d.TeardownNetwork(); err != nil { - return err + errNat := d.CleanupNat() + errNetwork := d.TeardownNetwork() + errSubnets := d.DeleteSubnets(d.retryCount) + + if errNat != nil { + return errNat } - if err := d.DeleteSubnets(d.retryCount); err != nil { - return err + if errNetwork != nil { + return errNetwork + } + if errSubnets != nil { + return errSubnets } return d.DeleteNetwork() } diff --git a/kubetest2-gke/deployer/nat.go b/kubetest2-gke/deployer/nat.go new file mode 100644 index 0000000..4c80aa0 --- /dev/null +++ b/kubetest2-gke/deployer/nat.go @@ -0,0 +1,98 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployer + +import ( + "fmt" + + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/exec" +) + +func (d *Deployer) EnsureNat() error { + if !d.CreateNat { + return nil + } + if d.Network == "default" { + return fmt.Errorf("NAT router should be set manually for the default network") + } + region := regionFromLocation(d.Regions, d.Zones, d.retryCount) + nat := d.getNatName() + hostProject := d.Projects[0] + + if runWithNoOutput(exec.Command("gcloud", "compute", "routers", "describe", nat, + "--project="+hostProject, + "--region="+region, + "--format=value(name)")) != nil { + klog.V(1).Infof("Couldn't describe router '%s', assuming it doesn't exist and creating it", nat) + if err := runWithOutput(exec.Command("gcloud", "compute", "routers", "create", nat, + "--project="+hostProject, + "--network="+d.Network, + "--region="+region)); err != nil { + return fmt.Errorf("error creating NAT router: %w", err) + } + } + // Create this unique NAT configuration only if it does not exist yet. + if runWithNoOutput(exec.Command("gcloud", "compute", "routers", "nats", "describe", nat, + "--project="+hostProject, + "--router="+nat, + "--region="+region, + "--format=value(name)")) != nil { + klog.V(1).Infof("Couldn't describe NAT '%s', assuming it doesn't exist and creating it", nat) + if err := runWithOutput(exec.Command("gcloud", "compute", "routers", "nats", "create", nat, + "--project="+hostProject, + "--router="+nat, + "--region="+region, + "--auto-allocate-nat-external-ips", + "--nat-primary-subnet-ip-ranges")); err != nil { + return fmt.Errorf("error adding NAT to a router: %w", err) + } + } + + return nil +} + +func (d *Deployer) getNatName() string { + return "nat-router-" + d.Clusters[0] +} + +func (d *Deployer) CleanupNat() error { + if !d.CreateNat { + return nil + } + region := regionFromLocation(d.Regions, d.Zones, d.retryCount) + nat := d.getNatName() + hostProject := d.Projects[0] + + // Delete NAT router. That will remove NAT configuration as well. + if runWithNoOutput(exec.Command("gcloud", "compute", "routers", "describe", nat, + "--project="+hostProject, + "--region="+region, + "--format=value(name)")) == nil { + klog.V(1).Infof("Found NAT router '%s', deleting", nat) + err := runWithOutput(exec.Command("gcloud", "compute", "routers", "delete", "-q", nat, + "--project="+hostProject, + "--region="+region)) + if err != nil { + return fmt.Errorf("error deleting NAT router: %w", err) + } + } else { + klog.V(1).Infof("Found no NAT router '%s', assuming resources are clean", nat) + } + + return nil +} diff --git a/kubetest2-gke/deployer/options/network.go b/kubetest2-gke/deployer/options/network.go index e6ca5d8..aa1bfa8 100644 --- a/kubetest2-gke/deployer/options/network.go +++ b/kubetest2-gke/deployer/options/network.go @@ -25,4 +25,5 @@ type NetworkOptions struct { EnableULAInternalIPv6 bool `flag:"~enable-ula-internal-ipv6" desc:"Enable Unique Local IPv6 Addresses (ULA). Adds the --enable-ula-internal-ipv6 flag to the gcloud compute networks create command"` RemoveNetwork bool `flag:"~remove-network" desc:"At the end of the test remove non-default network that was used by cluster. The 'default' network is never deleted. Defaults to true if not provided."` + CreateNat bool `flag:"~create-nat" desc:"Configure Cloud NAT allowing outbound connections in cluster with private nodes."` } diff --git a/kubetest2-gke/deployer/up.go b/kubetest2-gke/deployer/up.go index aa68d75..c6a9bf8 100644 --- a/kubetest2-gke/deployer/up.go +++ b/kubetest2-gke/deployer/up.go @@ -334,6 +334,9 @@ func (d *Deployer) TestSetup() error { if err := d.EnsureFirewallRules(); err != nil { return err } + if err := d.EnsureNat(); err != nil { + return err + } d.testPrepared = true return nil } From e90378245674ed73287196ee7426ebc5988ec944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kuli=C5=84ski?= Date: Tue, 21 Oct 2025 13:48:01 +0000 Subject: [PATCH 2/6] Update copyright year --- kubetest2-gke/deployer/nat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubetest2-gke/deployer/nat.go b/kubetest2-gke/deployer/nat.go index 4c80aa0..a665f45 100644 --- a/kubetest2-gke/deployer/nat.go +++ b/kubetest2-gke/deployer/nat.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Kubernetes Authors. +Copyright 2025 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From cbc3658f5be0fc62b7947ca07c923b4987d17c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kuli=C5=84ski?= Date: Wed, 22 Oct 2025 07:17:24 +0000 Subject: [PATCH 3/6] aggregate errors --- kubetest2-gke/deployer/down.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/kubetest2-gke/deployer/down.go b/kubetest2-gke/deployer/down.go index 7461507..484b662 100644 --- a/kubetest2-gke/deployer/down.go +++ b/kubetest2-gke/deployer/down.go @@ -17,6 +17,7 @@ limitations under the License. package deployer import ( + "errors" "fmt" "sync" @@ -59,14 +60,9 @@ func (d *Deployer) Down() error { errNetwork := d.TeardownNetwork() errSubnets := d.DeleteSubnets(d.retryCount) - if errNat != nil { - return errNat - } - if errNetwork != nil { - return errNetwork - } - if errSubnets != nil { - return errSubnets + errs := errors.Join(errNat, errNetwork, errSubnets) + if errs != nil { + return errs } return d.DeleteNetwork() } From e4f9e891d5ef3cf47a31c505072b50f7091d6266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kuli=C5=84ski?= Date: Wed, 29 Oct 2025 10:24:28 +0000 Subject: [PATCH 4/6] Log errors in Down() instead of returning --- kubetest2-gke/deployer/down.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/kubetest2-gke/deployer/down.go b/kubetest2-gke/deployer/down.go index 484b662..c0be3f8 100644 --- a/kubetest2-gke/deployer/down.go +++ b/kubetest2-gke/deployer/down.go @@ -56,14 +56,18 @@ func (d *Deployer) Down() error { klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules) } - errNat := d.CleanupNat() - errNetwork := d.TeardownNetwork() - errSubnets := d.DeleteSubnets(d.retryCount) + if err := d.CleanupNat(); err != nil { + klog.Errorf("Error cleaning-up nat: %v", err) + } + + if err := d.TeardownNetwork(); err != nil { + klog.Errorf("Error tearing-down network: %v", err) + } - errs := errors.Join(errNat, errNetwork, errSubnets) - if errs != nil { - return errs + if err := d.DeleteSubnets(d.retryCount); err != nil { + klog.Errorf("Error deleting subnets: %v", err) } + return d.DeleteNetwork() } From 4e79900b3e7b99bdee77a5c7282187a50335ef8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kuli=C5=84ski?= Date: Wed, 29 Oct 2025 10:29:15 +0000 Subject: [PATCH 5/6] Remove unused import --- kubetest2-gke/deployer/down.go | 1 - 1 file changed, 1 deletion(-) diff --git a/kubetest2-gke/deployer/down.go b/kubetest2-gke/deployer/down.go index c0be3f8..9e09be6 100644 --- a/kubetest2-gke/deployer/down.go +++ b/kubetest2-gke/deployer/down.go @@ -17,7 +17,6 @@ limitations under the License. package deployer import ( - "errors" "fmt" "sync" From 2ee926072d2dda7a12a18310e8ceb3ff851c5391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kuli=C5=84ski?= Date: Wed, 29 Oct 2025 14:32:10 +0000 Subject: [PATCH 6/6] Log and aggregate errors --- kubetest2-gke/deployer/down.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/kubetest2-gke/deployer/down.go b/kubetest2-gke/deployer/down.go index 9e09be6..df7112e 100644 --- a/kubetest2-gke/deployer/down.go +++ b/kubetest2-gke/deployer/down.go @@ -17,6 +17,7 @@ limitations under the License. package deployer import ( + "errors" "fmt" "sync" @@ -55,19 +56,27 @@ func (d *Deployer) Down() error { klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules) } - if err := d.CleanupNat(); err != nil { - klog.Errorf("Error cleaning-up nat: %v", err) + errCleanNat := d.CleanupNat() + if errCleanNat != nil { + klog.Errorf("Error cleaning up NAT: %v", errCleanNat) } - if err := d.TeardownNetwork(); err != nil { - klog.Errorf("Error tearing-down network: %v", err) + errTeardownNetwork := d.TeardownNetwork() + if errTeardownNetwork != nil { + klog.Errorf("Error tearing down network: %v", errTeardownNetwork) } - if err := d.DeleteSubnets(d.retryCount); err != nil { - klog.Errorf("Error deleting subnets: %v", err) + errDeleteSubnets := d.DeleteSubnets(d.retryCount) + if errDeleteSubnets != nil { + klog.Errorf("Error deleting subnets: %v", errDeleteSubnets) } - return d.DeleteNetwork() + errDeleteNetwork := d.DeleteNetwork() + if errDeleteNetwork != nil { + klog.Errorf("Error deleting network: %v", errDeleteNetwork) + } + + return errors.Join(errCleanFirewalls, errCleanNat, errTeardownNetwork, errDeleteSubnets, errDeleteNetwork) } func (d *Deployer) DeleteClusters(retryCount int) {