diff --git a/kubetest2-gke/deployer/down.go b/kubetest2-gke/deployer/down.go index bd08ea9..df7112e 100644 --- a/kubetest2-gke/deployer/down.go +++ b/kubetest2-gke/deployer/down.go @@ -17,6 +17,7 @@ limitations under the License. package deployer import ( + "errors" "fmt" "sync" @@ -55,13 +56,27 @@ func (d *Deployer) Down() error { klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules) } - if err := d.TeardownNetwork(); err != nil { - return err + errCleanNat := d.CleanupNat() + if errCleanNat != nil { + klog.Errorf("Error cleaning up NAT: %v", errCleanNat) } - if err := d.DeleteSubnets(d.retryCount); err != nil { - return err + + errTeardownNetwork := d.TeardownNetwork() + if errTeardownNetwork != nil { + klog.Errorf("Error tearing down network: %v", errTeardownNetwork) + } + + errDeleteSubnets := d.DeleteSubnets(d.retryCount) + if errDeleteSubnets != nil { + klog.Errorf("Error deleting subnets: %v", errDeleteSubnets) + } + + errDeleteNetwork := d.DeleteNetwork() + if errDeleteNetwork != nil { + klog.Errorf("Error deleting network: %v", errDeleteNetwork) } - return d.DeleteNetwork() + + return errors.Join(errCleanFirewalls, errCleanNat, errTeardownNetwork, errDeleteSubnets, errDeleteNetwork) } func (d *Deployer) DeleteClusters(retryCount int) { diff --git a/kubetest2-gke/deployer/nat.go b/kubetest2-gke/deployer/nat.go new file mode 100644 index 0000000..a665f45 --- /dev/null +++ b/kubetest2-gke/deployer/nat.go @@ -0,0 +1,98 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployer + +import ( + "fmt" + + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/exec" +) + +func (d *Deployer) EnsureNat() error { + if !d.CreateNat { + return nil + } + if d.Network == "default" { + return fmt.Errorf("NAT router should be set manually for the default network") + } + region := regionFromLocation(d.Regions, d.Zones, d.retryCount) + nat := d.getNatName() + hostProject := d.Projects[0] + + if runWithNoOutput(exec.Command("gcloud", "compute", "routers", "describe", nat, + "--project="+hostProject, + "--region="+region, + "--format=value(name)")) != nil { + klog.V(1).Infof("Couldn't describe router '%s', assuming it doesn't exist and creating it", nat) + if err := runWithOutput(exec.Command("gcloud", "compute", "routers", "create", nat, + "--project="+hostProject, + "--network="+d.Network, + "--region="+region)); err != nil { + return fmt.Errorf("error creating NAT router: %w", err) + } + } + // Create this unique NAT configuration only if it does not exist yet. + if runWithNoOutput(exec.Command("gcloud", "compute", "routers", "nats", "describe", nat, + "--project="+hostProject, + "--router="+nat, + "--region="+region, + "--format=value(name)")) != nil { + klog.V(1).Infof("Couldn't describe NAT '%s', assuming it doesn't exist and creating it", nat) + if err := runWithOutput(exec.Command("gcloud", "compute", "routers", "nats", "create", nat, + "--project="+hostProject, + "--router="+nat, + "--region="+region, + "--auto-allocate-nat-external-ips", + "--nat-primary-subnet-ip-ranges")); err != nil { + return fmt.Errorf("error adding NAT to a router: %w", err) + } + } + + return nil +} + +func (d *Deployer) getNatName() string { + return "nat-router-" + d.Clusters[0] +} + +func (d *Deployer) CleanupNat() error { + if !d.CreateNat { + return nil + } + region := regionFromLocation(d.Regions, d.Zones, d.retryCount) + nat := d.getNatName() + hostProject := d.Projects[0] + + // Delete NAT router. That will remove NAT configuration as well. + if runWithNoOutput(exec.Command("gcloud", "compute", "routers", "describe", nat, + "--project="+hostProject, + "--region="+region, + "--format=value(name)")) == nil { + klog.V(1).Infof("Found NAT router '%s', deleting", nat) + err := runWithOutput(exec.Command("gcloud", "compute", "routers", "delete", "-q", nat, + "--project="+hostProject, + "--region="+region)) + if err != nil { + return fmt.Errorf("error deleting NAT router: %w", err) + } + } else { + klog.V(1).Infof("Found no NAT router '%s', assuming resources are clean", nat) + } + + return nil +} diff --git a/kubetest2-gke/deployer/options/network.go b/kubetest2-gke/deployer/options/network.go index e6ca5d8..aa1bfa8 100644 --- a/kubetest2-gke/deployer/options/network.go +++ b/kubetest2-gke/deployer/options/network.go @@ -25,4 +25,5 @@ type NetworkOptions struct { EnableULAInternalIPv6 bool `flag:"~enable-ula-internal-ipv6" desc:"Enable Unique Local IPv6 Addresses (ULA). Adds the --enable-ula-internal-ipv6 flag to the gcloud compute networks create command"` RemoveNetwork bool `flag:"~remove-network" desc:"At the end of the test remove non-default network that was used by cluster. The 'default' network is never deleted. Defaults to true if not provided."` + CreateNat bool `flag:"~create-nat" desc:"Configure Cloud NAT allowing outbound connections in cluster with private nodes."` } diff --git a/kubetest2-gke/deployer/up.go b/kubetest2-gke/deployer/up.go index aa68d75..c6a9bf8 100644 --- a/kubetest2-gke/deployer/up.go +++ b/kubetest2-gke/deployer/up.go @@ -334,6 +334,9 @@ func (d *Deployer) TestSetup() error { if err := d.EnsureFirewallRules(); err != nil { return err } + if err := d.EnsureNat(); err != nil { + return err + } d.testPrepared = true return nil }