backport [release-v1.11.x]: feat: expose subnet refresh interval as configurable setting (#9150)

charts/karpenter/README.md

@@ -91,7 +91,7 @@ cosign verify public.ecr.aws/karpenter/karpenter:1.10.0 \
 | serviceMonitor.metricRelabelings | list | `[]` | Metric relabelings for the `http-metrics` endpoint on the ServiceMonitor. For more details on metric relabelings, see: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs |
 | serviceMonitor.relabelings | list | `[]` | Relabelings for the `http-metrics` endpoint on the ServiceMonitor. For more details on relabelings, see: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config |
 | serviceMonitor.sampleLimit | string | `nil` | Specifies the sampleLimit for prometheus scrapes. Per-scrape limit on the number of scraped samples that will be accepted. If more than this number of samples are present after metric relabeling the entire scrape will be treated as failed. 0 means no limit. |
-| settings | object | `{"batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","disableClusterStateObservability":false,"disableDryRun":false,"eksControlPlane":false,"featureGates":{"nodeOverlay":false,"nodeRepair":false,"reservedCapacity":true,"spotToSpotConsolidation":false,"staticCapacity":false},"ignoreDRARequests":true,"interruptionQueue":"","isolatedVPC":false,"minValuesPolicy":"Strict","preferencePolicy":"Respect","reservedENIs":"0","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter |
+| settings | object | `{"batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","disableClusterStateObservability":false,"disableDryRun":false,"eksControlPlane":false,"featureGates":{"nodeOverlay":false,"nodeRepair":false,"reservedCapacity":true,"spotToSpotConsolidation":false,"staticCapacity":false},"ignoreDRARequests":true,"interruptionQueue":"","isolatedVPC":false,"minValuesPolicy":"Strict","preferencePolicy":"Respect","reservedENIs":"0","securityGroupRefreshInterval":"1m","subnetRefreshInterval":"1m","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter |
 | settings.batchIdleDuration | string | `"1s"` | The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. |
 | settings.batchMaxDuration | string | `"10s"` | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. |
 | settings.clusterCABundle | string | `""` | Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. |
@@ -112,6 +112,8 @@ cosign verify public.ecr.aws/karpenter/karpenter:1.10.0 \
 | settings.minValuesPolicy | string | `"Strict"` | How the Karpenter scheduler treats min values. Options include 'Strict' (fails scheduling when min values can't be met) and 'BestEffort' (relaxes min values when they can't be met). |
 | settings.preferencePolicy | string | `"Respect"` | How the Karpenter scheduler should treat preferences. Preferences include preferredDuringSchedulingIgnoreDuringExecution node and pod affinities/anti-affinities and ScheduleAnyways topologySpreadConstraints. Can be one of 'Ignore' and 'Respect' |
 | settings.reservedENIs | string | `"0"` | Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html. |
+| settings.securityGroupRefreshInterval | string | `"1m"` | How often Karpenter refreshes security group data from EC2. Increasing this value will reduce the number of DescribeSecurityGroups API calls at the cost of increased staleness in security group discovery. Must be at least 1m. |
+| settings.subnetRefreshInterval | string | `"1m"` | How often Karpenter refreshes subnet data from EC2. Increasing this value will reduce the number of DescribeSubnets API calls at the cost of increased staleness in subnet discovery. Must be at least 1m. |
 | settings.vmMemoryOverheadPercent | float | `0.075` | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. The value of `0.075` equals to 7.5%. |
 | strategy | object | `{"rollingUpdate":{"maxUnavailable":1}}` | Strategy for updating the pod. |
 | terminationGracePeriodSeconds | string | `nil` | Override the default termination grace period for the pod. |

charts/karpenter/templates/deployment.yaml

@@ -185,6 +185,14 @@ spec:
             - name: DISABLE_DRY_RUN
               value: "{{ tpl (toString .) $ }}"
           {{- end }}
+          {{- with .Values.settings.subnetRefreshInterval }}
+            - name: SUBNET_REFRESH_INTERVAL
+              value: "{{ tpl (toString .) $ }}"
+          {{- end }}
+          {{- with .Values.settings.securityGroupRefreshInterval }}
+            - name: SECURITY_GROUP_REFRESH_INTERVAL
+              value: "{{ tpl (toString .) $ }}"
+          {{- end }}
           {{- with .Values.controller.env }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

charts/karpenter/values.yaml

@@ -216,6 +216,12 @@ settings:
   disableClusterStateObservability: false
   # -- Disable dry run validation for EC2NodeClasses.
   disableDryRun: false
+  # -- How often Karpenter refreshes subnet data from EC2. Increasing this value will reduce the number of
+  # DescribeSubnets API calls at the cost of increased staleness in subnet discovery. Must be at least 1m.
+  subnetRefreshInterval: 1m
+  # -- How often Karpenter refreshes security group data from EC2. Increasing this value will reduce the number of
+  # DescribeSecurityGroups API calls at the cost of increased staleness in security group discovery. Must be at least 1m.
+  securityGroupRefreshInterval: 1m
   # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates
   # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features.
   featureGates:

kwok/operator/operator.go

@@ -125,8 +125,11 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 	validationCache := cache.New(awscache.ValidationTTL, awscache.DefaultCleanupInterval)
 	recreationCache := cache.New(awscache.RecreationTTL, awscache.DefaultCleanupInterval)
 
-	subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval))
-	securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
+	subnetRefreshInterval := options.FromContext(ctx).SubnetRefreshInterval
+	subnetIPCacheTTL := max(awscache.AvailableIPAddressTTL, subnetRefreshInterval+(awscache.AvailableIPAddressTTL-awscache.DefaultTTL))
+	subnetPublicIPCacheTTL := max(awscache.AssociatePublicIPAddressTTL, subnetRefreshInterval+(awscache.AssociatePublicIPAddressTTL-awscache.DefaultTTL))
+	subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(subnetRefreshInterval, awscache.DefaultCleanupInterval), cache.New(subnetIPCacheTTL, awscache.DefaultCleanupInterval), cache.New(subnetPublicIPCacheTTL, awscache.DefaultCleanupInterval))
+	securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(options.FromContext(ctx).SecurityGroupRefreshInterval, awscache.DefaultCleanupInterval))
 	instanceProfileProvider := instanceprofile.NewDefaultProvider(
 		iam.NewFromConfig(cfg),
 		cache.New(awscache.InstanceProfileTTL, awscache.DefaultCleanupInterval),

pkg/operator/operator.go

@@ -140,8 +140,11 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 	validationCache := cache.New(awscache.ValidationTTL, awscache.DefaultCleanupInterval)
 	recreationCache := cache.New(awscache.RecreationTTL, awscache.DefaultCleanupInterval)
 
-	subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval), cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval))
-	securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
+	subnetRefreshInterval := options.FromContext(ctx).SubnetRefreshInterval
+	subnetIPCacheTTL := max(awscache.AvailableIPAddressTTL, subnetRefreshInterval+(awscache.AvailableIPAddressTTL-awscache.DefaultTTL))
+	subnetPublicIPCacheTTL := max(awscache.AssociatePublicIPAddressTTL, subnetRefreshInterval+(awscache.AssociatePublicIPAddressTTL-awscache.DefaultTTL))
+	subnetProvider := subnet.NewDefaultProvider(ec2api, cache.New(subnetRefreshInterval, awscache.DefaultCleanupInterval), cache.New(subnetIPCacheTTL, awscache.DefaultCleanupInterval), cache.New(subnetPublicIPCacheTTL, awscache.DefaultCleanupInterval))
+	securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(options.FromContext(ctx).SecurityGroupRefreshInterval, awscache.DefaultCleanupInterval))
 	instanceProfileProvider := instanceprofile.NewDefaultProvider(
 		iam.NewFromConfig(cfg),
 		cache.New(awscache.InstanceProfileTTL, awscache.DefaultCleanupInterval),

pkg/operator/options/options.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
 	coreoptions "sigs.k8s.io/karpenter/pkg/operator/options"
 	"sigs.k8s.io/karpenter/pkg/utils/env"
@@ -34,15 +35,17 @@ func init() {
 type optionsKey struct{}
 
 type Options struct {
-	ClusterCABundle         string
-	ClusterName             string
-	ClusterEndpoint         string
-	IsolatedVPC             bool
-	EKSControlPlane         bool
-	VMMemoryOverheadPercent float64
-	InterruptionQueue       string
-	ReservedENIs            int
-	DisableDryRun           bool
+	ClusterCABundle              string
+	ClusterName                  string
+	ClusterEndpoint              string
+	IsolatedVPC                  bool
+	EKSControlPlane              bool
+	VMMemoryOverheadPercent      float64
+	InterruptionQueue            string
+	ReservedENIs                 int
+	DisableDryRun                bool
+	SubnetRefreshInterval        time.Duration
+	SecurityGroupRefreshInterval time.Duration
 }
 
 func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
@@ -55,6 +58,8 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
 	fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is the name of the SQS queue used for processing interruption events from EC2. Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.")
 	fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.")
 	fs.BoolVarWithEnv(&o.DisableDryRun, "disable-dry-run", "DISABLE_DRY_RUN", false, "If true, then disable dry run validation for EC2NodeClasses.")
+	fs.DurationVar(&o.SubnetRefreshInterval, "subnet-refresh-interval", env.WithDefaultDuration("SUBNET_REFRESH_INTERVAL", time.Minute), "How often Karpenter refreshes subnet data from EC2. Increasing this value will reduce the number of DescribeSubnets API calls at the cost of increased staleness in subnet discovery. Must be at least 1m.")
+	fs.DurationVar(&o.SecurityGroupRefreshInterval, "security-group-refresh-interval", env.WithDefaultDuration("SECURITY_GROUP_REFRESH_INTERVAL", time.Minute), "How often Karpenter refreshes security group data from EC2. Increasing this value will reduce the number of DescribeSecurityGroups API calls at the cost of increased staleness in security group discovery. Must be at least 1m.")
 }
 
 func (o *Options) Parse(fs *coreoptions.FlagSet, args ...string) error {

pkg/operator/options/options_validation.go

@@ -17,6 +17,7 @@ package options
 import (
 	"fmt"
 	"net/url"
+	"time"
 
 	"github.com/awslabs/operatorpkg/serrors"
 	"go.uber.org/multierr"
@@ -28,9 +29,25 @@ func (o *Options) Validate() error {
 		o.validateVMMemoryOverheadPercent(),
 		o.validateReservedENIs(),
 		o.validateRequiredFields(),
+		o.validateSubnetRefreshInterval(),
+		o.validateSecurityGroupRefreshInterval(),
 	)
 }
 
+func (o *Options) validateSubnetRefreshInterval() error {
+	if o.SubnetRefreshInterval < time.Minute {
+		return fmt.Errorf("subnet-refresh-interval must be at least 1m")
+	}
+	return nil
+}
+
+func (o *Options) validateSecurityGroupRefreshInterval() error {
+	if o.SecurityGroupRefreshInterval < time.Minute {
+		return fmt.Errorf("security-group-refresh-interval must be at least 1m")
+	}
+	return nil
+}
+
 func (o *Options) validateEndpoint() error {
 	if o.ClusterEndpoint == "" {
 		return nil

pkg/operator/options/suite_test.go

@@ -19,6 +19,7 @@ import (
 	"flag"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/samber/lo"
 	coreoptions "sigs.k8s.io/karpenter/pkg/operator/options"
@@ -63,17 +64,21 @@ var _ = Describe("Options", func() {
 			"--vm-memory-overhead-percent", "0.1",
 			"--interruption-queue", "env-cluster",
 			"--reserved-enis", "10",
-			"--disable-dry-run")
+			"--disable-dry-run",
+			"--subnet-refresh-interval", "15m",
+			"--security-group-refresh-interval", "15m")
 		Expect(err).ToNot(HaveOccurred())
 		expectOptionsEqual(opts, test.Options(test.OptionsFields{
-			ClusterCABundle:         lo.ToPtr("env-bundle"),
-			ClusterName:             lo.ToPtr("env-cluster"),
-			ClusterEndpoint:         lo.ToPtr("https://env-cluster"),
-			IsolatedVPC:             lo.ToPtr(true),
-			VMMemoryOverheadPercent: lo.ToPtr[float64](0.1),
-			InterruptionQueue:       lo.ToPtr("env-cluster"),
-			ReservedENIs:            lo.ToPtr(10),
-			DisableDryRun:           lo.ToPtr(true),
+			ClusterCABundle:              lo.ToPtr("env-bundle"),
+			ClusterName:                  lo.ToPtr("env-cluster"),
+			ClusterEndpoint:              lo.ToPtr("https://env-cluster"),
+			IsolatedVPC:                  lo.ToPtr(true),
+			VMMemoryOverheadPercent:      lo.ToPtr[float64](0.1),
+			InterruptionQueue:            lo.ToPtr("env-cluster"),
+			ReservedENIs:                 lo.ToPtr(10),
+			DisableDryRun:                lo.ToPtr(true),
+			SubnetRefreshInterval:        lo.ToPtr(15 * time.Minute),
+			SecurityGroupRefreshInterval: lo.ToPtr(15 * time.Minute),
 		}))
 	})
 	It("should correctly fallback to env vars when CLI flags aren't set", func() {
@@ -85,24 +90,42 @@ var _ = Describe("Options", func() {
 		os.Setenv("INTERRUPTION_QUEUE", "env-cluster")
 		os.Setenv("RESERVED_ENIS", "10")
 		os.Setenv("DISABLE_DRY_RUN", "false")
+		os.Setenv("SUBNET_REFRESH_INTERVAL", "15m")
+		os.Setenv("SECURITY_GROUP_REFRESH_INTERVAL", "15m")
 
 		// Add flags after we set the environment variables so that the parsing logic correctly refers
 		// to the new environment variable values
 		opts.AddFlags(fs)
 		err := opts.Parse(fs)
 		Expect(err).ToNot(HaveOccurred())
 		expectOptionsEqual(opts, test.Options(test.OptionsFields{
-			ClusterCABundle:         lo.ToPtr("env-bundle"),
-			ClusterName:             lo.ToPtr("env-cluster"),
-			ClusterEndpoint:         lo.ToPtr("https://env-cluster"),
-			IsolatedVPC:             lo.ToPtr(true),
-			VMMemoryOverheadPercent: lo.ToPtr[float64](0.1),
-			InterruptionQueue:       lo.ToPtr("env-cluster"),
-			ReservedENIs:            lo.ToPtr(10),
-			DisableDryRun:           lo.ToPtr(false),
+			ClusterCABundle:              lo.ToPtr("env-bundle"),
+			ClusterName:                  lo.ToPtr("env-cluster"),
+			ClusterEndpoint:              lo.ToPtr("https://env-cluster"),
+			IsolatedVPC:                  lo.ToPtr(true),
+			VMMemoryOverheadPercent:      lo.ToPtr[float64](0.1),
+			InterruptionQueue:            lo.ToPtr("env-cluster"),
+			ReservedENIs:                 lo.ToPtr(10),
+			DisableDryRun:                lo.ToPtr(false),
+			SubnetRefreshInterval:        lo.ToPtr(15 * time.Minute),
+			SecurityGroupRefreshInterval: lo.ToPtr(15 * time.Minute),
 		}))
 	})
 
+	It("should correctly use default security-group-refresh-interval when not specified", func() {
+		opts.AddFlags(fs)
+		err := opts.Parse(fs, "--cluster-name", "test-cluster")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(opts.SecurityGroupRefreshInterval).To(Equal(time.Minute))
+	})
+
+	It("should correctly use default subnet-refresh-interval when not specified", func() {
+		opts.AddFlags(fs)
+		err := opts.Parse(fs, "--cluster-name", "test-cluster")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(opts.SubnetRefreshInterval).To(Equal(time.Minute))
+	})
+
 	Context("Validation", func() {
 		BeforeEach(func() {
 			opts.AddFlags(fs)
@@ -123,6 +146,14 @@ var _ = Describe("Options", func() {
 			err := opts.Parse(fs, "--cluster-name", "test-cluster", "--reserved-enis", "-1")
 			Expect(err).To(HaveOccurred())
 		})
+		It("should fail when subnet-refresh-interval is less than 1m", func() {
+			err := opts.Parse(fs, "--cluster-name", "test-cluster", "--subnet-refresh-interval", "30s")
+			Expect(err).To(HaveOccurred())
+		})
+		It("should fail when security-group-refresh-interval is less than 1m", func() {
+			err := opts.Parse(fs, "--cluster-name", "test-cluster", "--security-group-refresh-interval", "30s")
+			Expect(err).To(HaveOccurred())
+		})
 	})
 })
 
@@ -136,4 +167,6 @@ func expectOptionsEqual(optsA *options.Options, optsB *options.Options) {
 	Expect(optsA.InterruptionQueue).To(Equal(optsB.InterruptionQueue))
 	Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs))
 	Expect(optsA.DisableDryRun).To(Equal(optsB.DisableDryRun))
+	Expect(optsA.SubnetRefreshInterval).To(Equal(optsB.SubnetRefreshInterval))
+	Expect(optsA.SecurityGroupRefreshInterval).To(Equal(optsB.SecurityGroupRefreshInterval))
 }