Skip to content

Commit

Permalink
chore: Allow setting leader election namespace when running out of cl…
Browse files Browse the repository at this point in the history 
…uster (#1704)
  • Loading branch information
@jonathan-innis
jonathan-innis authored Sep 21, 2024
1 parent e410762 commit 04a921c
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 81 deletions.
1 change: 1 addition & 0 deletions pkg/operator/operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ func NewOperator() (context.Context, *Operator) {
Logger: logging.IgnoreDebugEvents(logger),
LeaderElection: !options.FromContext(ctx).DisableLeaderElection,
LeaderElectionID: "karpenter-leader-election",
LeaderElectionNamespace: options.FromContext(ctx).LeaderElectionNamespace,
LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
LeaderElectionReleaseOnCancel: true,
Metrics: server.Options{
Expand Down
32 changes: 17 additions & 15 deletions pkg/operator/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,20 +46,21 @@ type FeatureGates struct {

// Options contains all CLI flags / env vars for karpenter-core. It adheres to the options.Injectable interface.
type Options struct {
ServiceName string
MetricsPort int
HealthProbePort int
KubeClientQPS int
KubeClientBurst int
EnableProfiling bool
DisableLeaderElection bool
MemoryLimit int64
LogLevel string
LogOutputPaths string
LogErrorOutputPaths string
BatchMaxDuration time.Duration
BatchIdleDuration time.Duration
FeatureGates FeatureGates
ServiceName string
MetricsPort int
HealthProbePort int
KubeClientQPS int
KubeClientBurst int
EnableProfiling bool
DisableLeaderElection bool
LeaderElectionNamespace string
MemoryLimit int64
LogLevel string
LogOutputPaths string
LogErrorOutputPaths string
BatchMaxDuration time.Duration
BatchIdleDuration time.Duration
FeatureGates FeatureGates
}

type FlagSet struct {
Expand Down Expand Up @@ -87,6 +88,7 @@ func (o *Options) AddFlags(fs *FlagSet) {
fs.IntVar(&o.KubeClientBurst, "kube-client-burst", env.WithDefaultInt("KUBE_CLIENT_BURST", 300), "The maximum allowed burst of queries to the kube-apiserver")
fs.BoolVarWithEnv(&o.EnableProfiling, "enable-profiling", "ENABLE_PROFILING", false, "Enable the profiling on the metric endpoint")
fs.BoolVarWithEnv(&o.DisableLeaderElection, "disable-leader-election", "DISABLE_LEADER_ELECTION", false, "Disable the leader election client before executing the main loop. Disable when running replicated components for high availability is not desired.")
fs.StringVar(&o.LeaderElectionNamespace, "leader-election-namespace", env.WithDefaultString("LEADER_ELECTION_NAMESPACE", ""), "Leader election namespace to create and monitor the lease if running outside the cluster")
fs.Int64Var(&o.MemoryLimit, "memory-limit", env.WithDefaultInt64("MEMORY_LIMIT", -1), "Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value.")
fs.StringVar(&o.LogLevel, "log-level", env.WithDefaultString("LOG_LEVEL", "info"), "Log verbosity level. Can be one of 'debug', 'info', or 'error'")
fs.StringVar(&o.LogOutputPaths, "log-output-paths", env.WithDefaultString("LOG_OUTPUT_PATHS", "stdout"), "Optional comma separated paths for directing log output")
Expand All @@ -105,7 +107,7 @@ func (o *Options) Parse(fs *FlagSet, args ...string) error {
}

if !lo.Contains(validLogLevels, o.LogLevel) {
return fmt.Errorf("validating cli flags / env vars, invalid log level %q", o.LogLevel)
return fmt.Errorf("validating cli flags / env vars, invalid LOG_LEVEL %q", o.LogLevel)
}
gates, err := ParseFeatureGates(o.FeatureGates.inputStr)
if err != nil {
Expand Down
111 changes: 59 additions & 52 deletions pkg/operator/options/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ var _ = Describe("Options", func() {
"KUBE_CLIENT_BURST",
"ENABLE_PROFILING",
"DISABLE_LEADER_ELECTION",
"LEADER_ELECTION_NAMESPACE",
"MEMORY_LIMIT",
"LOG_LEVEL",
"LOG_OUTPUT_PATHS",
Expand Down Expand Up @@ -94,19 +95,20 @@ var _ = Describe("Options", func() {
err := opts.Parse(fs)
Expect(err).To(BeNil())
expectOptionsMatch(opts, test.Options(test.OptionsFields{
ServiceName: lo.ToPtr(""),
MetricsPort: lo.ToPtr(8080),
HealthProbePort: lo.ToPtr(8081),
KubeClientQPS: lo.ToPtr(200),
KubeClientBurst: lo.ToPtr(300),
EnableProfiling: lo.ToPtr(false),
DisableLeaderElection: lo.ToPtr(false),
MemoryLimit: lo.ToPtr[int64](-1),
LogLevel: lo.ToPtr("info"),
LogOutputPaths: lo.ToPtr("stdout"),
LogErrorOutputPaths: lo.ToPtr("stderr"),
BatchMaxDuration: lo.ToPtr(10 * time.Second),
BatchIdleDuration: lo.ToPtr(time.Second),
ServiceName: lo.ToPtr(""),
MetricsPort: lo.ToPtr(8080),
HealthProbePort: lo.ToPtr(8081),
KubeClientQPS: lo.ToPtr(200),
KubeClientBurst: lo.ToPtr(300),
EnableProfiling: lo.ToPtr(false),
DisableLeaderElection: lo.ToPtr(false),
LeaderElectionNamespace: lo.ToPtr(""),
MemoryLimit: lo.ToPtr[int64](-1),
LogLevel: lo.ToPtr("info"),
LogOutputPaths: lo.ToPtr("stdout"),
LogErrorOutputPaths: lo.ToPtr("stderr"),
BatchMaxDuration: lo.ToPtr(10 * time.Second),
BatchIdleDuration: lo.ToPtr(time.Second),
FeatureGates: test.FeatureGates{
SpotToSpotConsolidation: lo.ToPtr(false),
},
Expand All @@ -125,6 +127,7 @@ var _ = Describe("Options", func() {
"--kube-client-burst", "0",
"--enable-profiling",
"--disable-leader-election=true",
"--leader-election-namespace=karpenter",
"--memory-limit", "0",
"--log-level", "debug",
"--log-output-paths", "/etc/k8s/test",
Expand All @@ -135,19 +138,20 @@ var _ = Describe("Options", func() {
)
Expect(err).To(BeNil())
expectOptionsMatch(opts, test.Options(test.OptionsFields{
ServiceName: lo.ToPtr("cli"),
MetricsPort: lo.ToPtr(0),
HealthProbePort: lo.ToPtr(0),
KubeClientQPS: lo.ToPtr(0),
KubeClientBurst: lo.ToPtr(0),
EnableProfiling: lo.ToPtr(true),
DisableLeaderElection: lo.ToPtr(true),
MemoryLimit: lo.ToPtr[int64](0),
LogLevel: lo.ToPtr("debug"),
LogOutputPaths: lo.ToPtr("/etc/k8s/test"),
LogErrorOutputPaths: lo.ToPtr("/etc/k8s/testerror"),
BatchMaxDuration: lo.ToPtr(5 * time.Second),
BatchIdleDuration: lo.ToPtr(5 * time.Second),
ServiceName: lo.ToPtr("cli"),
MetricsPort: lo.ToPtr(0),
HealthProbePort: lo.ToPtr(0),
KubeClientQPS: lo.ToPtr(0),
KubeClientBurst: lo.ToPtr(0),
EnableProfiling: lo.ToPtr(true),
DisableLeaderElection: lo.ToPtr(true),
LeaderElectionNamespace: lo.ToPtr("karpenter"),
MemoryLimit: lo.ToPtr[int64](0),
LogLevel: lo.ToPtr("debug"),
LogOutputPaths: lo.ToPtr("/etc/k8s/test"),
LogErrorOutputPaths: lo.ToPtr("/etc/k8s/testerror"),
BatchMaxDuration: lo.ToPtr(5 * time.Second),
BatchIdleDuration: lo.ToPtr(5 * time.Second),
FeatureGates: test.FeatureGates{
SpotToSpotConsolidation: lo.ToPtr(true),
},
Expand All @@ -162,6 +166,7 @@ var _ = Describe("Options", func() {
os.Setenv("KUBE_CLIENT_BURST", "0")
os.Setenv("ENABLE_PROFILING", "true")
os.Setenv("DISABLE_LEADER_ELECTION", "true")
os.Setenv("LEADER_ELECTION_NAMESPACE", "karpenter")
os.Setenv("MEMORY_LIMIT", "0")
os.Setenv("LOG_LEVEL", "debug")
os.Setenv("LOG_OUTPUT_PATHS", "/etc/k8s/test")
Expand All @@ -176,19 +181,20 @@ var _ = Describe("Options", func() {
err := opts.Parse(fs)
Expect(err).To(BeNil())
expectOptionsMatch(opts, test.Options(test.OptionsFields{
ServiceName: lo.ToPtr("env"),
MetricsPort: lo.ToPtr(0),
HealthProbePort: lo.ToPtr(0),
KubeClientQPS: lo.ToPtr(0),
KubeClientBurst: lo.ToPtr(0),
EnableProfiling: lo.ToPtr(true),
DisableLeaderElection: lo.ToPtr(true),
MemoryLimit: lo.ToPtr[int64](0),
LogLevel: lo.ToPtr("debug"),
LogOutputPaths: lo.ToPtr("/etc/k8s/test"),
LogErrorOutputPaths: lo.ToPtr("/etc/k8s/testerror"),
BatchMaxDuration: lo.ToPtr(5 * time.Second),
BatchIdleDuration: lo.ToPtr(5 * time.Second),
ServiceName: lo.ToPtr("env"),
MetricsPort: lo.ToPtr(0),
HealthProbePort: lo.ToPtr(0),
KubeClientQPS: lo.ToPtr(0),
KubeClientBurst: lo.ToPtr(0),
EnableProfiling: lo.ToPtr(true),
DisableLeaderElection: lo.ToPtr(true),
LeaderElectionNamespace: lo.ToPtr("karpenter"),
MemoryLimit: lo.ToPtr[int64](0),
LogLevel: lo.ToPtr("debug"),
LogOutputPaths: lo.ToPtr("/etc/k8s/test"),
LogErrorOutputPaths: lo.ToPtr("/etc/k8s/testerror"),
BatchMaxDuration: lo.ToPtr(5 * time.Second),
BatchIdleDuration: lo.ToPtr(5 * time.Second),
FeatureGates: test.FeatureGates{
SpotToSpotConsolidation: lo.ToPtr(true),
},
Expand Down Expand Up @@ -219,19 +225,20 @@ var _ = Describe("Options", func() {
)
Expect(err).To(BeNil())
expectOptionsMatch(opts, test.Options(test.OptionsFields{
ServiceName: lo.ToPtr("cli"),
MetricsPort: lo.ToPtr(0),
HealthProbePort: lo.ToPtr(0),
KubeClientQPS: lo.ToPtr(0),
KubeClientBurst: lo.ToPtr(0),
EnableProfiling: lo.ToPtr(true),
DisableLeaderElection: lo.ToPtr(true),
MemoryLimit: lo.ToPtr[int64](0),
LogLevel: lo.ToPtr("debug"),
LogOutputPaths: lo.ToPtr("/etc/k8s/test"),
LogErrorOutputPaths: lo.ToPtr("/etc/k8s/testerror"),
BatchMaxDuration: lo.ToPtr(5 * time.Second),
BatchIdleDuration: lo.ToPtr(5 * time.Second),
ServiceName: lo.ToPtr("cli"),
MetricsPort: lo.ToPtr(0),
HealthProbePort: lo.ToPtr(0),
KubeClientQPS: lo.ToPtr(0),
KubeClientBurst: lo.ToPtr(0),
EnableProfiling: lo.ToPtr(true),
DisableLeaderElection: lo.ToPtr(true),
LeaderElectionNamespace: lo.ToPtr(""),
MemoryLimit: lo.ToPtr[int64](0),
LogLevel: lo.ToPtr("debug"),
LogOutputPaths: lo.ToPtr("/etc/k8s/test"),
LogErrorOutputPaths: lo.ToPtr("/etc/k8s/testerror"),
BatchMaxDuration: lo.ToPtr(5 * time.Second),
BatchIdleDuration: lo.ToPtr(5 * time.Second),
FeatureGates: test.FeatureGates{
SpotToSpotConsolidation: lo.ToPtr(true),
},
Expand Down
29 changes: 15 additions & 14 deletions pkg/test/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,21 @@ import (

type OptionsFields struct {
// Vendor Neutral
ServiceName *string
MetricsPort *int
HealthProbePort *int
KubeClientQPS *int
KubeClientBurst *int
EnableProfiling *bool
DisableLeaderElection *bool
MemoryLimit *int64
LogLevel *string
LogOutputPaths *string
LogErrorOutputPaths *string
BatchMaxDuration *time.Duration
BatchIdleDuration *time.Duration
FeatureGates FeatureGates
ServiceName *string
MetricsPort *int
HealthProbePort *int
KubeClientQPS *int
KubeClientBurst *int
EnableProfiling *bool
DisableLeaderElection *bool
LeaderElectionNamespace *string
MemoryLimit *int64
LogLevel *string
LogOutputPaths *string
LogErrorOutputPaths *string
BatchMaxDuration *time.Duration
BatchIdleDuration *time.Duration
FeatureGates FeatureGates
}

type FeatureGates struct {
Expand Down

0 comments on commit 04a921c

Please sign in to comment.