Skip to content

Commit 1bd2ce9

Browse files
committed
wip: reserved capacity
1 parent cd92f5b commit 1bd2ce9

File tree

19 files changed

+513
-198
lines changed

19 files changed

+513
-198
lines changed

kwok/cloudprovider/helpers.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func setDefaultOptions(opts InstanceTypeOptions) InstanceTypeOptions {
135135

136136
// make sure all the instance types are available
137137
for i := range opts.Offerings {
138-
opts.Offerings[i].Available = true
138+
opts.Offerings[i].OfferingAvailabilityResolver = cloudprovider.TrueStaticAvailabilityResolver
139139
}
140140

141141
return opts
@@ -174,11 +174,11 @@ func newInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType {
174174
Requirements: requirements,
175175
Offerings: lo.Map(options.Offerings, func(off KWOKOffering, _ int) cloudprovider.Offering {
176176
return cloudprovider.Offering{
177+
OfferingAvailabilityResolver: off.Offering.OfferingAvailabilityResolver,
177178
Requirements: scheduling.NewRequirements(lo.Map(off.Requirements, func(req corev1.NodeSelectorRequirement, _ int) *scheduling.Requirement {
178179
return scheduling.NewRequirement(req.Key, req.Operator, req.Values...)
179180
})...),
180181
Price: off.Offering.Price,
181-
Available: off.Offering.Available,
182182
}
183183
}),
184184
Capacity: options.Resources,

kwok/tools/gen_instance_types.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,8 @@ func constructGenericInstanceTypes() []kwok.InstanceTypeOptions {
9898
corev1.NodeSelectorRequirement{Key: corev1.LabelTopologyZone, Operator: corev1.NodeSelectorOpIn, Values: []string{zone}},
9999
},
100100
Offering: cloudprovider.Offering{
101-
Price: lo.Ternary(ct == v1.CapacityTypeSpot, price*.7, price),
102-
Available: true,
101+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
102+
Price: lo.Ternary(ct == v1.CapacityTypeSpot, price*.7, price),
103103
},
104104
})
105105
}

pkg/apis/v1/labels.go

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ const (
3333
ArchitectureArm64 = "arm64"
3434
CapacityTypeSpot = "spot"
3535
CapacityTypeOnDemand = "on-demand"
36+
CapacityTypeReserved = "reserved"
3637
)
3738

3839
// Karpenter specific domains and labels

pkg/cloudprovider/fake/instancetype.go

+42-22
Original file line numberDiff line numberDiff line change
@@ -65,26 +65,46 @@ func NewInstanceTypeWithCustomRequirement(options InstanceTypeOptions, customReq
6565
}
6666
if len(options.Offerings) == 0 {
6767
options.Offerings = []cloudprovider.Offering{
68-
{Requirements: scheduling.NewLabelRequirements(map[string]string{
69-
v1.CapacityTypeLabelKey: "spot",
70-
corev1.LabelTopologyZone: "test-zone-1",
71-
}), Price: PriceFromResources(options.Resources), Available: true},
72-
{Requirements: scheduling.NewLabelRequirements(map[string]string{
73-
v1.CapacityTypeLabelKey: "spot",
74-
corev1.LabelTopologyZone: "test-zone-2",
75-
}), Price: PriceFromResources(options.Resources), Available: true},
76-
{Requirements: scheduling.NewLabelRequirements(map[string]string{
77-
v1.CapacityTypeLabelKey: "on-demand",
78-
corev1.LabelTopologyZone: "test-zone-1",
79-
}), Price: PriceFromResources(options.Resources), Available: true},
80-
{Requirements: scheduling.NewLabelRequirements(map[string]string{
81-
v1.CapacityTypeLabelKey: "on-demand",
82-
corev1.LabelTopologyZone: "test-zone-2",
83-
}), Price: PriceFromResources(options.Resources), Available: true},
84-
{Requirements: scheduling.NewLabelRequirements(map[string]string{
85-
v1.CapacityTypeLabelKey: "on-demand",
86-
corev1.LabelTopologyZone: "test-zone-3",
87-
}), Price: PriceFromResources(options.Resources), Available: true},
68+
{
69+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
70+
Requirements: scheduling.NewLabelRequirements(map[string]string{
71+
v1.CapacityTypeLabelKey: "spot",
72+
corev1.LabelTopologyZone: "test-zone-1",
73+
}),
74+
Price: PriceFromResources(options.Resources),
75+
},
76+
{
77+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
78+
Requirements: scheduling.NewLabelRequirements(map[string]string{
79+
v1.CapacityTypeLabelKey: "spot",
80+
corev1.LabelTopologyZone: "test-zone-2",
81+
}),
82+
Price: PriceFromResources(options.Resources),
83+
},
84+
{
85+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
86+
Requirements: scheduling.NewLabelRequirements(map[string]string{
87+
v1.CapacityTypeLabelKey: "on-demand",
88+
corev1.LabelTopologyZone: "test-zone-1",
89+
}),
90+
Price: PriceFromResources(options.Resources),
91+
},
92+
{
93+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
94+
Requirements: scheduling.NewLabelRequirements(map[string]string{
95+
v1.CapacityTypeLabelKey: "on-demand",
96+
corev1.LabelTopologyZone: "test-zone-2",
97+
}),
98+
Price: PriceFromResources(options.Resources),
99+
},
100+
{
101+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
102+
Requirements: scheduling.NewLabelRequirements(map[string]string{
103+
v1.CapacityTypeLabelKey: "on-demand",
104+
corev1.LabelTopologyZone: "test-zone-3",
105+
}),
106+
Price: PriceFromResources(options.Resources),
107+
},
88108
}
89109
}
90110
if len(options.Architecture) == 0 {
@@ -153,12 +173,12 @@ func InstanceTypesAssorted() []*cloudprovider.InstanceType {
153173
price := PriceFromResources(opts.Resources)
154174
opts.Offerings = []cloudprovider.Offering{
155175
{
176+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
156177
Requirements: scheduling.NewLabelRequirements(map[string]string{
157178
v1.CapacityTypeLabelKey: ct,
158179
corev1.LabelTopologyZone: zone,
159180
}),
160-
Price: price,
161-
Available: true,
181+
Price: price,
162182
},
163183
}
164184
instanceTypes = append(instanceTypes, NewInstanceType(opts))

pkg/cloudprovider/types.go

+109-4
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ import (
3838
var (
3939
SpotRequirement = scheduling.NewRequirements(scheduling.NewRequirement(v1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, v1.CapacityTypeSpot))
4040
OnDemandRequirement = scheduling.NewRequirements(scheduling.NewRequirement(v1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, v1.CapacityTypeOnDemand))
41+
42+
TrueStaticAvailabilityResolver OfferingAvailabilityResolver = staticAvailabilityResolver{available: true}
43+
FalseStaticAvailabilityResolver OfferingAvailabilityResolver = staticAvailabilityResolver{available: false}
4144
)
4245

4346
type DriftReason string
@@ -224,6 +227,15 @@ func (its InstanceTypes) Truncate(requirements scheduling.Requirements, maxItems
224227
return truncatedInstanceTypes, nil
225228
}
226229

230+
func (its InstanceTypes) Difference(other InstanceTypes) InstanceTypes {
231+
names := sets.New(lo.Map(other, func(it *InstanceType, _ int) string {
232+
return it.Name
233+
})...)
234+
return lo.Reject(its, func(it *InstanceType, _ int) bool {
235+
return names.Has(it.Name)
236+
})
237+
}
238+
227239
type InstanceTypeOverhead struct {
228240
// KubeReserved returns the default resources allocated to kubernetes system daemons by default
229241
KubeReserved corev1.ResourceList
@@ -237,24 +249,78 @@ func (i InstanceTypeOverhead) Total() corev1.ResourceList {
237249
return resources.Merge(i.KubeReserved, i.SystemReserved, i.EvictionThreshold)
238250
}
239251

252+
// An OfferingAvailabilityResolver is used to determine if there is available capacity for a given offering. To ensure
253+
// consistency between multiple controllers attempting to provision a NodeClaim with a given offering, offerings should
254+
// be "reserved" by the controller. Once a launch decision has been made, all offerings which were reserved may be
255+
// released, enabling their use once again.
256+
type OfferingAvailabilityResolver interface {
257+
Available() bool
258+
Reserve(string) bool
259+
GetReservation(string) OfferingReservation
260+
}
261+
262+
type OfferingReservation interface {
263+
Release()
264+
Commit()
265+
Matches(*v1.NodeClaim) bool
266+
}
267+
268+
type OfferingReservations []OfferingReservation
269+
270+
func (r OfferingReservations) Commit() {
271+
for _, reservation := range r {
272+
reservation.Commit()
273+
}
274+
}
275+
276+
func (r OfferingReservations) Release() {
277+
for _, reservation := range r {
278+
reservation.Release()
279+
}
280+
}
281+
282+
func (r OfferingReservations) Matching(nc *v1.NodeClaim) OfferingReservations {
283+
return lo.Filter(r, func(reservation OfferingReservation, _ int) bool {
284+
return reservation.Matches(nc)
285+
})
286+
}
287+
288+
240289
// An Offering describes where an InstanceType is available to be used, with the expectation that its properties
241290
// may be tightly coupled (e.g. the availability of an instance type in some zone is scoped to a capacity type) and
242291
// these properties are captured with labels in Requirements.
243292
// Requirements are required to contain the keys v1.CapacityTypeLabelKey and corev1.LabelTopologyZone
244293
type Offering struct {
294+
OfferingAvailabilityResolver
295+
245296
Requirements scheduling.Requirements
246297
Price float64
247-
// Available is added so that Offerings can return all offerings that have ever existed for an instance type,
248-
// so we can get historical pricing data for calculating savings in consolidation
249-
Available bool
250298
}
251299

252300
type Offerings []Offering
253301

302+
// Reserve attempts to make a reservation for each offering, returning true if it was successful for any.
303+
func (ofs Offerings) Reserve(id string) bool {
304+
success := false
305+
for i := range ofs {
306+
success = success || ofs[i].Reserve(id)
307+
}
308+
return success
309+
}
310+
311+
func (ofs Offerings) Reservations(id string) OfferingReservations {
312+
return lo.FilterMap(ofs, func(o Offering, _ int) (OfferingReservation, bool) {
313+
if reservation := o.GetReservation(id); reservation != nil {
314+
return reservation, true
315+
}
316+
return nil, false
317+
})
318+
}
319+
254320
// Available filters the available offerings from the returned offerings
255321
func (ofs Offerings) Available() Offerings {
256322
return lo.Filter(ofs, func(o Offering, _ int) bool {
257-
return o.Available
323+
return o.Available()
258324
})
259325
}
260326

@@ -397,3 +463,42 @@ func NewCreateError(err error, message string) *CreateError {
397463
ConditionMessage: message,
398464
}
399465
}
466+
467+
type staticAvailabilityResolver struct {
468+
requirements scheduling.Requirements
469+
available bool
470+
}
471+
472+
type noopReservation struct {
473+
requirements scheduling.Requirements
474+
}
475+
476+
func (r staticAvailabilityResolver) Available() bool {
477+
return r.available
478+
}
479+
480+
func (r staticAvailabilityResolver) Reserve(_ string) bool {
481+
return r.available
482+
}
483+
484+
func (r staticAvailabilityResolver) GetReservation(_ string) OfferingReservation {
485+
return noopReservation{
486+
requirements: r.requirements,
487+
}
488+
}
489+
490+
func (r noopReservation) Commit() {}
491+
492+
func (r noopReservation) Release() {}
493+
494+
func (r noopReservation) Matches(nc *v1.NodeClaim) bool {
495+
reqs := scheduling.NewLabelRequirements(nc.Labels)
496+
return reqs.IsCompatible(r.requirements, scheduling.AllowUndefinedWellKnownLabels)
497+
}
498+
499+
func NewStaticAvailabilityResolver(available bool, requirements scheduling.Requirements) OfferingAvailabilityResolver {
500+
return staticAvailabilityResolver{
501+
available: available,
502+
requirements: requirements,
503+
}
504+
}

pkg/controllers/disruption/consolidation_test.go

+27-27
Original file line numberDiff line numberDiff line change
@@ -2050,29 +2050,29 @@ var _ = Describe("Consolidation", func() {
20502050
Name: "current-on-demand",
20512051
Offerings: []cloudprovider.Offering{
20522052
{
2053-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
2054-
Price: 0.5,
2055-
Available: false,
2053+
OfferingAvailabilityResolver: cloudprovider.FalseStaticAvailabilityResolver,
2054+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
2055+
Price: 0.5,
20562056
},
20572057
},
20582058
})
20592059
replacementInstance := fake.NewInstanceType(fake.InstanceTypeOptions{
20602060
Name: "potential-spot-replacement",
20612061
Offerings: []cloudprovider.Offering{
20622062
{
2063-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1a"}),
2064-
Price: 1.0,
2065-
Available: true,
2063+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
2064+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1a"}),
2065+
Price: 1.0,
20662066
},
20672067
{
2068-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1b"}),
2069-
Price: 0.2,
2070-
Available: true,
2068+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
2069+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1b"}),
2070+
Price: 0.2,
20712071
},
20722072
{
2073-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1c"}),
2074-
Price: 0.4,
2075-
Available: true,
2073+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
2074+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1c"}),
2075+
Price: 0.4,
20762076
},
20772077
},
20782078
})
@@ -2134,34 +2134,34 @@ var _ = Describe("Consolidation", func() {
21342134
Name: "current-on-demand",
21352135
Offerings: []cloudprovider.Offering{
21362136
{
2137-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
2138-
Price: 0.5,
2139-
Available: false,
2137+
OfferingAvailabilityResolver: cloudprovider.FalseStaticAvailabilityResolver,
2138+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
2139+
Price: 0.5,
21402140
},
21412141
},
21422142
})
21432143
replacementInstance := fake.NewInstanceType(fake.InstanceTypeOptions{
21442144
Name: "on-demand-replacement",
21452145
Offerings: []cloudprovider.Offering{
21462146
{
2147-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
2148-
Price: 0.6,
2149-
Available: true,
2147+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
2148+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
2149+
Price: 0.6,
21502150
},
21512151
{
2152-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1b"}),
2153-
Price: 0.6,
2154-
Available: true,
2152+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
2153+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1b"}),
2154+
Price: 0.6,
21552155
},
21562156
{
2157-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1b"}),
2158-
Price: 0.2,
2159-
Available: true,
2157+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
2158+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1b"}),
2159+
Price: 0.2,
21602160
},
21612161
{
2162-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1c"}),
2163-
Price: 0.3,
2164-
Available: true,
2162+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
2163+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeSpot, corev1.LabelTopologyZone: "test-zone-1c"}),
2164+
Price: 0.3,
21652165
},
21662166
},
21672167
})

pkg/controllers/disruption/drift_test.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -775,19 +775,19 @@ var _ = Describe("Drift", func() {
775775
Name: "current-on-demand",
776776
Offerings: []cloudprovider.Offering{
777777
{
778-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
779-
Price: 0.5,
780-
Available: false,
778+
OfferingAvailabilityResolver: cloudprovider.FalseStaticAvailabilityResolver,
779+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
780+
Price: 0.5,
781781
},
782782
},
783783
})
784784
replacementInstance := fake.NewInstanceType(fake.InstanceTypeOptions{
785785
Name: "replacement-on-demand",
786786
Offerings: []cloudprovider.Offering{
787787
{
788-
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
789-
Price: 0.3,
790-
Available: true,
788+
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
789+
Requirements: scheduling.NewLabelRequirements(map[string]string{v1.CapacityTypeLabelKey: v1.CapacityTypeOnDemand, corev1.LabelTopologyZone: "test-zone-1a"}),
790+
Price: 0.3,
791791
},
792792
},
793793
Resources: map[corev1.ResourceName]resource.Quantity{corev1.ResourceCPU: resource.MustParse("3")},

0 commit comments

Comments
 (0)