Skip to content

Commit afe2a47

Browse files
committed
wip
1 parent f611c32 commit afe2a47

23 files changed

+325
-416
lines changed

kwok/apis/crds/karpenter.kwok.sh_kwoknodeclasses.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
33
kind: CustomResourceDefinition
44
metadata:
55
annotations:
6-
controller-gen.kubebuilder.io/version: v0.17.1
6+
controller-gen.kubebuilder.io/version: v0.16.5
77
name: kwoknodeclasses.karpenter.kwok.sh
88
spec:
99
group: karpenter.kwok.sh

kwok/charts/crds/karpenter.sh_nodeclaims.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
33
kind: CustomResourceDefinition
44
metadata:
55
annotations:
6-
controller-gen.kubebuilder.io/version: v0.17.1
6+
controller-gen.kubebuilder.io/version: v0.16.5
77
name: nodeclaims.karpenter.sh
88
spec:
99
group: karpenter.sh

kwok/charts/crds/karpenter.sh_nodepools.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
33
kind: CustomResourceDefinition
44
metadata:
55
annotations:
6-
controller-gen.kubebuilder.io/version: v0.17.1
6+
controller-gen.kubebuilder.io/version: v0.16.5
77
name: nodepools.karpenter.sh
88
spec:
99
group: karpenter.sh

kwok/cloudprovider/helpers.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ func setDefaultOptions(opts InstanceTypeOptions) InstanceTypeOptions {
147147

148148
// make sure all the instance types are available
149149
for i := range opts.Offerings {
150-
opts.Offerings[i].OfferingAvailabilityResolver = cloudprovider.TrueStaticAvailabilityResolver
150+
opts.Offerings[i].Available = true
151151
}
152152

153153
return opts
@@ -186,11 +186,11 @@ func newInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType {
186186
Requirements: requirements,
187187
Offerings: lo.Map(options.Offerings, func(off KWOKOffering, _ int) cloudprovider.Offering {
188188
return cloudprovider.Offering{
189-
OfferingAvailabilityResolver: off.Offering.OfferingAvailabilityResolver,
189+
ReservationManager: off.Offering.ReservationManager,
190190
Requirements: scheduling.NewRequirements(lo.Map(off.Requirements, func(req corev1.NodeSelectorRequirement, _ int) *scheduling.Requirement {
191191
return scheduling.NewRequirement(req.Key, req.Operator, req.Values...)
192192
})...),
193-
Price: off.Offering.Price,
193+
Price: off.Offering.Price,
194194
}
195195
}),
196196
Capacity: options.Resources,

kwok/tools/gen_instance_types.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,8 @@ func constructGenericInstanceTypes() []kwok.InstanceTypeOptions {
9898
corev1.NodeSelectorRequirement{Key: corev1.LabelTopologyZone, Operator: corev1.NodeSelectorOpIn, Values: []string{zone}},
9999
},
100100
Offering: cloudprovider.Offering{
101-
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
102-
Price: lo.Ternary(ct == v1.CapacityTypeSpot, price*.7, price),
101+
Available: true,
102+
Price: lo.Ternary(ct == v1.CapacityTypeSpot, price*.7, price),
103103
},
104104
})
105105
}

pkg/apis/crds/karpenter.sh_nodeclaims.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
33
kind: CustomResourceDefinition
44
metadata:
55
annotations:
6-
controller-gen.kubebuilder.io/version: v0.17.1
6+
controller-gen.kubebuilder.io/version: v0.16.5
77
name: nodeclaims.karpenter.sh
88
spec:
99
group: karpenter.sh

pkg/apis/crds/karpenter.sh_nodepools.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
33
kind: CustomResourceDefinition
44
metadata:
55
annotations:
6-
controller-gen.kubebuilder.io/version: v0.17.1
6+
controller-gen.kubebuilder.io/version: v0.16.5
77
name: nodepools.karpenter.sh
88
spec:
99
group: karpenter.sh

pkg/cloudprovider/fake/instancetype.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -66,39 +66,39 @@ func NewInstanceTypeWithCustomRequirement(options InstanceTypeOptions, customReq
6666
if len(options.Offerings) == 0 {
6767
options.Offerings = []cloudprovider.Offering{
6868
{
69-
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
69+
Available: true,
7070
Requirements: scheduling.NewLabelRequirements(map[string]string{
7171
v1.CapacityTypeLabelKey: "spot",
7272
corev1.LabelTopologyZone: "test-zone-1",
7373
}),
7474
Price: PriceFromResources(options.Resources),
7575
},
7676
{
77-
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
77+
Available: true,
7878
Requirements: scheduling.NewLabelRequirements(map[string]string{
7979
v1.CapacityTypeLabelKey: "spot",
8080
corev1.LabelTopologyZone: "test-zone-2",
8181
}),
8282
Price: PriceFromResources(options.Resources),
8383
},
8484
{
85-
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
85+
Available: true,
8686
Requirements: scheduling.NewLabelRequirements(map[string]string{
8787
v1.CapacityTypeLabelKey: "on-demand",
8888
corev1.LabelTopologyZone: "test-zone-1",
8989
}),
9090
Price: PriceFromResources(options.Resources),
9191
},
9292
{
93-
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
93+
Available: true,
9494
Requirements: scheduling.NewLabelRequirements(map[string]string{
9595
v1.CapacityTypeLabelKey: "on-demand",
9696
corev1.LabelTopologyZone: "test-zone-2",
9797
}),
9898
Price: PriceFromResources(options.Resources),
9999
},
100100
{
101-
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
101+
Available: true,
102102
Requirements: scheduling.NewLabelRequirements(map[string]string{
103103
v1.CapacityTypeLabelKey: "on-demand",
104104
corev1.LabelTopologyZone: "test-zone-3",
@@ -173,7 +173,7 @@ func InstanceTypesAssorted() []*cloudprovider.InstanceType {
173173
price := PriceFromResources(opts.Resources)
174174
opts.Offerings = []cloudprovider.Offering{
175175
{
176-
OfferingAvailabilityResolver: cloudprovider.TrueStaticAvailabilityResolver,
176+
Available: true,
177177
Requirements: scheduling.NewLabelRequirements(map[string]string{
178178
v1.CapacityTypeLabelKey: ct,
179179
corev1.LabelTopologyZone: zone,

pkg/cloudprovider/types.go

+43-92
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,6 @@ import (
3838
var (
3939
SpotRequirement = scheduling.NewRequirements(scheduling.NewRequirement(v1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, v1.CapacityTypeSpot))
4040
OnDemandRequirement = scheduling.NewRequirements(scheduling.NewRequirement(v1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, v1.CapacityTypeOnDemand))
41-
42-
TrueStaticAvailabilityResolver OfferingAvailabilityResolver = staticAvailabilityResolver{available: true}
43-
FalseStaticAvailabilityResolver OfferingAvailabilityResolver = staticAvailabilityResolver{available: false}
4441
)
4542

4643
type DriftReason string
@@ -249,81 +246,74 @@ func (i InstanceTypeOverhead) Total() corev1.ResourceList {
249246
return resources.Merge(i.KubeReserved, i.SystemReserved, i.EvictionThreshold)
250247
}
251248

252-
// An OfferingAvailabilityResolver is used to determine if there is available capacity for a given offering. To ensure
253-
// consistency between multiple controllers attempting to provision a NodeClaim with a given offering, offerings should
254-
// be "reserved" by the controller. Once a launch decision has been made, all offerings which were reserved may be
255-
// released, enabling their use once again.
256-
type OfferingAvailabilityResolver interface {
257-
Available() bool
249+
// ReservationManager is used to track the availability of a reserved offering over the course of a scheduling
250+
// simulation. Reserved offerings may have a limited number of available instances associated with them,
251+
// This is exposed as an interface for cloudprovider's to implement to give flexibility when dealing with separate
252+
// offerings with associated availablility.
253+
type ReservationManager interface {
254+
// Reserve takes a unique identifier for a reservation, and returns a boolean indicating if the reservation was
255+
// successful. Reserve should be idempotent, i.e. multiple calls with the same reservation ID should only count for a
256+
// single reservation.
258257
Reserve(string) bool
259-
GetReservation(string) OfferingReservation
260-
}
261-
262-
type OfferingReservation interface {
263-
Release()
264-
Commit()
265-
Matches(*v1.NodeClaim) bool
266-
}
267-
268-
type OfferingReservations []OfferingReservation
269-
270-
func (r OfferingReservations) Commit() {
271-
for _, reservation := range r {
272-
reservation.Commit()
273-
}
274-
}
275-
276-
func (r OfferingReservations) Release() {
277-
for _, reservation := range r {
278-
reservation.Release()
279-
}
280-
}
281-
282-
func (r OfferingReservations) Matching(nc *v1.NodeClaim) OfferingReservations {
283-
return lo.Filter(r, func(reservation OfferingReservation, _ int) bool {
284-
return reservation.Matches(nc)
285-
})
258+
// Release takes a unique identifier for a reservation, and should discard any matching reservations. If no
259+
// reservations exist for the given id, release should be a no-op.
260+
Release(string)
286261
}
287262

288-
289263
// An Offering describes where an InstanceType is available to be used, with the expectation that its properties
290264
// may be tightly coupled (e.g. the availability of an instance type in some zone is scoped to a capacity type) and
291265
// these properties are captured with labels in Requirements.
292-
// Requirements are required to contain the keys v1.CapacityTypeLabelKey and corev1.LabelTopologyZone
266+
// Requirements are required to contain the keys v1.CapacityTypeLabelKey and corev1.LabelTopologyZone.
293267
type Offering struct {
294-
OfferingAvailabilityResolver
268+
// ReservationManager is used for tracking availabity of reserved offerings over the course of a scheduling loop. It
269+
// must be non-nil for offerings with capacity type "reserved", but may be nil otherwise.
270+
ReservationManager
295271

296272
Requirements scheduling.Requirements
297273
Price float64
274+
Available bool
298275
}
299276

300277
type Offerings []Offering
301278

302-
// Reserve attempts to make a reservation for each offering, returning true if it was successful for any.
303-
func (ofs Offerings) Reserve(id string) bool {
304-
success := false
305-
for i := range ofs {
306-
success = success || ofs[i].Reserve(id)
307-
}
308-
return success
279+
// WithCapacityType filters the offerings by the provided capacity type.
280+
func (ofs Offerings) WithCapacityType(capacityType string) Offerings {
281+
return lo.Filter(ofs, func(o Offering, _ int) bool {
282+
return o.Requirements.Get(v1.CapacityTypeLabelKey).Any() == capacityType
283+
})
309284
}
310285

311-
func (ofs Offerings) Reservations(id string) OfferingReservations {
312-
return lo.FilterMap(ofs, func(o Offering, _ int) (OfferingReservation, bool) {
313-
if reservation := o.GetReservation(id); reservation != nil {
314-
return reservation, true
315-
}
316-
return nil, false
286+
// Reserve attempts to make a reservation for each offering, returning true if it was successful for any.
287+
func (ofs Offerings) Reserve(id string) Offerings {
288+
return lo.Filter(ofs, func(o Offering, _ int) bool {
289+
return o.Reserve(id)
317290
})
318291
}
319292

293+
func (ofs Offerings) Release(id string) {
294+
for i := range ofs {
295+
ofs[i].Release(id)
296+
}
297+
}
298+
320299
// Available filters the available offerings from the returned offerings
321300
func (ofs Offerings) Available() Offerings {
322301
return lo.Filter(ofs, func(o Offering, _ int) bool {
323-
return o.Available()
302+
return o.Available
324303
})
325304
}
326305

306+
func (ofs Offerings) PartitionCompatible(reqs scheduling.Requirements) (compatible Offerings, incompatible Offerings) {
307+
for _, o := range ofs {
308+
if reqs.IsCompatible(o.Requirements, scheduling.AllowUndefinedWellKnownLabels) {
309+
compatible = append(compatible, o)
310+
} else {
311+
incompatible = append(incompatible, o)
312+
}
313+
}
314+
return
315+
}
316+
327317
// Compatible returns the offerings based on the passed requirements
328318
func (ofs Offerings) Compatible(reqs scheduling.Requirements) Offerings {
329319
return lo.Filter(ofs, func(offering Offering, _ int) bool {
@@ -465,42 +455,3 @@ func NewCreateError(err error, reason, message string) *CreateError {
465455
ConditionMessage: message,
466456
}
467457
}
468-
469-
type staticAvailabilityResolver struct {
470-
requirements scheduling.Requirements
471-
available bool
472-
}
473-
474-
type noopReservation struct {
475-
requirements scheduling.Requirements
476-
}
477-
478-
func (r staticAvailabilityResolver) Available() bool {
479-
return r.available
480-
}
481-
482-
func (r staticAvailabilityResolver) Reserve(_ string) bool {
483-
return r.available
484-
}
485-
486-
func (r staticAvailabilityResolver) GetReservation(_ string) OfferingReservation {
487-
return noopReservation{
488-
requirements: r.requirements,
489-
}
490-
}
491-
492-
func (r noopReservation) Commit() {}
493-
494-
func (r noopReservation) Release() {}
495-
496-
func (r noopReservation) Matches(nc *v1.NodeClaim) bool {
497-
reqs := scheduling.NewLabelRequirements(nc.Labels)
498-
return reqs.IsCompatible(r.requirements, scheduling.AllowUndefinedWellKnownLabels)
499-
}
500-
501-
func NewStaticAvailabilityResolver(available bool, requirements scheduling.Requirements) OfferingAvailabilityResolver {
502-
return staticAvailabilityResolver{
503-
available: available,
504-
requirements: requirements,
505-
}
506-
}

0 commit comments

Comments
 (0)