Skip to content

Commit 77a05e5

Browse files
dweomerdaxmc99
andauthored
implement ttlSecondsAfterFinished for jobs (#48)
* implement ttlSecondsAfterFinished for jobs - bump default backofflimit to the default for the job controller, aka 6 Addresses #24 by implementing cleanup of jobs based on configured .spec.ttlSecondsAfterFinished on jobs. This should work pretty almost identically to the by-default, gated TTL Controll for Finished Resources. See https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ * Update pkg/upgrade/handle_batch.go Co-authored-by: Dax McDonald <[email protected]>
1 parent 8b629ea commit 77a05e5

File tree

10 files changed

+339
-22
lines changed

10 files changed

+339
-22
lines changed

e2e/framework/framework.go

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ import (
99
"github.com/rancher/system-upgrade-controller/e2e/framework/controller"
1010
upgradeapi "github.com/rancher/system-upgrade-controller/pkg/apis/upgrade.cattle.io"
1111
upgradeapiv1 "github.com/rancher/system-upgrade-controller/pkg/apis/upgrade.cattle.io/v1"
12+
"github.com/rancher/system-upgrade-controller/pkg/condition"
1213
upgradecln "github.com/rancher/system-upgrade-controller/pkg/generated/clientset/versioned"
1314
upgradescheme "github.com/rancher/system-upgrade-controller/pkg/generated/clientset/versioned/scheme"
14-
"github.com/rancher/wrangler/pkg/condition"
15+
upgradejob "github.com/rancher/system-upgrade-controller/pkg/upgrade/job"
1516
appsv1 "k8s.io/api/apps/v1"
1617
batchv1 "k8s.io/api/batch/v1"
1718
corev1 "k8s.io/api/core/v1"
@@ -130,9 +131,6 @@ func (c *Client) WaitForPlanCondition(name string, cond condition.Cond, timeout
130131
}
131132

132133
func (c *Client) WaitForPlanJobs(plan *upgradeapiv1.Plan, count int, timeout time.Duration) (jobs []batchv1.Job, err error) {
133-
complete := condition.Cond(batchv1.JobComplete)
134-
failed := condition.Cond(batchv1.JobFailed)
135-
136134
labelSelector := labels.SelectorFromSet(labels.Set{
137135
upgradeapi.LabelPlan: plan.Name,
138136
})
@@ -145,7 +143,7 @@ func (c *Client) WaitForPlanJobs(plan *upgradeapiv1.Plan, count int, timeout tim
145143
return false, err
146144
}
147145
for _, item := range list.Items {
148-
if failed.IsTrue(&item) || complete.IsTrue(&item) {
146+
if upgradejob.ConditionFailed.IsTrue(&item) || upgradejob.ConditionComplete.IsTrue(&item) {
149147
jobs = append(jobs, item)
150148
}
151149
}

go.sum

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -599,8 +599,6 @@ github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.2-
599599
github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.2-k3s1/go.mod h1:e70DTGI+y72YcU8iGDifilyP5uW6sPmBflDEjtiH/p0=
600600
github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.17.2-k3s1/go.mod h1:Ey49UYt1h2yQl0P61nfZEo/LZUu78Bj55oBL/VOejW4=
601601
github.com/rancher/wrangler v0.2.1-0.20191025041946-1fd360590735/go.mod h1:zmXTOSzU0vhumCyl0+Acq2FEP5WJOJRqnCQDegknyWA=
602-
github.com/rancher/wrangler v0.4.2-0.20200225192203-e5307acd8846 h1:jiA0wj/6QAIW3w1bSC0jKHzRVA2lxybR9bgj2wFfPE8=
603-
github.com/rancher/wrangler v0.4.2-0.20200225192203-e5307acd8846/go.mod h1:txHSBkPtVgNH/0pUCvdP0Ak0HptAOc9ffBmFxQnL4z4=
604602
github.com/rancher/wrangler v0.5.0 h1:zTchAfY9DzchLvXpRpQuNB0PbNfl/HSuvFL1wHN6mDU=
605603
github.com/rancher/wrangler v0.5.0/go.mod h1:txHSBkPtVgNH/0pUCvdP0Ak0HptAOc9ffBmFxQnL4z4=
606604
github.com/rancher/wrangler-api v0.2.1-0.20191025043713-b1ca9c21825a h1:dH3D71jzKuYRY7qLe9AKuwz8DKO6G9PCfkeafETMmZo=

manifests/system-upgrade-controller.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,11 @@ data:
3232
SYSTEM_UPGRADE_CONTROLLER_DEBUG: "false"
3333
SYSTEM_UPGRADE_CONTROLLER_THREADS: "2"
3434
SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: "900"
35-
SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "2"
35+
SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "6"
3636
SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: "IfNotPresent"
3737
SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: "rancher/kubectl:v1.17.0"
3838
SYSTEM_UPGRADE_JOB_PRIVILEGED: "true"
39+
SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: "900"
3940
SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: "15m"
4041
---
4142
apiVersion: apps/v1

pkg/apis/upgrade.cattle.io/constants.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ const (
1212
// LabelPlan is the plan being applied.
1313
LabelPlan = GroupName + `/plan`
1414

15-
// LabelPlan is the version being applied.
15+
// LabelVersion is the version of the plan being applied.
1616
LabelVersion = GroupName + `/version`
1717

1818
// LabelPlanSuffix is used for composing labels specific to a plan.

pkg/apis/upgrade.cattle.io/v1/types.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ package v1
66
import (
77
"time"
88

9-
"github.com/rancher/wrangler/pkg/condition"
9+
"github.com/rancher/system-upgrade-controller/pkg/condition"
1010
"github.com/rancher/wrangler/pkg/genericcondition"
1111
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1212
)

pkg/condition/condition.go

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
package condition
2+
3+
// adapted from rancher/wrangler
4+
5+
import (
6+
"reflect"
7+
"time"
8+
9+
"github.com/sirupsen/logrus"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
)
12+
13+
type Cond string
14+
15+
func (c Cond) GetStatus(obj interface{}) string {
16+
return getStatus(obj, string(c))
17+
}
18+
19+
func (c Cond) SetError(obj interface{}, reason string, err error) {
20+
if err == nil {
21+
c.True(obj)
22+
c.Message(obj, "")
23+
c.Reason(obj, reason)
24+
return
25+
}
26+
if reason == "" {
27+
reason = "Error"
28+
}
29+
c.False(obj)
30+
c.Message(obj, err.Error())
31+
c.Reason(obj, reason)
32+
}
33+
34+
func (c Cond) MatchesError(obj interface{}, reason string, err error) bool {
35+
if err == nil {
36+
return c.IsTrue(obj) &&
37+
c.GetMessage(obj) == "" &&
38+
c.GetReason(obj) == reason
39+
}
40+
if reason == "" {
41+
reason = "Error"
42+
}
43+
return c.IsFalse(obj) &&
44+
c.GetMessage(obj) == err.Error() &&
45+
c.GetReason(obj) == reason
46+
}
47+
48+
func (c Cond) SetStatus(obj interface{}, status string) {
49+
setStatus(obj, string(c), status)
50+
}
51+
52+
func (c Cond) SetStatusBool(obj interface{}, val bool) {
53+
if val {
54+
setStatus(obj, string(c), "True")
55+
} else {
56+
setStatus(obj, string(c), "False")
57+
}
58+
}
59+
60+
func (c Cond) True(obj interface{}) {
61+
setStatus(obj, string(c), "True")
62+
}
63+
64+
func (c Cond) IsTrue(obj interface{}) bool {
65+
return getStatus(obj, string(c)) == "True"
66+
}
67+
68+
func (c Cond) False(obj interface{}) {
69+
setStatus(obj, string(c), "False")
70+
}
71+
72+
func (c Cond) IsFalse(obj interface{}) bool {
73+
return getStatus(obj, string(c)) == "False"
74+
}
75+
76+
func (c Cond) Unknown(obj interface{}) {
77+
setStatus(obj, string(c), "Unknown")
78+
}
79+
80+
func (c Cond) IsUnknown(obj interface{}) bool {
81+
return getStatus(obj, string(c)) == "Unknown"
82+
}
83+
84+
func (c Cond) LastUpdated(obj interface{}, ts string) {
85+
setTS(obj, string(c), ts)
86+
}
87+
88+
func (c Cond) GetLastUpdated(obj interface{}) string {
89+
return getTS(obj, string(c))
90+
}
91+
92+
func (c Cond) GetLastTransitionTime(obj interface{}) time.Time {
93+
return getLastTransitionTime(obj, string(c))
94+
}
95+
96+
func (c Cond) CreateUnknownIfNotExists(obj interface{}) {
97+
condSlice := getValue(obj, "Status", "Conditions")
98+
cond := findCond(obj, condSlice, string(c))
99+
if cond == nil {
100+
c.Unknown(obj)
101+
}
102+
}
103+
104+
func (c Cond) Reason(obj interface{}, reason string) {
105+
cond := findOrCreateCond(obj, string(c))
106+
getFieldValue(cond, "Reason").SetString(reason)
107+
}
108+
109+
func (c Cond) GetReason(obj interface{}) string {
110+
cond := findOrNotCreateCond(obj, string(c))
111+
if cond == nil {
112+
return ""
113+
}
114+
return getFieldValue(*cond, "Reason").String()
115+
}
116+
117+
func (c Cond) SetMessageIfBlank(obj interface{}, message string) {
118+
if c.GetMessage(obj) == "" {
119+
c.Message(obj, message)
120+
}
121+
}
122+
123+
func (c Cond) Message(obj interface{}, message string) {
124+
cond := findOrCreateCond(obj, string(c))
125+
setValue(cond, "Message", message)
126+
}
127+
128+
func (c Cond) GetMessage(obj interface{}) string {
129+
cond := findOrNotCreateCond(obj, string(c))
130+
if cond == nil {
131+
return ""
132+
}
133+
return getFieldValue(*cond, "Message").String()
134+
}
135+
136+
func touchTS(value reflect.Value) {
137+
now := time.Now().Format(time.RFC3339)
138+
getFieldValue(value, "LastUpdateTime").SetString(now)
139+
}
140+
141+
func getStatus(obj interface{}, condName string) string {
142+
cond := findOrNotCreateCond(obj, condName)
143+
if cond == nil {
144+
return ""
145+
}
146+
return getFieldValue(*cond, "Status").String()
147+
}
148+
149+
func setTS(obj interface{}, condName, ts string) {
150+
cond := findOrCreateCond(obj, condName)
151+
getFieldValue(cond, "LastUpdateTime").SetString(ts)
152+
}
153+
154+
func getTS(obj interface{}, condName string) string {
155+
cond := findOrNotCreateCond(obj, condName)
156+
if cond == nil {
157+
return ""
158+
}
159+
return getFieldValue(*cond, "LastUpdateTime").String()
160+
}
161+
162+
func getLastTransitionTime(obj interface{}, condName string) time.Time {
163+
cond := findOrNotCreateCond(obj, condName)
164+
if cond == nil {
165+
return time.Time{}
166+
}
167+
value := getFieldValue(*cond, "LastTransitionTime").Interface()
168+
if value == nil {
169+
return time.Time{}
170+
}
171+
return value.(metav1.Time).Time
172+
}
173+
174+
func setStatus(obj interface{}, condName, status string) {
175+
cond := findOrCreateCond(obj, condName)
176+
setValue(cond, "Status", status)
177+
}
178+
179+
func setValue(cond reflect.Value, fieldName, newValue string) {
180+
value := getFieldValue(cond, fieldName)
181+
if value.String() != newValue {
182+
value.SetString(newValue)
183+
touchTS(cond)
184+
}
185+
}
186+
187+
func findOrNotCreateCond(obj interface{}, condName string) *reflect.Value {
188+
condSlice := getValue(obj, "Status", "Conditions")
189+
return findCond(obj, condSlice, condName)
190+
}
191+
192+
func findOrCreateCond(obj interface{}, condName string) reflect.Value {
193+
condSlice := getValue(obj, "Status", "Conditions")
194+
if !condSlice.IsValid() {
195+
condSlice = getValue(obj, "Conditions")
196+
}
197+
cond := findCond(obj, condSlice, condName)
198+
if cond != nil {
199+
return *cond
200+
}
201+
202+
newCond := reflect.New(condSlice.Type().Elem()).Elem()
203+
newCond.FieldByName("Type").SetString(condName)
204+
newCond.FieldByName("Status").SetString("Unknown")
205+
condSlice.Set(reflect.Append(condSlice, newCond))
206+
return *findCond(obj, condSlice, condName)
207+
}
208+
209+
func findCond(obj interface{}, val reflect.Value, name string) *reflect.Value {
210+
defer func() {
211+
if recover() != nil {
212+
logrus.Fatalf("failed to find .Status.Conditions field on %v", reflect.TypeOf(obj))
213+
}
214+
}()
215+
216+
for i := 0; i < val.Len(); i++ {
217+
cond := val.Index(i)
218+
typeVal := getFieldValue(cond, "Type")
219+
if typeVal.String() == name {
220+
return &cond
221+
}
222+
}
223+
224+
return nil
225+
}
226+
227+
func getValue(obj interface{}, name ...string) reflect.Value {
228+
if obj == nil {
229+
return reflect.Value{}
230+
}
231+
v := reflect.ValueOf(obj)
232+
t := v.Type()
233+
if t.Kind() == reflect.Ptr {
234+
v = v.Elem()
235+
t = v.Type()
236+
}
237+
238+
field := v.FieldByName(name[0])
239+
if len(name) == 1 {
240+
return field
241+
}
242+
return getFieldValue(field, name[1:]...)
243+
}
244+
245+
func getFieldValue(v reflect.Value, name ...string) reflect.Value {
246+
field := v.FieldByName(name[0])
247+
if len(name) == 1 {
248+
return field
249+
}
250+
return getFieldValue(field, name[1:]...)
251+
}
252+
253+
func Error(reason string, err error) error {
254+
return &conditionError{
255+
reason: reason,
256+
message: err.Error(),
257+
}
258+
}
259+
260+
type conditionError struct {
261+
reason string
262+
message string
263+
}
264+
265+
func (e *conditionError) Error() string {
266+
return e.message
267+
}

pkg/generated/controllers/upgrade.cattle.io/v1/plan.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)