fix: use exponential backoff for csr approvals (#35)

Author: anjmao

actions/approve_csr_handler.go

@@ -43,7 +43,7 @@ func (h *approveCSRHandler) Handle(ctx context.Context, data interface{}) error
 	log := h.log.WithField("node_name", req.NodeName)
 
 	b := backoff.WithContext(
-		backoff.WithMaxRetries(backoff.NewConstantBackOff(h.retryAfter), h.maxRetries),
+		newApproveCSRExponentialBackoff(),
 		ctx,
 	)
 	return backoff.RetryNotify(func() error {
@@ -60,6 +60,9 @@ func (h *approveCSRHandler) handle(ctx context.Context, log logrus.FieldLogger,
 	log.Debug("getting initial csr")
 	cert, err := h.getInitialNodeCSR(ctx, req.NodeName)
 	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return backoff.Permanent(fmt.Errorf("getting initial csr: %w", err))
+		}
 		return fmt.Errorf("getting initial csr: %w", err)
 	}
 
@@ -116,3 +119,11 @@ func (h *approveCSRHandler) getInitialNodeCSR(ctx context.Context, nodeName stri
 		}
 	}
 }
+
+func newApproveCSRExponentialBackoff() *backoff.ExponentialBackOff {
+	b := backoff.NewExponentialBackOff()
+	b.Multiplier = 2
+	b.MaxElapsedTime = 4 * time.Minute
+	b.Reset()
+	return b
+}

actions/approve_csr_handler_test.go

@@ -23,11 +23,12 @@ import (
 )
 
 func TestApproveCSRHandler(t *testing.T) {
-	r := require.New(t)
 	log := logrus.New()
 	log.SetLevel(logrus.DebugLevel)
 
 	t.Run("approve v1 csr successfully", func(t *testing.T) {
+		r := require.New(t)
+
 		csr := &certv1.CertificateSigningRequest{
 			TypeMeta: metav1.TypeMeta{},
 			ObjectMeta: metav1.ObjectMeta{
@@ -99,6 +100,8 @@ AiAHVYZXHxxspoV0hcfn2Pdsl89fIPCOFy/K1PqSUR6QNAIgYdt51ZbQt9rgM2BD
 	})
 
 	t.Run("approve v1beta1 csr successfully", func(t *testing.T) {
+		r := require.New(t)
+
 		csr := &certv1beta1.CertificateSigningRequest{
 			TypeMeta: metav1.TypeMeta{},
 			ObjectMeta: metav1.ObjectMeta{
@@ -158,6 +161,8 @@ AiAHVYZXHxxspoV0hcfn2Pdsl89fIPCOFy/K1PqSUR6QNAIgYdt51ZbQt9rgM2BD
 	})
 
 	t.Run("return timeout error when no initial csr found for node", func(t *testing.T) {
+		r := require.New(t)
+
 		client := fake.NewSimpleClientset()
 		h := &approveCSRHandler{
 			log:                    log,
@@ -171,3 +176,14 @@ AiAHVYZXHxxspoV0hcfn2Pdsl89fIPCOFy/K1PqSUR6QNAIgYdt51ZbQt9rgM2BD
 		r.EqualError(err, "getting initial csr: context deadline exceeded")
 	})
 }
+
+func TestApproveCSRExponentialBackoff(t *testing.T) {
+	r := require.New(t)
+	b := newApproveCSRExponentialBackoff()
+	var sum time.Duration
+	for i := 0; i < 10; i++ {
+		tmp := b.NextBackOff()
+		sum += tmp
+	}
+	r.Truef(100 < sum.Seconds(), "actual elapsed seconds %s", sum.Seconds())
+}

go.mod

@@ -138,7 +138,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
 	k8s.io/apiextensions-apiserver v0.23.5 // indirect
 	k8s.io/component-base v0.23.5 // indirect
 	k8s.io/klog/v2 v2.30.0 // indirect

go.sum

@@ -1785,8 +1785,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=