Skip to content

Commit 76b74c2

Browse files
radito3radito3
authored
Improve resiliency and add more logging (#15)
* Improve resiliency * Add some more logging in http executor * Make the connection retries non-blocking * Add more logging * Add more configurable retrier * Improve retrier
1 parent daa200c commit 76b74c2

21 files changed

+252
-63
lines changed

‎cmd/remote-work-processor/main.go

+7-14
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/SAP/remote-work-processor/internal/kubernetes/controller"
2626
meta "github.com/SAP/remote-work-processor/internal/kubernetes/metadata"
2727
"github.com/SAP/remote-work-processor/internal/opt"
28+
"github.com/SAP/remote-work-processor/internal/utils"
2829
"k8s.io/apimachinery/pkg/runtime"
2930
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
3031
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -81,23 +82,23 @@ func main() {
8182

8283
connAttemptChan := make(chan struct{}, 1)
8384
connAttemptChan <- struct{}{}
84-
var connAttempts uint = 0
85+
retryConfig := utils.CreateRetryConfig(opts.RetryInterval, opts.RetryStrategy.Unmarshall(), opts.MaxConnRetries, connAttemptChan)
8586

8687
Loop:
87-
for connAttempts < opts.MaxConnRetries {
88+
for retryConfig.CanRetry() {
8889
select {
8990
case <-rootCtx.Done():
9091
log.Println("Received cancellation signal. Stopping Remote Work Processor...")
9192
break Loop
9293
case <-connAttemptChan:
9394
err := grpcClient.InitSession(rootCtx, rwpMetadata.SessionID())
9495
if err != nil {
95-
signalRetry(&connAttempts, connAttemptChan, err)
96+
utils.Retry(rootCtx, retryConfig, err)
9697
}
9798
default:
9899
operation, err := grpcClient.ReceiveMsg()
99100
if err != nil {
100-
signalRetry(&connAttempts, connAttemptChan, err)
101+
utils.Retry(rootCtx, retryConfig, err)
101102
continue
102103
}
103104
if operation == nil {
@@ -116,15 +117,15 @@ Loop:
116117

117118
msg, err := processor.Process(rootCtx)
118119
if err != nil {
119-
signalRetry(&connAttempts, connAttemptChan, fmt.Errorf("error processing operation: %v", err))
120+
utils.Retry(rootCtx, retryConfig, fmt.Errorf("error processing operation: %v", err))
120121
continue
121122
}
122123
if msg == nil {
123124
continue
124125
}
125126

126127
if err = grpcClient.Send(msg); err != nil {
127-
signalRetry(&connAttempts, connAttemptChan, err)
128+
utils.Retry(rootCtx, retryConfig, err)
128129
}
129130
}
130131
}
@@ -154,11 +155,3 @@ func getKubeConfig() *rest.Config {
154155
}
155156
return config
156157
}
157-
158-
func signalRetry(attempts *uint, retryChan chan<- struct{}, err error) {
159-
if err != nil {
160-
log.Println(err)
161-
}
162-
retryChan <- struct{}{}
163-
*attempts++
164-
}

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/SAP/remote-work-processor
33
go 1.20
44

55
require (
6+
github.com/google/uuid v1.3.0
67
github.com/itchyny/gojq v0.12.12
78
google.golang.org/grpc v1.58.3
89
google.golang.org/protobuf v1.33.0
@@ -29,7 +30,6 @@ require (
2930
github.com/google/gnostic v0.5.7-v3refs // indirect
3031
github.com/google/go-cmp v0.5.9 // indirect
3132
github.com/google/gofuzz v1.1.0 // indirect
32-
github.com/google/uuid v1.3.0 // indirect
3333
github.com/imdario/mergo v0.3.12 // indirect
3434
github.com/itchyny/timefmt-go v0.1.5 // indirect
3535
github.com/josharian/intern v1.0.0 // indirect

‎internal/executors/executor_result.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ type ExecutorResult struct {
1111
type ExecutorResultOption func(*ExecutorResult)
1212

1313
func NewExecutorResult(opts ...ExecutorResultOption) *ExecutorResult {
14-
r := &ExecutorResult{}
14+
r := &ExecutorResult{
15+
Output: make(map[string]string),
16+
}
1517

1618
for _, opt := range opts {
1719
opt(r)
@@ -22,7 +24,9 @@ func NewExecutorResult(opts ...ExecutorResultOption) *ExecutorResult {
2224

2325
func Output(m map[string]string) ExecutorResultOption {
2426
return func(er *ExecutorResult) {
25-
er.Output = m
27+
for key, val := range m {
28+
er.Output[key] = val
29+
}
2630
}
2731
}
2832

@@ -34,11 +38,9 @@ func Status(s pb.TaskExecutionResponseMessage_TaskState) ExecutorResultOption {
3438

3539
func Error(err error) ExecutorResultOption {
3640
return func(er *ExecutorResult) {
37-
if err == nil {
38-
return
41+
if err != nil {
42+
er.Error = err.Error()
3943
}
40-
41-
er.Error = err.Error()
4244
}
4345
}
4446

‎internal/executors/factory/executor_factory.go

+15-4
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,24 @@ import (
88
"github.com/SAP/remote-work-processor/internal/executors/void"
99
)
1010

11-
func CreateExecutor(t pb.TaskType) (executors.Executor, error) {
11+
type errorExecutor func() error
12+
13+
func CreateExecutor(t pb.TaskType) executors.Executor {
1214
switch t {
1315
case pb.TaskType_TASK_TYPE_VOID:
14-
return void.VoidExecutor{}, nil
16+
return void.VoidExecutor{}
1517
case pb.TaskType_TASK_TYPE_HTTP:
16-
return http.NewDefaultHttpRequestExecutor(), nil
18+
return http.NewDefaultHttpRequestExecutor()
1719
default:
18-
return nil, fmt.Errorf("cannot create executor of type %q", t)
20+
return errorExecutor(func() error {
21+
return fmt.Errorf("cannot create executor of type %q: unsupported task type", t)
22+
})
1923
}
2024
}
25+
26+
func (exec errorExecutor) Execute(_ executors.Context) *executors.ExecutorResult {
27+
return executors.NewExecutorResult(
28+
executors.Status(pb.TaskExecutionResponseMessage_TASK_STATE_FAILED_NON_CHARGEABLE),
29+
executors.Error(exec()),
30+
)
31+
}

‎internal/executors/http/authorization_header.go

+7
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package http
22

33
import (
44
"github.com/SAP/remote-work-processor/internal/executors"
5+
"log"
56
"regexp"
67
)
78

@@ -16,6 +17,7 @@ var iasTokenUrlRegex = regexp.MustCompile(IasTokenUrlPattern)
1617
// OAuth 2.0 will be added later
1718

1819
func CreateAuthorizationHeader(params *HttpRequestParameters) (string, error) {
20+
log.Println("HTTP Client: creating authorization header...")
1921
authHeader := params.GetAuthorizationHeader()
2022

2123
if authHeader != "" {
@@ -28,19 +30,24 @@ func CreateAuthorizationHeader(params *HttpRequestParameters) (string, error) {
2830

2931
if tokenUrl != "" {
3032
if user != "" && iasTokenUrlRegex.Match([]byte(tokenUrl)) {
33+
log.Println("HTTP Client: using IAS Authorization Header...")
3134
return NewIasAuthorizationHeader(tokenUrl, user, params.GetCertificateAuthentication().GetClientCertificate()).Generate()
3235
}
36+
log.Println("HTTP Client: using OAuth Authorization Header...")
3337
return NewOAuthHeaderGenerator(params).GenerateWithCacheAside()
3438
}
3539

3640
if user != "" {
41+
log.Println("HTTP Client: using basic auth...")
3742
return NewBasicAuthorizationHeader(user, pass).Generate()
3843
}
3944

4045
if noAuthorizationRequired(params) {
46+
log.Println("HTTP Client: not using authorization...")
4147
return "", nil
4248
}
4349

50+
log.Println("HTTP Client: failed to determine auth header...")
4451
return "", executors.NewNonRetryableError("Input values for the authentication-related keys " +
4552
"(user, password & authorizationHeader) are not combined properly.")
4653
}

‎internal/executors/http/basic_authorization_header.go

+2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package http
33
import (
44
"encoding/base64"
55
"fmt"
6+
"log"
67
)
78

89
type basicAuthorizationHeader struct {
@@ -18,6 +19,7 @@ func NewBasicAuthorizationHeader(u string, p string) AuthorizationHeaderGenerato
1819
}
1920

2021
func (h *basicAuthorizationHeader) Generate() (string, error) {
22+
log.Println("Basic Authorization Header: generating auth header...")
2123
encoded := base64.StdEncoding.EncodeToString(
2224
fmt.Appendf(nil, "%s:%s", h.username, h.password),
2325
)

‎internal/executors/http/csrf_token_fetcher.go

+5
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package http
33
import (
44
"fmt"
55
"github.com/SAP/remote-work-processor/internal/utils"
6+
"log"
67
"net/http"
78
)
89

@@ -27,10 +28,12 @@ func NewCsrfTokenFetcher(p *HttpRequestParameters, authHeader string) TokenFetch
2728
}
2829

2930
func (f *csrfTokenFetcher) Fetch() (string, error) {
31+
log.Println("CSRF token fetcher: fetching new CSRF token from:", f.csrfUrl)
3032
params, _ := f.createRequestParameters()
3133

3234
resp, err := f.HttpExecutor.ExecuteWithParameters(params)
3335
if err != nil {
36+
log.Println("CSRF token fetcher: failed to fetch CSRF token:", err)
3437
return "", err
3538
}
3639

@@ -39,6 +42,8 @@ func (f *csrfTokenFetcher) Fetch() (string, error) {
3942
return value, nil
4043
}
4144
}
45+
46+
log.Println("CSRF token fetcher: CSRF token header not found in response")
4247
return "", fmt.Errorf("no csrf header present in response from %s", f.csrfUrl)
4348
}
4449

‎internal/executors/http/http_client.go

+4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package http
22

33
import (
4+
"log"
45
"net/http"
56
"time"
67

@@ -12,10 +13,12 @@ const (
1213
)
1314

1415
func CreateHttpClient(timeoutInS uint64, certAuth *tls.CertificateAuthentication) (*http.Client, error) {
16+
log.Println("HTTP Client: creating HTTP Client...")
1517
var tp http.RoundTripper
1618
if certAuth != nil {
1719
var err error
1820

21+
log.Println("HTTP Client: creating TLS transport...")
1922
tp, err = tls.NewTLSConfigurationProvider(certAuth).CreateTransport()
2023
if err != nil {
2124
return nil, err
@@ -32,6 +35,7 @@ func CreateHttpClient(timeoutInS uint64, certAuth *tls.CertificateAuthentication
3235
} else {
3336
c.Timeout = time.Duration(timeoutInS) * time.Second
3437
}
38+
log.Println("HTTP Client: using timeout:", c.Timeout.String())
3539

3640
return c, nil
3741
}

‎internal/executors/http/http_executor.go

+18-3
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func (e *HttpRequestExecutor) Execute(ctx executors.Context) *executors.Executor
3232
log.Println("Executing HttpRequest command...")
3333
params, err := NewHttpRequestParametersFromContext(ctx)
3434
if err != nil {
35+
log.Println("Could not create HTTP request params: returning Task state Failed Non-Retryable Error with:", err)
3536
return executors.NewExecutorResult(
3637
executors.Status(pb.TaskExecutionResponseMessage_TASK_STATE_FAILED_NON_RETRYABLE),
3738
executors.Error(err),
@@ -42,25 +43,29 @@ func (e *HttpRequestExecutor) Execute(ctx executors.Context) *executors.Executor
4243

4344
switch typedErr := err.(type) {
4445
case *executors.RetryableError:
46+
log.Println("Returning Task state Failed Retryable Error...")
4547
return executors.NewExecutorResult(
4648
executors.Status(pb.TaskExecutionResponseMessage_TASK_STATE_FAILED_RETRYABLE),
4749
executors.Error(typedErr),
4850
)
4951
case *executors.NonRetryableError:
52+
log.Println("Returning Task state Failed Non-Retryable Error...")
5053
return executors.NewExecutorResult(
5154
executors.Status(pb.TaskExecutionResponseMessage_TASK_STATE_FAILED_NON_RETRYABLE),
5255
executors.Error(typedErr),
5356
)
5457
default:
5558
m := resp.ToMap()
5659
if !resp.successful {
60+
log.Println("Returning Task state Failed Retryable Error from HTTP response...")
5761
return executors.NewExecutorResult(
5862
executors.Output(m),
5963
executors.Status(pb.TaskExecutionResponseMessage_TASK_STATE_FAILED_RETRYABLE),
6064
executors.ErrorString(buildHttpError(resp)),
6165
)
6266
}
6367

68+
log.Println("Returning Task state Completed...")
6469
return executors.NewExecutorResult(
6570
executors.Output(m),
6671
executors.Status(pb.TaskExecutionResponseMessage_TASK_STATE_COMPLETED),
@@ -104,26 +109,34 @@ func execute(c *http.Client, p *HttpRequestParameters, authHeader string) (*Http
104109
return nil, executors.NewNonRetryableError("could not create http request: %v", err).WithCause(err)
105110
}
106111

107-
log.Printf("Executing request %s %s...\n", p.method, p.url)
112+
log.Printf("HTTP Client: executing request %s %s...\n", p.method, p.url)
108113
resp, err := c.Do(req)
109114
if requestTimedOut(err) {
115+
log.Println("HTTP Client: request timed out after", p.timeout, "seconds")
110116
if p.succeedOnTimeout {
117+
log.Println("HTTP Client: SucceedOnTimeout has been configured. Returning successful response...")
111118
return newTimedOutHttpResponse(req, resp)
112119
}
113120

114121
return nil, executors.NewRetryableError("HTTP request timed out after %d seconds", p.timeout).WithCause(err)
115122
}
116123

117124
if err != nil {
118-
return nil, executors.NewNonRetryableError("Error occurred while trying to execute actual HTTP request: %v", err).WithCause(err)
125+
log.Println("HTTP Client: error occurred while executing request:", err)
126+
return nil, executors.NewNonRetryableError("Error occurred while executing HTTP request: %v", err).WithCause(err)
119127
}
120128
defer resp.Body.Close()
121129

130+
log.Println("HTTP Client: received response:", resp.Status)
131+
132+
log.Println("HTTP Client: reading response body...")
122133
body, err := io.ReadAll(resp.Body)
123134
if err != nil {
135+
log.Println("HTTP Client: error reading response body:", err)
124136
return nil, executors.NewNonRetryableError("Error occurred while trying to read HTTP response body: %v", err).WithCause(err)
125137
}
126138

139+
log.Println("HTTP Client: building response object...")
127140
r, err := NewHttpResponse(
128141
Url(req.URL.String()),
129142
Method(req.Method),
@@ -135,9 +148,9 @@ func execute(c *http.Client, p *HttpRequestParameters, authHeader string) (*Http
135148
Time(<-timeCh),
136149
)
137150
if err != nil {
151+
log.Println("HTTP Client: could not build response object:", err)
138152
return nil, executors.NewNonRetryableError("Error occurred while trying to build HTTP response: %v", err).WithCause(err)
139153
}
140-
141154
return r, nil
142155
}
143156

@@ -147,10 +160,12 @@ func requestTimedOut(err error) bool {
147160
}
148161

149162
func createRequest(method string, url string, headers map[string]string, body, authHeader string) (*http.Request, <-chan int64, error) {
163+
log.Println("HTTP Client: creating request:", method, url)
150164
timeCh := make(chan int64, 1)
151165

152166
req, err := http.NewRequest(method, url, strings.NewReader(body))
153167
if err != nil {
168+
log.Println("HTTP Client: error creating request:", err)
154169
return nil, nil, err
155170
}
156171
addHeaders(req, headers, authHeader)

‎internal/executors/http/ias_authorization_header.go

+4
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package http
33
import (
44
"fmt"
55
"github.com/SAP/remote-work-processor/internal/utils"
6+
"log"
67
)
78

89
const PASSCODE string = "passcode"
@@ -27,13 +28,16 @@ func (h *iasAuthorizationHeader) Generate() (string, error) {
2728

2829
parsed := make(map[string]any)
2930
if err = utils.FromJson(raw, &parsed); err != nil {
31+
log.Println("IAS authorization header: failed to parse IAS token response:", err)
3032
return "", fmt.Errorf("failed to parse IAS token response: %v", err)
3133
}
3234

3335
pass, prs := parsed[PASSCODE]
3436
if !prs {
37+
log.Println("IAS authorization header: passcode does not exist in the HTTP response")
3538
return "", fmt.Errorf("passcode does not exist in the HTTP response")
3639
}
3740

41+
log.Println("IAS authorization header: using basic auth with passcode...")
3842
return NewBasicAuthorizationHeader(h.user, pass.(string)).Generate()
3943
}

0 commit comments

Comments
 (0)