feat: add retry with exp. backoff to all RPC network calls (#148)

This PR adds retry logic to all RPC network calls. It relies on the
avast/retry-go library, and is configured with the following parameters:

* delays: [0.5s, 2s, 8s, 32s]
* attempts: 5
* timeout of each network call: 30s

In addition, the PR "fixes" the semantic of the
is{Operation,Ready,Pending,Done} methods. Previously, they would return
false in case of a network error -- which naturally let to unexpected
behavior in the core event handling logic. Now, we return (bool, error)
and the calling code properly handles the network failures.

--

Test coverage is low due to (1) the complexity of mocking the failed
calls on a code base that was not designed with a clear separation
between i/o and business logic and (2) the short lifespan of changes to
the timelock-worker service, which should be deprecated soon.

To compensate, I spent a significant amount of time running local manual
tests, trying to validate all failure scenarios.

Author: gustavogama-cll

.mockery.yaml

@@ -0,0 +1,23 @@
+# Mockery configuration file
+
+# All interfaces in a package are generated by default. You can set this to false in the config of
+# of each package to disable this behavior.
+all: false
+with-expecter: true
+# The name of the mock file to be generated is the snake cased interface name.
+filename: "{{.InterfaceName | snakecase | lower}}.go"
+# Creates the mocks in a `mocks` dir inside the package containing the interfaces.
+dir: "{{.InterfaceDir}}/mocks"
+# The name of the mock struct is the interface name.
+mockname: "{{.InterfaceName}}"
+# Places the mock package in the same package as the interface.
+outpkg: "mocks"
+packages:
+  github.com/ethereum/go-ethereum/accounts/abi/bind:
+    interfaces:
+      ContractBackend:
+        config:
+          # inpackage: false
+          filename: "{{.InterfaceName | snakecase}}.go"
+          # mockname: "{{.InterfaceName}}"
+          dir: "./pkg/timelock/mocks"

.tool-versions

@@ -1,3 +1,4 @@
 golang 1.22.9
 golangci-lint 1.62.0
-task 3.40.0
+task 3.40.0
+mockery 2.50.0

go.mod

@@ -3,6 +3,7 @@ module github.com/smartcontractkit/timelock-worker
 go 1.22
 
 require (
+	github.com/avast/retry-go/v4 v4.6.1
 	github.com/docker/go-connections v0.5.0
 	github.com/ethereum/go-ethereum v1.13.15
 	github.com/google/go-cmp v0.6.0
@@ -12,7 +13,7 @@ require (
 	github.com/smartcontractkit/chain-selectors v1.0.17
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/testcontainers/testcontainers-go v0.34.0
 	go.uber.org/zap v1.27.0
 )
@@ -116,6 +117,7 @@ require (
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/status-im/keycard-go v0.2.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/supranational/blst v0.3.11 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect

go.sum

@@ -24,6 +24,8 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk=
+github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -425,8 +427,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=

pkg/timelock/const.go

@@ -13,6 +13,7 @@ const (
 
 	fieldTXHash      string = "TX Hash"
 	fieldBlockNumber string = "Block Number"
+	operationID      string = "Operation ID"
 
 	logPath         string = "/tmp/"
 	logFile         string = "timelock-worker.log"