Skip to content

Commit 9bd8933

Browse files
authored
Merge pull request #27 from easyops-cn/fal/api_retry
微服务调用重试
2 parents a605ff5 + 8d159f7 commit 9bd8933

File tree

10 files changed

+525
-29
lines changed

10 files changed

+525
-29
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.idea

codes/code.pb.go

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
module github.com/easyops-cn/giraffe-micro
22

3-
go 1.13
3+
go 1.17
44

55
require (
66
github.com/easyops-cn/go-proto-giraffe v0.2.0
77
github.com/go-test/deep v1.0.7
8-
github.com/gogo/protobuf v1.3.1
8+
github.com/gogo/protobuf v1.3.2
99
)

go.sum

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,36 @@ github.com/easyops-cn/go-proto-giraffe v0.2.0/go.mod h1:T0Re9wVBu/7qUs/u0NEsmw1q
33
github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
44
github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
55
github.com/gogo/protobuf v0.0.0-20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
6-
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
7-
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
6+
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
7+
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
88
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
9+
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
910
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
11+
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
12+
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
13+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
14+
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
15+
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
16+
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
17+
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
18+
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
19+
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
20+
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
21+
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
22+
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
23+
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
24+
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
25+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
26+
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
27+
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
28+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
29+
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
30+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
1031
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
32+
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
33+
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
34+
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
35+
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
36+
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
37+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
38+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

plugins/restv2/client.go

Lines changed: 101 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,32 @@ import (
66
"io"
77
"io/ioutil"
88
"net/http"
9+
"sort"
10+
"strconv"
11+
"syscall"
12+
"time"
913

1014
"github.com/easyops-cn/giraffe-micro"
1115
)
1216

13-
//Middleware 中间件定义
17+
// Middleware 中间件定义
1418
type Middleware interface {
1519
NewRequest(rule giraffe.HttpRule, in interface{}) (*http.Request, error)
1620
ParseResponse(rule giraffe.HttpRule, resp *http.Response, out interface{}) error
1721
}
1822

19-
//Client REST Client对象
23+
// Client REST Client对象
2024
type Client struct {
2125
*http.Client
2226
Middleware Middleware
2327
NameService giraffe.NameService
28+
retryConf RetryConfig
2429
}
2530

26-
//ClientOption Client 配置函数
31+
// ClientOption Client 配置函数
2732
type ClientOption func(c *Client)
2833

29-
//Invoke 单次请求方法
34+
// Invoke 单次请求方法
3035
func (c *Client) Invoke(ctx context.Context, md *giraffe.MethodDesc, in interface{}, out interface{}, opts ...giraffe.CallOption) error {
3136
req, err := c.middleware().NewRequest(md.HttpRule, in)
3237
if err != nil {
@@ -50,7 +55,7 @@ func (c *Client) Invoke(ctx context.Context, md *giraffe.MethodDesc, in interfac
5055
return c.middleware().ParseResponse(md.HttpRule, resp, out)
5156
}
5257

53-
//NewStream 流式请求方法(未实现)
58+
// NewStream 流式请求方法(未实现)
5459
func (c *Client) NewStream(context.Context, *giraffe.StreamDesc, ...giraffe.CallOption) (giraffe.ClientStream, error) {
5560
return nil, errors.New("not supported")
5661
}
@@ -69,7 +74,7 @@ func (c *Client) httpClient() *http.Client {
6974
return c.Client
7075
}
7176

72-
//Call 请求函数
77+
// Call 请求函数
7378
func (c *Client) Call(contract giraffe.Contract, req *http.Request, opts ...giraffe.CallOption) (*http.Response, error) {
7479
if req == nil {
7580
return nil, errors.New("request was nil")
@@ -90,35 +95,108 @@ func (c *Client) Call(contract giraffe.Contract, req *http.Request, opts ...gira
9095
if hostname := req.Header.Get("host"); hostname != "" {
9196
req.Host = hostname
9297
}
98+
if c.NameService == nil {
99+
// 原逻辑中NameService没有设置也会发送http请求,保留这个特征
100+
return c.httpClient().Do(req)
101+
}
102+
return c.sendWithENS(req, contract)
103+
}
93104

94-
if c.NameService != nil {
95-
addr, err := c.NameService.GetAddress(req.Context(), contract)
96-
if err != nil {
97-
return nil, err
105+
func (c *Client) sendWithENS(req *http.Request, contract giraffe.Contract) (resp *http.Response, err error) {
106+
// ENS服务发现
107+
addrList, err := c.getAllAddressesWithENS(req.Context(), contract)
108+
if err != nil {
109+
return
110+
}
111+
112+
// 备份 http body
113+
var originalBody []byte
114+
req.URL.Scheme = "http"
115+
if req != nil && req.Body != nil {
116+
originalBody, _ = copyBody(req)
117+
}
118+
119+
// 如果 retryConf.Enabled 为false, 获取 sendCount 的值为1,表示只执行一次,不会重试
120+
sendCount := c.retryConf.getSendCount()
121+
122+
retryInterval := c.retryConf.RetryInterval
123+
addr := addrList[0]
124+
i := 0
125+
unavailableRetry := false
126+
for ; i < sendCount; i++ {
127+
// 当i为1时,表示已经是重试循环, 需要根据情况进行等待
128+
// 当 单节点连接被拒绝重试 或者 503重试, 需要等待一段时间后再发起请求
129+
// 多节点下,连接被拒绝,就不等待了,直接访问其他节点
130+
if i > 0 && (len(addrList) <= 0 || unavailableRetry) {
131+
time.Sleep(retryInterval)
132+
}
133+
// 服务重试,以轮询策略为节点选择策略
134+
if addr == "" {
135+
addr = addrList[i%len(addrList)]
98136
}
137+
// 根据执行次数, 轮询节点
99138
req.URL.Host = addr
100-
req.URL.Scheme = "http"
139+
140+
resp, err = c.httpClient().Do(req)
141+
if err != nil {
142+
// connection refuse 重试机制
143+
if errors.Is(err, syscall.ECONNREFUSED) {
144+
resetBody(req, originalBody)
145+
addr = "" // 当前节点的访问被拒绝, addr置空, 获取其他节点
146+
unavailableRetry = false
147+
continue
148+
}
149+
return // 非 connection refuse 错误直接退出
150+
}
151+
if err == nil && resp.StatusCode != http.StatusServiceUnavailable {
152+
return // 非 503 异常, 直接退出不重试
153+
}
154+
155+
// 503异常的重试机制
156+
retryAfterStr := resp.Header.Get("Retry-After")
157+
retryAfter, _ := strconv.Atoi(retryAfterStr)
158+
if retryAfter <= 0 {
159+
// 没有设置 Retry-After , 重试机制失效, 直接退出
160+
return
161+
}
162+
// Retry-After单位为秒: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
163+
// TODO 对获取的retryAfter进行校验,不要让它因为错误的设置而等待超长的时间
164+
retryInterval = time.Duration(retryAfter) * time.Second
165+
unavailableRetry = true
166+
_ = resp.Body.Close() // 重试, 所以释放当前fd
167+
resetBody(req, originalBody)
101168
}
169+
return
170+
}
102171

103-
return c.httpClient().Do(req)
172+
func (c *Client) getAllAddressesWithENS(ctx context.Context, contract giraffe.Contract) (addresses []string, err error) {
173+
if c.NameService == nil {
174+
return
175+
}
176+
addresses, err = c.NameService.GetAllAddresses(ctx, contract)
177+
if err != nil {
178+
return
179+
}
180+
// 排序
181+
sort.Strings(addresses)
182+
return
104183
}
105184

106-
//NewClient Client实例化函数
185+
// NewClient Client实例化函数
107186
func NewClient(opts ...ClientOption) *Client {
108187
c := &Client{
109188
Client: &http.Client{},
110189
Middleware: DefaultMiddleware,
111190
NameService: nil,
112191
}
113-
114192
for _, o := range opts {
115193
o(c)
116194
}
117-
195+
c.retryConf.init()
118196
return c
119197
}
120198

121-
//WithClient 注入 http.Client
199+
// WithClient 注入 http.Client
122200
func WithClient(client *http.Client) ClientOption {
123201
return func(c *Client) {
124202
if client == nil {
@@ -128,9 +206,15 @@ func WithClient(client *http.Client) ClientOption {
128206
}
129207
}
130208

131-
//WithNameService 注入 NameService
209+
// WithNameService 注入 NameService
132210
func WithNameService(n giraffe.NameService) ClientOption {
133211
return func(c *Client) {
134212
c.NameService = n
135213
}
136214
}
215+
216+
func WithRetryConfig(conf RetryConfig) ClientOption {
217+
return func(c *Client) {
218+
c.retryConf = conf
219+
}
220+
}

0 commit comments

Comments
 (0)