Skip to content

attempt #1117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

attempt #1117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 70 additions & 14 deletions pkg/loop/internal/net/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package net

import (
"context"
"errors"
"sync"
"sync/atomic"
"time"
Expand Down Expand Up @@ -54,7 +55,11 @@ func (c *clientConn) Invoke(ctx context.Context, method string, args interface{}
c.mu.RUnlock()

if cc == nil {
cc = c.refresh(ctx, nil)
var err error
cc, err = c.refresh(ctx, nil)
if err != nil {
return err
}
}
for cc != nil {
err := cc.Invoke(ctx, method, args, reply, opts...)
Expand All @@ -65,7 +70,11 @@ func (c *clientConn) Invoke(ctx context.Context, method string, args interface{}
return err
}
c.Logger.Warnw("clientConn: Invoke: terminal error, refreshing connection", "method", method, "err", err)
cc = c.refresh(ctx, cc)
cc, err = c.refresh(ctx, cc)
if err != nil {
return err
}

continue
}
return err
Expand All @@ -79,27 +88,59 @@ func (c *clientConn) NewStream(ctx context.Context, desc *grpc.StreamDesc, metho
c.mu.RUnlock()

if cc == nil {
cc = c.refresh(ctx, nil)
var err error
cc, err = c.refresh(ctx, nil)
if err != nil {
return nil, err
}
}
for cc != nil {
s, err := cc.NewStream(ctx, desc, method, opts...)
if isErrTerminal(err) {
c.Logger.Warnw("clientConn: NewStream: terminal error, refreshing connection", "err", err)
cc = c.refresh(ctx, cc)
cc, err = c.refresh(ctx, cc)
if err != nil {
return nil, err
}
continue
}
return s, err
}
return nil, context.Cause(ctx)
}

// AppError represents a custom application error that wraps another error.
type AppError struct {
Err error // Wrapped error
}

// Error implements the error interface for AppError.
func (e *AppError) Error() string {
if e.Err != nil {
return e.Err.Error()
}
return ""
}

// Unwrap allows access to the wrapped error.
func (e *AppError) Unwrap() error {
return e.Err
}

// NewAppError creates a new AppError instance.
func NewAppError(err error) *AppError {
return &AppError{
Err: err,
}
}

// refresh replaces c.cc with a new (different from orig) *grpc.ClientConn, and returns it as well.
// It will block until a new connection is successfully dialed, or return nil if the context expires.
func (c *clientConn) refresh(ctx context.Context, orig *grpc.ClientConn) *grpc.ClientConn {
func (c *clientConn) refresh(ctx context.Context, orig *grpc.ClientConn) (*grpc.ClientConn, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.cc != orig {
return c.cc
return c.cc, nil
}
if c.cc != nil {
if err := c.cc.Close(); err != nil {
Expand All @@ -108,13 +149,19 @@ func (c *clientConn) refresh(ctx context.Context, orig *grpc.ClientConn) *grpc.C
c.CloseAll(c.deps...)
}

try := func() bool {
try := func() (bool, error) {
c.Logger.Debug("Client refresh")
id, deps, err := c.newClient(ctx)
if err != nil {
c.Logger.Errorw("Client refresh attempt failed", "err", err)
c.CloseAll(deps...)
return false

var appErr *AppError
if errors.As(err, &appErr) {
return false, err
}

return false, nil
}
c.deps = deps

Expand All @@ -126,29 +173,38 @@ func (c *clientConn) refresh(ctx context.Context, orig *grpc.ClientConn) *grpc.C
lggr.Errorw("Client dial failed", "err", ErrConnDial{Name: c.name, ID: id, Err: err})
}
c.CloseAll(c.deps...)
return false
return false, nil
}
return true
return true, nil
}

b := backoff.Backoff{
Min: 100 * time.Millisecond,
Max: 5 * time.Second,
Factor: 2,
}
for !try() {
for {
success, err := try()
if success {
break
}

if err != nil {
return nil, err
}

if ctx.Err() != nil {
c.Logger.Errorw("Client refresh failed: aborting refresh due to context error", "err", ctx.Err())
return nil
return nil, nil
}
wait := b.Duration()
c.Logger.Infow("Waiting to refresh", "wait", wait)
select {
case <-ctx.Done():
return nil
return nil, nil
case <-time.After(wait):
}
}

return c.cc
return c.cc, nil
}
6 changes: 4 additions & 2 deletions pkg/loop/internal/relayerset/relayer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,15 @@ func (r *relayerClient) NewContractReader(_ context.Context, contractReaderConfi
cc := r.relayerSetClient.NewClientConn("ContractReader", func(ctx context.Context) (uint32, net.Resources, error) {
contractReaderID, err := r.relayerSetClient.NewContractReader(ctx, r.relayerID, contractReaderConfig)
if err != nil {
return 0, nil, fmt.Errorf("error getting NewContractReader from relayerSetServer: %w", err)
return 0, nil, net.NewAppError(err)
}

return contractReaderID, nil, nil
})

return contractreader.NewClient(r.relayerSetClient.BrokerExt.WithName("ContractReaderClientInRelayerSet"), cc), nil
client := contractreader.NewClient(r.relayerSetClient.BrokerExt.WithName("ContractReaderClientInRelayerSet"), cc)
err := client.Ready()
return client, err
}

func (r *relayerClient) NewContractWriter(_ context.Context, contractWriterConfig []byte) (types.ContractWriter, error) {
Expand Down
Loading