Merge pull request #21 from tailingchen/dynamic-discover-ethclient-list

Dynamic discover ethclients in k8s cluster

Author: tailingchen

multiclient/client.go

@@ -34,74 +34,78 @@ import (
 
 const (
 	dialTimeout = 5 * time.Second
+	retryPeriod = 10 * time.Second
 )
 
 var (
 	ErrInvalidTypeCast = errors.New("invalid type cast")
 	ErrNoEthClient     = errors.New("no eth client")
-
-	retryPeriod = 10 * time.Second
 )
 
 type Client struct {
+	ctx          context.Context
+	cancel       context.CancelFunc
 	rpcClientMap *Map
+	retrydialWg  sync.WaitGroup
 }
 
 func New(ctx context.Context, opts ...Option) (*Client, error) {
+	// create client own context to control the internal go routines
+	myCtx, myCancel := context.WithCancel(context.Background())
 	mc := &Client{
+		ctx:          myCtx,
+		cancel:       myCancel,
 		rpcClientMap: NewMap(),
 	}
+
+	var newErr error
+	defer func() {
+		if newErr != nil {
+			// cancel go routines
+			mc.Close()
+		}
+	}()
+
 	for _, opt := range opts {
-		if err := opt(mc); err != nil {
-			return nil, err
+		if newErr = opt(mc); newErr != nil {
+			return nil, newErr
 		}
 	}
 
-	urls := mc.rpcClientMap.Keys()
-	lens := len(urls)
-	if lens == 0 {
-		return nil, ErrNoEthClient
-	}
+	// Dial each eth client
+	mc.DialClients(ctx)
 
-	log.Debug("Create multiclient", "urls", lens)
-	errCh := make(chan error, lens)
-	for _, rawURL := range urls {
-		go func(ctx context.Context, rawURL string) {
-			ctx, cancel := context.WithTimeout(ctx, dialTimeout)
-			defer cancel()
-			c, err := rpc.DialContext(ctx, rawURL)
-			if err == nil {
-				log.Info("Connect to ethclient successfully", "url", rawURL)
-				mc.rpcClientMap.Set(rawURL, c)
-			} else {
-				log.Error("Failed to dial eth client", "rawURL", rawURL, "err", err)
-			}
-			errCh <- err
-		}(ctx, rawURL)
+	// Return error if have no available eth client.
+	if len(mc.rpcClientMap.List()) == 0 {
+		newErr = ErrNoEthClient
+		return nil, newErr
 	}
 
-	var dialErr error
-	for i := 0; i < lens; i++ {
-		err := <-errCh
-		if err != nil {
-			dialErr = err
-		}
-	}
-	if dialErr != nil {
-		mc.Close()
-		return nil, dialErr
-	}
+	mc.retrydialWg.Add(1)
+	go mc.retrydial()
+
 	return mc, nil
 }
 
 // Close closes an existing RPC connection.
 func (mc *Client) Close() {
+	// stop go routines
+	mc.cancel()
+	mc.retrydialWg.Wait()
 	clients := mc.rpcClientMap.List()
 	for _, c := range clients {
 		c.Close()
 	}
 }
 
+func (mc *Client) Context() context.Context {
+	return mc.ctx
+}
+
+func (mc *Client) ClientMap() *Map {
+	return mc.rpcClientMap
+}
+
 func (mc *Client) EthClients() []*ethclient.Client {
 	clients := mc.rpcClientMap.List()
 	ethClients := make([]*ethclient.Client, len(clients))
@@ -727,3 +731,50 @@ func postToAll(ctx context.Context, fns []postFn) error {
 	}
 	return lastError
 }
+
+type dialedClient struct {
+	url    string
+	client *rpc.Client
+}
+
+func (mc *Client) DialClients(ctx context.Context) {
+	urls := mc.rpcClientMap.NilClients()
+	dialCh := make(chan *dialedClient, len(urls))
+	for _, rawURL := range urls {
+		go func(rawURL string) {
+			dialCtx, cancel := context.WithTimeout(ctx, dialTimeout)
+			defer cancel()
+			c, err := rpc.DialContext(dialCtx, rawURL)
+			if err == nil {
+				log.Info("Connect to eth client successfully", "url", rawURL)
+			} else {
+				log.Warn("Failed to dial eth client", "url", rawURL, "err", err)
+			}
+			dialCh <- &dialedClient{url: rawURL, client: c}
+		}(rawURL)
+	}
+
+	for i := 0; i < len(urls); i++ {
+		dialed := <-dialCh
+		if dialed.client != nil {
+			mc.rpcClientMap.Replace(dialed.url, dialed.client)
+		}
+	}
+}
+
+func (mc *Client) retrydial() {
+	defer mc.retrydialWg.Done()
+
+	ticker := time.NewTicker(retryPeriod)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			mc.DialClients(mc.ctx)
+		case <-mc.ctx.Done():
+			// mc is closed, stop retry
+			return
+		}
+	}
+}

multiclient/consul.go

@@ -0,0 +1,83 @@
+// Copyright 2018 AMIS Technologies
+// This file is part of the hypereth library.
+//
+// The hypereth library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The hypereth library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the hypereth library. If not, see <http://www.gnu.org/licenses/>.
+
+package multiclient
+
+import (
+	"fmt"
+	netURL "net/url"
+
+	"github.com/getamis/sirius/log"
+	consulAPI "github.com/hashicorp/consul/api"
+)
+
+// ConsulDiscovery discovers the dynamic ethclient endpoints through consul server.
+// TODO: should watch the change of endpoints
+func ConsulDiscovery(rawURL, serviceID, serviceScheme string) Option {
+	return func(mc *Client) error {
+		client, err := createConsulClient(rawURL)
+		if err != nil {
+			return err
+		}
+		urls, err := getEthURLsFromConsul(client, serviceID, serviceScheme)
+		if err != nil {
+			return err
+		}
+		log.Info("EthClients from consul", "urls", urls)
+		for _, url := range urls {
+			mc.ClientMap().Set(url, nil)
+		}
+		return nil
+	}
+}
+
+func getEthURLsFromConsul(client *consulAPI.Client, serviceID, serviceScheme string) ([]string, error) {
+	list, _, err := client.Catalog().Service(serviceID, "", nil)
+	if err != nil {
+		log.Error("Failed to get service from consul", "serviceID", serviceID, "err", err)
+		return nil, err
+	}
+
+	ethURLs := make([]string, len(list))
+	for i, srv := range list {
+		addr := srv.ServiceAddress
+		if addr == "" {
+			addr = srv.Address
+		}
+		ethURLs[i] = fmt.Sprintf("%s://%s:%d", serviceScheme, addr, srv.ServicePort)
+	}
+	return ethURLs, nil
+}
+
+func createConsulClient(rawURL string) (*consulAPI.Client, error) {
+	consulURL, err := netURL.Parse(rawURL)
+	if err != nil {
+		log.Error("Failed to parse consul url", "url", rawURL, "err", err)
+		return nil, err
+	}
+
+	config := &consulAPI.Config{
+		Address: consulURL.Host,
+		Scheme:  consulURL.Scheme,
+	}
+
+	client, err := consulAPI.NewClient(config)
+	if err != nil {
+		log.Error("Failed to create consul client", "err", err)
+		return nil, err
+	}
+	return client, nil
+}