use private ips for libp2p && DigitalOcean

Author: swift1337

activities/loadtest/mocks/nodei.go

@@ -68,6 +68,10 @@ func (m MockNode) GetIP(ctx context.Context) (string, error) {
 	return m.IP, nil
 }
 
+func (m MockNode) GetPrivateIP(ctx context.Context) (string, error) {
+	return m.GetIP(ctx)
+}
+
 func (m MockNode) GetExternalAddress(ctx context.Context, s string) (string, error) {
 	//TODO implement me
 	panic("implement me")

petri/core/provider/digitalocean/task.go

@@ -252,6 +252,8 @@ func (t *Task) DownloadDir(ctx context.Context, s string, s2 string) error {
 	panic("implement me")
 }
 
+// GetIP returns *Tailscale* IP.
+// todo refactor to GetIP(ctx, type=[private, tailscale, public, ...])
 func (t *Task) GetIP(ctx context.Context) (string, error) {
 	return t.getTailscaleIp(ctx)
 }
@@ -265,6 +267,16 @@ func (t *Task) GetExternalAddress(ctx context.Context, port string) (string, err
 	return net.JoinHostPort(ip, port), nil
 }
 
+// GetPrivateIP returns node's private IP address
+func (t *Task) GetPrivateIP(ctx context.Context) (string, error) {
+	droplet, err := t.GetDroplet(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to get droplet: %w", err)
+	}
+
+	return droplet.PrivateIPv4()
+}
+
 func (t *Task) RunCommand(ctx context.Context, cmd []string) (string, string, int, error) {
 	status, err := t.GetStatus(ctx)
 	if err != nil {

petri/core/provider/docker/task.go

@@ -131,6 +131,11 @@ func (t *Task) GetIP(ctx context.Context) (string, error) {
 	return ip, nil
 }
 
+// GetPrivateIP returns node's private IP address
+func (t *Task) GetPrivateIP(ctx context.Context) (string, error) {
+	return t.GetIP(ctx)
+}
+
 func (t *Task) WaitForStatus(ctx context.Context, interval time.Duration, desiredStatus provider.TaskStatus) error {
 	for {
 		select {

petri/core/provider/provider.go

@@ -36,6 +36,7 @@ type TaskI interface {
 	DownloadDir(context.Context, string, string) error
 
 	GetIP(context.Context) (string, error)
+	GetPrivateIP(context.Context) (string, error)
 	GetExternalAddress(context.Context, string) (string, error)
 	DialContext() func(context.Context, string, string) (net.Conn, error)
 

petri/core/types/node.go

@@ -103,8 +103,12 @@ type NodeI interface {
 	GetDefinition() provider.TaskDefinition
 
 	// GetIP returns the IP address of the node
+	// todo refactor to GetIP(ctx, type=[private, tailscale, public, ...])
 	GetIP(context.Context) (string, error)
 
+	// GetPrivateIP returns node's private IP address
+	GetPrivateIP(context.Context) (string, error)
+
 	// Serialize serializes the node
 	Serialize(context.Context, provider.ProviderI) ([]byte, error)
 }

petri/cosmos/chain/chain.go

@@ -1008,7 +1008,6 @@ func configureNode(
 		return err
 	}
 
-	// todo if libp2p=true, then store addressbook.toml in a proper format!
 	persistentPeersString, err := persistentPeers.AsCometPeerString(ctx, useExternalAddress)
 	if err != nil {
 		return fmt.Errorf("failed to get comet peer string for persistent peers: %w", err)
@@ -1032,7 +1031,15 @@ func configureNode(
 	useLibP2P, addressBookFile := chainConfig.UseLibP2P()
 	if useLibP2P {
 		logger.Info("Using lib-p2p!", zap.String("address_book_file", addressBookFile))
-		err = writeLibP2PAddressBook(ctx, node, addressBookFile, useExternalAddress, seeds, persistentPeers)
+		err = writeLibP2PAddressBook(
+			ctx,
+			node,
+			addressBookFile,
+			useExternalAddress,
+			seeds,
+			persistentPeers,
+		)
+
 		if err != nil {
 			return fmt.Errorf("writeLibP2PAddressBook: %w", err)
 		}
@@ -1051,13 +1058,15 @@ func writeLibP2PAddressBook(
 ) error {
 	peers := []any{}
 
+	isDocker := !useExternalAddress
+
 	// combine all the peer sets into list of peers
 	for _, peerSet := range peerSets {
 		if peerSet.Empty() {
 			continue
 		}
 
-		elements, err := peerSet.AsLibP2PAddressBook(ctx, useExternalAddress)
+		elements, err := peerSet.AsLibP2PAddressBook(ctx, isDocker)
 		if err != nil {
 			return fmt.Errorf("failed to get libp2p address book for peer set: %w", err)
 		}

petri/cosmos/chain/peers.go

@@ -62,12 +62,19 @@ func (ps *PeerSet) AsCometPeerString(ctx context.Context, useExternal bool) (str
 // AsLibP2PAddressBook returns a map of node IDs to addresses of chain nodes.
 // Format it [{host: "1.2.3.4:26656", id: "<lib-p2p-peer-id>"}, {...}, ...]
 // @see https://github.com/cometbft/cometbft/blob/608fe92cbc3774c6cdf36c59c56b6c8362489ef1/lp2p/addressbook.go#L16
-func (ps *PeerSet) AsLibP2PAddressBook(ctx context.Context, useExternal bool) ([]any, error) {
+func (ps *PeerSet) AsLibP2PAddressBook(ctx context.Context, isDocker bool) ([]any, error) {
 	peers := make([]any, 0, len(ps.peers))
 
 	resolveHost := peerHostInternal
-	if useExternal {
-		resolveHost = peerHostExternal
+	if !isDocker {
+		// This breaks geo-distributed testnet for go-libp2p.
+		//
+		// Currently DigitalOcean Droplet are configured via Tailscale, which
+		// causes issues go-libp2p connection (tailscale's IP are fetched via peerHostExternal)
+		// Thus, we use VMs private IPs.
+		//
+		// TODO: come up with a better solution that doesn't use Tailscale, but supports multiple regions.
+		resolveHost = peerHostPrivate
 	}
 
 	for _, n := range ps.peers {
@@ -105,6 +112,15 @@ func peerHostInternal(ctx context.Context, n petri.NodeI) (string, error) {
 	return fmt.Sprintf("%s:%s", ip, cometPort), nil
 }
 
+func peerHostPrivate(ctx context.Context, n petri.NodeI) (string, error) {
+	ip, err := n.GetPrivateIP(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s:%s", ip, cometPort), nil
+}
+
 // used for digitalocean
 func peerIDFromNode(ctx context.Context, n petri.NodeI) (peer.ID, error) {
 	cometPubKey, err := n.PubKey(ctx)