Skip to content

Commit 959c6dc

Browse files
davissp14davissp14
committed
Transition away from private ips to dns
1 parent 5ea392e commit 959c6dc

File tree

6 files changed

+179
-118
lines changed

6 files changed

+179
-118
lines changed

‎fly.toml‎

Lines changed: 37 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,29 @@
1-
# fly.toml app configuration file generated for shaun-etcd-test on 2024-10-14T16:06:44-05:00
1+
# fly.toml app configuration file generated for shaun-etcd-test on 2025-02-18T16:18:05-06:00
22
#
33
# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
44
#
55

66
app = 'shaun-etcd-test'
7+
primary_region = 'ord'
78
kill_signal = 'SIGTERM'
89
kill_timeout = '30s'
910

1011
[experimental]
1112
auto_rollback = true
1213

13-
[[mounts]]
14-
source = 'etcd_data'
15-
destination = '/etcd_data'
14+
[build]
1615

17-
[checks]
18-
[checks.etcd]
19-
grace_period = "30s"
20-
interval = "15s"
21-
method = "get"
22-
path = "/flycheck/etcd"
23-
port = 5500
24-
timeout = "10s"
25-
type = "http"
16+
[[mounts]]
17+
source = 'data'
18+
destination = '/data'
2619

27-
[checks.vm]
28-
grace_period = "1s"
29-
interval = "1m"
30-
method = "get"
31-
path = "/flycheck/vm"
32-
port = 5500
33-
timeout = "10s"
34-
type = "http"
20+
[http_service]
21+
internal_port = 2379
22+
force_https = true
23+
auto_stop_machines = 'stop'
24+
auto_start_machines = true
25+
min_machines_running = 3
26+
processes = ['app']
3527

3628
[[services]]
3729
protocol = 'tcp'
@@ -53,10 +45,31 @@ kill_timeout = '30s'
5345
hard_limit = 1000
5446
soft_limit = 300
5547

56-
[[services.tcp_checks]]
57-
interval = '10s'
58-
timeout = '2s'
48+
[checks]
49+
[checks.etcd]
50+
port = 5500
51+
type = 'http'
52+
interval = '15s'
53+
timeout = '10s'
54+
grace_period = '30s'
55+
method = 'get'
56+
path = '/flycheck/etcd'
57+
58+
[checks.vm]
59+
port = 5500
60+
type = 'http'
61+
interval = '1m0s'
62+
timeout = '10s'
63+
grace_period = '1s'
64+
method = 'get'
65+
path = '/flycheck/vm'
66+
67+
[[vm]]
68+
memory = '1gb'
69+
cpu_kind = 'shared'
70+
cpus = 1
5971

6072
[[metrics]]
6173
port = 2379
6274
path = '/metrics'
75+
https = false

‎internal/flyetcd/client.go‎

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func NewClient(endpoints []string) (*Client, error) {
4848
return &Client{c}, nil
4949
}
5050

51-
func (c *Client) MemberId(ctx context.Context, name string) (uint64, error) {
51+
func (c *Client) MemberID(ctx context.Context, name string) (uint64, error) {
5252
resp, err := c.MemberList(ctx)
5353
if err != nil {
5454
return 0, err
@@ -67,7 +67,7 @@ func (c *Client) IsLeader(ctx context.Context, node *Node) (bool, error) {
6767
return false, err
6868
}
6969

70-
id, err := c.MemberId(ctx, node.Config.Name)
70+
id, err := c.MemberID(ctx, node.Config.Name)
7171
if err != nil {
7272
return false, err
7373
}
@@ -78,24 +78,3 @@ func (c *Client) IsLeader(ctx context.Context, node *Node) (bool, error) {
7878

7979
return false, nil
8080
}
81-
82-
// func (c *EtcdClient) InitializeAuth(ctx context.Context) error {
83-
// if err := c.CreateUser(ctx, "root", envOrDefault("ETCD_PASSWORD", "password")); err != nil {
84-
// switch err {
85-
// case rpctypes.ErrUserAlreadyExist:
86-
// case rpctypes.ErrUserEmpty: // Auth has already been enabled.
87-
// return nil
88-
// default:
89-
// return err
90-
// }
91-
// }
92-
93-
// if err := c.GrantRoleToUser(ctx, "root", "root"); err != nil {
94-
// return err
95-
// }
96-
97-
// if err := c.EnableAuthentication(ctx); err != nil {
98-
// return err
99-
// }
100-
// return nil
101-
// }

‎internal/flyetcd/config.go‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,21 +38,29 @@ type Config struct {
3838

3939
func NewConfig(endpoint *Endpoint) (*Config, error) {
4040
cfg := &Config{
41-
Name: endpoint.Name,
42-
ListenPeerUrls: endpoint.PeerURL,
43-
AdvertiseClientUrls: endpoint.ClientURL,
44-
DataDir: dataDir,
45-
ListenClientUrls: "http://[::]:2379",
41+
Name: endpoint.Name,
42+
// Listen on all interfaces (IPv4/IPv6) at the default ports
43+
// so etcd doesn’t complain about needing an IP.
44+
ListenPeerUrls: "http://[::]:2380",
45+
ListenClientUrls: "http://[::]:2379",
46+
47+
// Advertise the DNS name (or ephemeral IP) so other members
48+
// can connect to it.
4649
InitialAdvertisePeerUrls: endpoint.PeerURL,
47-
InitialCluster: fmt.Sprintf("%s=%s", endpoint.Name, endpoint.PeerURL),
48-
InitialClusterToken: getMD5Hash(os.Getenv("FLY_APP_NAME")),
49-
InitialClusterState: "new",
50-
AutoCompactionMode: "periodic",
51-
AutoCompactionRetention: "1",
52-
AuthToken: "",
53-
MaxSnapshots: 10,
54-
MaxWals: 10,
55-
SnapshotCount: 10000, // Default
50+
AdvertiseClientUrls: endpoint.ClientURL,
51+
52+
// Etcd data directory
53+
DataDir: dataDir,
54+
55+
InitialCluster: fmt.Sprintf("%s=%s", endpoint.Name, endpoint.PeerURL),
56+
InitialClusterToken: getMD5Hash(os.Getenv("FLY_APP_NAME")),
57+
InitialClusterState: "new",
58+
AutoCompactionMode: "periodic",
59+
AutoCompactionRetention: "1",
60+
AuthToken: "",
61+
MaxSnapshots: 10,
62+
MaxWals: 10,
63+
SnapshotCount: 10000, // Default
5664
}
5765

5866
if err := cfg.SetAuthToken(); err != nil {

‎internal/flyetcd/endpoint.go‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,35 @@ type Endpoint struct {
1717

1818
func NewEndpoint(addr string) *Endpoint {
1919
return &Endpoint{
20-
Name: getMD5Hash(addr),
20+
Name: os.Getenv("FLY_MACHINE_ID"),
2121
Addr: addr,
22-
ClientURL: fmt.Sprintf("http://[%s]:2379", addr),
23-
PeerURL: fmt.Sprintf("http://[%s]:2380", addr),
22+
ClientURL: fmt.Sprintf("http://%s:2379", addr),
23+
PeerURL: fmt.Sprintf("http://%s:2380", addr),
2424
}
2525
}
2626

27-
func currentEndpoint() (*Endpoint, error) {
28-
privateIP, err := privnet.PrivateIPv6()
29-
if err != nil {
30-
return nil, err
31-
}
32-
return NewEndpoint(privateIP.String()), nil
33-
}
34-
27+
// AllEndpoints uses DNS to return all Machines associated with the app.
3528
func AllEndpoints(ctx context.Context) ([]*Endpoint, error) {
36-
addrs, err := privnet.AllPeers(ctx, os.Getenv("FLY_APP_NAME"))
29+
machines, err := privnet.AllMachines(ctx, os.Getenv("FLY_APP_NAME"))
3730
if err != nil {
3831
return nil, err
3932
}
4033
var endpoints []*Endpoint
41-
for _, addr := range addrs {
42-
endpoints = append(endpoints, NewEndpoint(addr.String()))
34+
for _, m := range machines {
35+
36+
endpoints = append(endpoints, endpointFromMachine(m))
4337
}
4438
return endpoints, nil
4539
}
40+
41+
func currentEndpoint() (*Endpoint, error) {
42+
endpoint := fmt.Sprintf("%s.vm.%s.internal",
43+
os.Getenv(("FLY_MACHINE_ID")),
44+
os.Getenv("FLY_APP_NAME"))
45+
46+
return NewEndpoint(endpoint), nil
47+
}
48+
49+
func endpointFromMachine(machine privnet.Machine) *Endpoint {
50+
return NewEndpoint(fmt.Sprintf("%s.vm.%s.internal", machine.ID, os.Getenv("FLY_APP_NAME")))
51+
}

‎internal/privnet/sixpn.go‎

Lines changed: 82 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -9,60 +9,103 @@ import (
99
"time"
1010
)
1111

12-
func AllPeers(ctx context.Context, appName string) ([]net.IPAddr, error) {
13-
return Get6PN(ctx, fmt.Sprintf("%s.internal", appName))
12+
// func AllPeers(ctx context.Context, appName string) ([]net.IPAddr, error) {
13+
// return Get6PN(ctx, fmt.Sprintf("%s.internal", appName))
14+
// }
15+
16+
// func Get6PN(ctx context.Context, hostname string) ([]net.IPAddr, error) {
17+
// nameserver := os.Getenv("FLY_NAMESERVER")
18+
// if nameserver == "" {
19+
// nameserver = "fdaa::3"
20+
// }
21+
// nameserver = net.JoinHostPort(nameserver, "53")
22+
// r := &net.Resolver{
23+
// PreferGo: true,
24+
// Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
25+
// d := net.Dialer{
26+
// Timeout: 1 * time.Second,
27+
// }
28+
// return d.DialContext(ctx, "udp6", nameserver)
29+
// },
30+
// }
31+
// ips, err := r.LookupIPAddr(ctx, hostname)
32+
33+
// if err != nil {
34+
// return ips, err
35+
// }
36+
37+
// // make sure we're including the local ip, just in case it's not in service discovery yet
38+
// local, err := r.LookupIPAddr(ctx, "fly-local-6pn")
39+
40+
// if err != nil || len(local) < 1 {
41+
// return ips, err
42+
// }
43+
44+
// localExists := false
45+
// for _, v := range ips {
46+
// if v.IP.String() == local[0].IP.String() {
47+
// localExists = true
48+
// }
49+
// }
50+
51+
// if !localExists {
52+
// ips = append(ips, local[0])
53+
// }
54+
// return ips, err
55+
// }
56+
57+
type Machine struct {
58+
ID string
59+
Region string
60+
}
61+
62+
func AllMachines(ctx context.Context, appName string) ([]Machine, error) {
63+
r := getResolver()
64+
txts, err := r.LookupTXT(ctx, fmt.Sprintf("vms.%s.internal", appName))
65+
if err != nil {
66+
return nil, err
67+
}
68+
69+
machines := make([]Machine, 0)
70+
for _, txt := range txts {
71+
parts := strings.Split(txt, ",")
72+
for _, part := range parts {
73+
parts := strings.Split(part, " ")
74+
if len(parts) != 2 {
75+
return nil, fmt.Errorf("invalid machine DNS TXT format: %s", txt)
76+
}
77+
machines = append(machines, Machine{ID: parts[0], Region: parts[1]})
78+
}
79+
}
80+
return machines, nil
1481
}
1582

16-
func Get6PN(ctx context.Context, hostname string) ([]net.IPAddr, error) {
83+
func getResolver() *net.Resolver {
1784
nameserver := os.Getenv("FLY_NAMESERVER")
1885
if nameserver == "" {
1986
nameserver = "fdaa::3"
2087
}
2188
nameserver = net.JoinHostPort(nameserver, "53")
22-
r := &net.Resolver{
89+
return &net.Resolver{
2390
PreferGo: true,
24-
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
91+
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
2592
d := net.Dialer{
2693
Timeout: 1 * time.Second,
2794
}
2895
return d.DialContext(ctx, "udp6", nameserver)
2996
},
3097
}
31-
ips, err := r.LookupIPAddr(ctx, hostname)
32-
33-
if err != nil {
34-
return ips, err
35-
}
36-
37-
// make sure we're including the local ip, just in case it's not in service discovery yet
38-
local, err := r.LookupIPAddr(ctx, "fly-local-6pn")
39-
40-
if err != nil || len(local) < 1 {
41-
return ips, err
42-
}
43-
44-
localExists := false
45-
for _, v := range ips {
46-
if v.IP.String() == local[0].IP.String() {
47-
localExists = true
48-
}
49-
}
50-
51-
if !localExists {
52-
ips = append(ips, local[0])
53-
}
54-
return ips, err
5598
}
5699

57-
func PrivateIPv6() (net.IP, error) {
58-
ips, err := net.LookupIP("fly-local-6pn")
59-
if err != nil && !strings.HasSuffix(err.Error(), "no such host") && !strings.HasSuffix(err.Error(), "server misbehaving") {
60-
return nil, err
61-
}
100+
// func PrivateIPv6() (net.IP, error) {
101+
// ips, err := net.LookupIP("fly-local-6pn")
102+
// if err != nil && !strings.HasSuffix(err.Error(), "no such host") && !strings.HasSuffix(err.Error(), "server misbehaving") {
103+
// return nil, err
104+
// }
62105

63-
if len(ips) > 0 {
64-
return ips[0], nil
65-
}
106+
// if len(ips) > 0 {
107+
// return ips[0], nil
108+
// }
66109

67-
return net.ParseIP("127.0.0.1"), nil
68-
}
110+
// return net.ParseIP("127.0.0.1"), nil
111+
// }

0 commit comments

Comments
 (0)