Skip to content

Commit d4418f3

Browse files
0pcom0pcom
authored
Fix dmsgweb (skycoin#303)
* fix some issues with dmsg web * remove some commented code * update flags to include URL & DMSG Address of dmsg-discovery. * Remove multi-disc from dmsgcurl. * move some flags to internal/flags package * fix CI errors * implement three methods for connecting to dmsg network in dmsg curl and other utilities * minor refactor / optimization for dmsgcurl * other small improvements and optimizations
1 parent 78c6883 commit d4418f3

File tree

89 files changed

+15956
-278
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+15956
-278
lines changed

‎cmd/dmsgcurl/commands/dmsgcurl.go‎

Lines changed: 52 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,12 @@ import (
2626
"golang.org/x/net/proxy"
2727

2828
"github.com/skycoin/dmsg/internal/cli"
29+
"github.com/skycoin/dmsg/internal/flags"
2930
"github.com/skycoin/dmsg/pkg/dmsg"
3031
"github.com/skycoin/dmsg/pkg/dmsghttp"
3132
)
3233

3334
var (
34-
ctxs []context.Context
35-
cancels []func()
36-
dmsgDiscs []string
37-
dmsgSessions int
3835
dmsgcurlData string
3936
sk cipher.SecKey
4037
pk cipher.PubKey
@@ -46,21 +43,15 @@ var (
4643
dmsgcurlWait int
4744
dmsgcurlOutput string
4845
replace bool
49-
proxyAddr []string
50-
httpClients []*http.Client
46+
proxyAddr string
5147
dialer = proxy.Direct //nolint unused
52-
dmsgHTTPPath string
53-
useHTTP bool
5448
err error
5549
)
5650

5751
func init() {
5852
RootCmd.Flags().SortFlags = false
59-
RootCmd.Flags().BoolVarP(&useHTTP, "http", "z", false, "use regular http to connect to dmsg discovery")
60-
RootCmd.Flags().StringSliceVarP(&dmsgDiscs, "dmsg-disc", "c", []string{dmsg.DiscAddr(false)}, "dmsg discovery url(s)\033[0m\n\r")
61-
RootCmd.Flags().StringVarP(&dmsgHTTPPath, "dmsgconf", "D", "", "dmsghttp-config path")
62-
RootCmd.Flags().StringSliceVarP(&proxyAddr, "proxy", "p", proxyAddr, "connect to dmsg via proxy (i.e. '127.0.0.1:1080')")
63-
RootCmd.Flags().IntVarP(&dmsgSessions, "sess", "e", 1, "number of dmsg servers to connect to\033[0m\n\r")
53+
flags.InitFlags(RootCmd)
54+
RootCmd.Flags().StringVarP(&proxyAddr, "proxy", "p", proxyAddr, "connect to DMSG via proxy (i.e. '127.0.0.1:1080')")
6455
RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "fatal", "[ debug | warn | error | fatal | panic | trace | info ]\033[0m\n\r")
6556
RootCmd.Flags().StringVarP(&dmsgcurlData, "data", "d", "", "dmsghttp POST data")
6657
RootCmd.Flags().StringVarP(&dmsgcurlOutput, "out", "o", "", "output filepath")
@@ -92,8 +83,8 @@ var RootCmd = &cobra.Command{
9283
}
9384
}
9485

95-
if dmsgHTTPPath != "" {
96-
dmsg.DmsghttpJSON, err = os.ReadFile(dmsgHTTPPath) //nolint
86+
if flags.DmsgHTTPPath != "" {
87+
dmsg.DmsghttpJSON, err = os.ReadFile(flags.DmsgHTTPPath) //nolint
9788
if err != nil {
9889
dlog.WithError(err).Fatal("Failed to read specified dmsghttp-config")
9990
}
@@ -130,70 +121,31 @@ var RootCmd = &cobra.Command{
130121
}
131122

132123
var cErr curlError
133-
if useHTTP {
134-
if len(dmsgDiscs) == 0 || dmsgDiscs[0] == "" {
135-
dmsgDiscs = []string{dmsg.DiscAddr(false)}
136-
}
137-
dlog.Debug("DMSG Discovery: ", dmsgDiscs)
138-
for i := range dmsgDiscs {
139-
ctx, cancel := cmdutil.SignalContext(context.Background(), dlog)
140-
defer cancel()
141-
ctxs = append(ctxs, ctx)
142-
cancels = append(cancels, cancel)
143-
144-
httpClient := &http.Client{}
145-
146-
if i < len(proxyAddr) && proxyAddr[i] != "" {
147-
// Use SOCKS5 proxy dialer if specified
148-
dialer, err := proxy.SOCKS5("tcp", proxyAddr[i], nil, proxy.Direct)
149-
if err != nil {
150-
dlog.WithError(fmt.Errorf("Error creating SOCKS5 dialer: %v", err)).Error(errorDesc["COULDNT_RESOLVE_PROXY"])
151-
os.Exit(errorCode["COULDNT_RESOLVE_PROXY"])
152-
}
153-
transport := &http.Transport{
154-
Dial: dialer.Dial,
155-
}
156-
httpClient = &http.Client{
157-
Transport: transport,
158-
}
159-
ctxs[i] = context.WithValue(context.Background(), "socks5_proxy", proxyAddr[i]) //nolint
160-
}
161-
httpClients = append(httpClients, httpClient)
124+
ctx, cancel := cmdutil.SignalContext(context.Background(), dlog)
125+
defer cancel()
162126

163-
cErr = handleRequest(ctxs[i], dlog, pk, sk, httpClients[i], dmsgDiscs[i], dmsgSessions, parsedURL, dmsgcurlData, !useHTTP)
164-
if cErr.Code == 0 {
165-
return nil
166-
}
167-
dlog.WithError(cErr.Error).Debug("An error occurred\n")
127+
httpClient := &http.Client{}
128+
if proxyAddr != "" {
129+
// Use SOCKS5 proxy dialer if specified
130+
dialer, err := proxy.SOCKS5("tcp", proxyAddr, nil, proxy.Direct)
131+
if err != nil {
132+
dlog.WithError(fmt.Errorf("Error creating SOCKS5 dialer: %v", err)).Error(errorDesc["COULDNT_RESOLVE_PROXY"])
133+
os.Exit(errorCode["COULDNT_RESOLVE_PROXY"])
168134
}
169-
} else { //Use direct dmsg client & embedded config
170-
ctx, cancel := cmdutil.SignalContext(context.Background(), dlog)
171-
defer cancel()
172-
ctxs = append(ctxs, ctx)
173-
174-
httpClient := &http.Client{}
175-
if 0 < len(proxyAddr) && proxyAddr[0] != "" {
176-
// Use SOCKS5 proxy dialer if specified
177-
dialer, err := proxy.SOCKS5("tcp", proxyAddr[0], nil, proxy.Direct)
178-
if err != nil {
179-
dlog.WithError(fmt.Errorf("Error creating SOCKS5 dialer: %v", err)).Error(errorDesc["COULDNT_RESOLVE_PROXY"])
180-
os.Exit(errorCode["COULDNT_RESOLVE_PROXY"])
181-
}
182-
transport := &http.Transport{
183-
Dial: dialer.Dial,
184-
}
185-
httpClient = &http.Client{
186-
Transport: transport,
187-
}
188-
ctxs[0] = context.WithValue(context.Background(), "socks5_proxy", proxyAddr[0]) //nolint
135+
transport := &http.Transport{
136+
Dial: dialer.Dial,
189137
}
190-
191-
cErr = handleRequest(ctxs[0], dlog, pk, sk, httpClient, "", dmsgSessions, parsedURL, dmsgcurlData, !useHTTP)
192-
if cErr.Code == 0 {
193-
return nil
138+
httpClient = &http.Client{
139+
Transport: transport,
194140
}
195-
dlog.WithError(cErr.Error).Debug("An error occurred\n")
141+
ctx = context.WithValue(context.Background(), "socks5_proxy", proxyAddr) //nolint
142+
}
143+
144+
cErr = handleRequest(ctx, pk, sk, httpClient, parsedURL, dmsgcurlData)
145+
if cErr.Code == 0 {
146+
return nil
196147
}
148+
197149
if cErr.Code != 0 {
198150
dlog.WithError(cErr.Error).Error("An error occurred\n")
199151
return cErr.Error
@@ -202,7 +154,7 @@ var RootCmd = &cobra.Command{
202154
},
203155
}
204156

205-
func handleRequest(ctx context.Context, dmsgLogger *logging.Logger, pk cipher.PubKey, sk cipher.SecKey, httpClient *http.Client, dmsgDisc string, dmsgSessions int, parsedURL *url.URL, dmsgcurlData string, dmsgHTTP bool) curlError {
157+
func handleRequest(ctx context.Context, pk cipher.PubKey, sk cipher.SecKey, httpClient *http.Client, parsedURL *url.URL, dmsgcurlData string) curlError {
206158
file, err := prepareOutputFile()
207159
if err != nil {
208160
return curlError{
@@ -213,17 +165,35 @@ func handleRequest(ctx context.Context, dmsgLogger *logging.Logger, pk cipher.Pu
213165
defer closeAndCleanFile(file, err)
214166
var dmsgC *dmsg.Client
215167
var closeDmsg func()
216-
if !dmsgHTTP {
217-
dmsgC, closeDmsg, err = cli.StartDmsg(ctx, dmsgLogger, pk, sk, httpClient, dmsgDisc, dmsgSessions)
168+
169+
if flags.UseDC {
170+
dmsgC, closeDmsg, err = cli.StartDmsgDirect(ctx, dlog, pk, sk, httpClient, "", flags.DmsgSessions, pk.String())
218171
} else {
219-
dmsgC, closeDmsg, err = cli.StartDmsgDirect(ctx, dmsgLogger, pk, sk, httpClient, dmsgDisc, dmsgSessions, destPK.String())
172+
if flags.UseHTTP {
173+
dmsgC, closeDmsg, err = cli.StartDmsg(ctx, dlog, pk, sk, httpClient, flags.DmsgDiscURL, flags.DmsgSessions)
174+
} else {
175+
var dmsgDC *dmsg.Client
176+
var closeDmsgDC func()
177+
dmsgDC, closeDmsgDC, err = cli.StartDmsgDirect(ctx, dlog, pk, sk, httpClient, "", flags.DmsgSessions, dmsg.ExtractPKFromDmsgAddr(flags.DmsgDiscAddr))
178+
if err != nil {
179+
dlog.WithError(err).Error("Error connecting to dmsg network")
180+
return curlError{
181+
Error: fmt.Errorf("%s", errorDesc["DMSG_INIT"]),
182+
Code: errorCode["DMSG_INIT"],
183+
}
184+
}
185+
defer closeDmsgDC()
186+
dmsgHTTP := &http.Client{Transport: dmsghttp.MakeHTTPTransport(ctx, dmsgDC)}
187+
dmsgC, closeDmsg, err = cli.StartDmsg(ctx, dlog, pk, sk, dmsgHTTP, flags.DmsgDiscAddr, flags.DmsgSessions)
188+
}
220189
}
190+
221191
if err != nil {
222192
dlog.WithError(err).Debug("Error connecting to dmsg network")
223-
// return curlError{
224-
// Error: fmt.Errorf("%s", errorDesc["DMSG_INIT"]),
225-
// Code: errorCode["DMSG_INIT"],
226-
// }
193+
return curlError{
194+
Error: fmt.Errorf("%s", errorDesc["DMSG_INIT"]),
195+
Code: errorCode["DMSG_INIT"],
196+
}
227197
}
228198
defer closeDmsg()
229199

‎cmd/dmsghttp/commands/dmsghttp.go‎

Lines changed: 66 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -23,37 +23,32 @@ import (
2323
"golang.org/x/net/proxy"
2424

2525
"github.com/skycoin/dmsg/internal/cli"
26+
"github.com/skycoin/dmsg/internal/flags"
2627
dmsg "github.com/skycoin/dmsg/pkg/dmsg"
28+
"github.com/skycoin/dmsg/pkg/dmsghttp"
2729
)
2830

2931
var (
30-
dlog = logging.MustGetLogger("dmsghttp")
31-
dmsgDisc = dmsg.DiscAddr(false)
32-
dmsgPort uint
33-
dmsgHTTPPath string
34-
dmsgSessions int
35-
logLvl string
36-
proxyAddr string
37-
sk cipher.SecKey
38-
pk cipher.PubKey
39-
serveDir string
40-
useHTTP bool
41-
wl string
42-
wlkeys []cipher.PubKey
43-
err error
32+
dlog = logging.MustGetLogger("dmsghttp")
33+
dmsgPort uint
34+
logLvl string
35+
proxyAddr string
36+
sk cipher.SecKey
37+
pk cipher.PubKey
38+
serveDir string
39+
wl []string
40+
wlkeys []cipher.PubKey
41+
err error
4442
)
4543

4644
func init() {
4745
RootCmd.Flags().SortFlags = false
48-
RootCmd.Flags().BoolVarP(&useHTTP, "http", "z", false, "use regular http to connect to dmsg discovery")
49-
RootCmd.Flags().StringVarP(&dmsgHTTPPath, "dmsgconf", "F", "", "dmsghttp-config path")
50-
RootCmd.Flags().StringVarP(&proxyAddr, "proxy", "p", proxyAddr, "connect to dmsg via proxy (i.e. '127.0.0.1:1080')")
51-
RootCmd.Flags().IntVarP(&dmsgSessions, "sess", "e", 1, "number of dmsg servers to connect to\033[0m\n\r")
46+
flags.InitFlags(RootCmd)
47+
RootCmd.Flags().StringVarP(&proxyAddr, "proxy", "p", proxyAddr, "connect to DMSG via proxy (i.e. '127.0.0.1:1080')")
5248
RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "debug", "[ debug | warn | error | fatal | panic | trace | info ]\033[0m\n\r")
5349
RootCmd.Flags().StringVarP(&serveDir, "dir", "r", ".", "local dir to serve via dmsghttp\033[0m\n\r")
54-
RootCmd.Flags().UintVarP(&dmsgPort, "port", "d", 80, "dmsg port to serve from\033[0m\n\r")
55-
RootCmd.Flags().StringVarP(&wl, "wl", "w", "", "whitelist keys to access server, comma separated")
56-
RootCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", dmsgDisc, "dmsg discovery url\033[0m\n\r")
50+
RootCmd.Flags().UintVarP(&dmsgPort, "port", "d", 80, "DMSG port to serve from\033[0m\n\r")
51+
RootCmd.Flags().StringSliceVarP(&wl, "wl", "w", []string{}, "whitelist keys to access server, comma separated")
5752
if os.Getenv("DMSGHTTP_SK") != "" {
5853
sk.Set(os.Getenv("DMSGHTTP_SK")) //nolint
5954
}
@@ -84,42 +79,33 @@ func server() {
8479
wg := new(sync.WaitGroup)
8580
wg.Add(1)
8681

87-
log := logging.MustGetLogger("dmsghttp")
88-
89-
if dmsgHTTPPath != "" {
90-
dmsg.DmsghttpJSON, err = os.ReadFile(dmsgHTTPPath) //nolint
91-
if err != nil {
92-
dlog.WithError(err).Fatal("Failed to read specified dmsghttp-config")
93-
}
94-
err = dmsg.InitConfig()
95-
if err != nil {
96-
dlog.WithError(err).Fatal("Failed to unmarshal dmsghttp-config")
97-
}
82+
err = flags.InitConfig()
83+
if err != nil {
84+
dlog.WithError(err).Fatal("Failed to read specified dmsghttp-config")
9885
}
9986

10087
pk, err = sk.PubKey()
10188
if err != nil {
10289
pk, sk = cipher.GenerateKeyPair()
10390
}
104-
if wl != "" {
105-
wlk := strings.Split(wl, ",")
106-
for _, key := range wlk {
107-
var pk1 cipher.PubKey
108-
err := pk1.Set(key)
109-
if err == nil {
110-
wlkeys = append(wlkeys, pk1)
111-
}
91+
92+
for _, key := range wl {
93+
var pk1 cipher.PubKey
94+
err := pk1.Set(key)
95+
if err == nil {
96+
wlkeys = append(wlkeys, pk1)
11297
}
11398
}
99+
114100
if len(wlkeys) > 0 {
115101
if len(wlkeys) == 1 {
116-
log.Info(fmt.Sprintf("%d key whitelisted", len(wlkeys)))
102+
dlog.Info(fmt.Sprintf("%d key whitelisted", len(wlkeys)))
117103
} else {
118-
log.Info(fmt.Sprintf("%d keys whitelisted", len(wlkeys)))
104+
dlog.Info(fmt.Sprintf("%d keys whitelisted", len(wlkeys)))
119105
}
120106
}
121107

122-
ctx, cancel := cmdutil.SignalContext(context.Background(), log)
108+
ctx, cancel := cmdutil.SignalContext(context.Background(), dlog)
123109
defer cancel()
124110

125111
httpClient := &http.Client{}
@@ -142,13 +128,45 @@ func server() {
142128
var dmsgC *dmsg.Client
143129
var closeDmsg func()
144130

145-
if useHTTP {
146-
dmsgC, closeDmsg, err = cli.StartDmsg(ctx, dlog, pk, sk, httpClient, dmsgDisc, dmsgSessions)
131+
if flags.UseDC {
132+
//TODO: implement direct client which gets the servers from dmsg-disc via http client that can be configured to use a proxy
133+
/*
134+
if flags.UseHTTP {
135+
//TODO: support proxy configuration here & in all direct client functions that interact with dmsg-disc
136+
// Need to be able to specify http client as input to dmsghttp.GetServers
137+
servers := dmsghttp.GetServers(ctx, flags.DmsgDiscURL, "", dlog)
138+
config := &dmsg.Config{
139+
MinSessions: flags.DmsgSessions,
140+
UpdateInterval: dmsg.DefaultUpdateInterval,
141+
// ConnectedServersType: dmsgServerType,
142+
}
143+
var keys cipher.PubKeys
144+
keys = append(keys, pk)
145+
dClient := direct.NewClient(direct.GetAllEntries(keys, servers), log)
146+
147+
dmsgC, closeDmsg, err := direct.StartDmsg(ctx, log, pk, sk, dClient, config)
148+
} else {
149+
*/
150+
dmsgC, closeDmsg, err = cli.StartDmsgDirect(ctx, dlog, pk, sk, httpClient, "", flags.DmsgSessions, pk.String())
151+
// }
147152
} else {
148-
dmsgC, closeDmsg, err = cli.StartDmsgDirect(ctx, dlog, pk, sk, httpClient, dmsgDisc, dmsgSessions, pk.String())
153+
if flags.UseHTTP {
154+
dmsgC, closeDmsg, err = cli.StartDmsg(ctx, dlog, pk, sk, httpClient, flags.DmsgDiscURL, flags.DmsgSessions)
155+
} else {
156+
var dmsgDC *dmsg.Client
157+
var closeDmsgDC func()
158+
dmsgDC, closeDmsgDC, err = cli.StartDmsgDirect(ctx, dlog, pk, sk, httpClient, "", flags.DmsgSessions, dmsg.ExtractPKFromDmsgAddr(flags.DmsgDiscAddr))
159+
if err != nil {
160+
dlog.WithError(err).Error("Error connecting to dmsg network")
161+
return
162+
}
163+
defer closeDmsgDC()
164+
dmsgHTTP := &http.Client{Transport: dmsghttp.MakeHTTPTransport(ctx, dmsgDC)}
165+
dmsgC, closeDmsg, err = cli.StartDmsg(ctx, dlog, pk, sk, dmsgHTTP, flags.DmsgDiscAddr, flags.DmsgSessions)
166+
}
149167
}
150168
if err != nil {
151-
dlog.WithError(err).Debug("Error connecting to dmsg network")
169+
dlog.WithError(err).Error("Error connecting to dmsg network")
152170
return
153171
}
154172
defer closeDmsg()
@@ -186,7 +204,7 @@ func server() {
186204

187205
// Start serving
188206
go func() {
189-
log.WithField("dmsg_addr", lis.Addr().String()).Debug("Serving...\n")
207+
dlog.WithField("dmsg_addr", lis.Addr().String()).Debug("Serving...\n")
190208
if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed {
191209
dlog.WithError(err).Debug("Server error\n")
192210
}

0 commit comments

Comments
 (0)