Skip to content

Commit ed46434

Browse files
djkazicdjkazic
authored
Merge pull request #127 from lightninglabs/fix/deadline-exceeded-collectors
collectors: tolerate DeadlineExceeded in all scrape-cycle RPCs
2 parents 79d4936 + 0a2570a commit ed46434

5 files changed

Lines changed: 97 additions & 23 deletions

File tree

‎collectors/channels_collector.go‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,12 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error,
202202
for {
203203
err := collector.refreshClosedChannelsCache()
204204
if err != nil {
205-
errChan <- err
205+
Logger.Errorf("ChannelsCollector refreshClosedChannelsCache "+
206+
"failed with: %v", err)
207+
208+
if !IsDeadlineExceeded(err) {
209+
errChan <- err
210+
}
206211
}
207212

208213
select {
@@ -293,8 +298,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
293298
// pending channel balances.
294299
chanBalResp, err := c.lnd.ChannelBalance(context.Background())
295300
if err != nil {
296-
c.errChan <- fmt.Errorf("ChannelsCollector ChannelBalance "+
297-
"failed with: %v", err)
301+
errWithContext := fmt.Errorf("ChannelsCollector ChannelBalance "+
302+
"failed with: %w", err)
303+
Logger.Error(errWithContext)
304+
305+
if !IsDeadlineExceeded(err) {
306+
c.errChan <- errWithContext
307+
}
308+
298309
return
299310
}
300311

@@ -341,8 +352,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
341352
// as well as the number of pending HTLCs.
342353
listChannelsResp, err := c.lnd.ListChannels(context.Background(), false, false)
343354
if err != nil {
344-
c.errChan <- fmt.Errorf("ChannelsCollector ListChannels "+
345-
"failed with: %v", err)
355+
errWithContext := fmt.Errorf("ChannelsCollector ListChannels "+
356+
"failed with: %w", err)
357+
Logger.Error(errWithContext)
358+
359+
if !IsDeadlineExceeded(err) {
360+
c.errChan <- errWithContext
361+
}
362+
346363
return
347364
}
348365

@@ -452,8 +469,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
452469
// Get the list of pending channels
453470
pendingChannelsResp, err := c.lnd.PendingChannels(context.Background())
454471
if err != nil {
455-
c.errChan <- fmt.Errorf("ChannelsCollector PendingChannels "+
456-
"failed with: %v", err)
472+
errWithContext := fmt.Errorf("ChannelsCollector PendingChannels "+
473+
"failed with: %w", err)
474+
Logger.Error(errWithContext)
475+
476+
if !IsDeadlineExceeded(err) {
477+
c.errChan <- errWithContext
478+
}
479+
457480
return
458481
}
459482
ch <- prometheus.MustNewConstMetric(
@@ -539,8 +562,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
539562
// Get all remote policies
540563
remotePolicies, err := c.getRemotePolicies(getInfoResp.IdentityPubkey)
541564
if err != nil {
542-
c.errChan <- fmt.Errorf("ChannelsCollector getRemotePolicies "+
543-
"failed with: %v", err)
565+
errWithContext := fmt.Errorf("ChannelsCollector getRemotePolicies "+
566+
"failed with: %w", err)
567+
Logger.Error(errWithContext)
568+
569+
if !IsDeadlineExceeded(err) {
570+
c.errChan <- errWithContext
571+
}
572+
544573
return
545574
}
546575

‎collectors/graph_collector.go‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,14 @@ func (g *GraphCollector) Describe(ch chan<- *prometheus.Desc) {
322322
func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) {
323323
resp, err := g.lnd.DescribeGraph(context.Background(), false)
324324
if err != nil {
325-
g.errChan <- fmt.Errorf("GraphCollector DescribeGraph failed "+
326-
"with: %v", err)
325+
errWithContext := fmt.Errorf("GraphCollector DescribeGraph failed "+
326+
"with: %w", err)
327+
Logger.Error(errWithContext)
328+
329+
if !IsDeadlineExceeded(err) {
330+
g.errChan <- errWithContext
331+
}
332+
327333
return
328334
}
329335

@@ -340,8 +346,14 @@ func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) {
340346

341347
networkInfo, err := g.lnd.NetworkInfo(context.Background())
342348
if err != nil {
343-
g.errChan <- fmt.Errorf("GraphCollector NetworkInfo failed "+
344-
"with: %v", err)
349+
errWithContext := fmt.Errorf("GraphCollector NetworkInfo failed "+
350+
"with: %w", err)
351+
Logger.Error(errWithContext)
352+
353+
if !IsDeadlineExceeded(err) {
354+
g.errChan <- errWithContext
355+
}
356+
345357
return
346358
}
347359

‎collectors/info_collector.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,17 @@ func (c *InfoCollector) Describe(ch chan<- *prometheus.Desc) {
5050
func (c *InfoCollector) Collect(ch chan<- prometheus.Metric) {
5151
resp, err := c.lnd.GetInfo(context.Background())
5252
if err != nil {
53-
c.errChan <- fmt.Errorf("InfoCollector GetInfo failed with: "+
54-
"%v", err)
53+
errWithContext := fmt.Errorf("InfoCollector GetInfo failed with: "+
54+
"%w", err)
55+
Logger.Error(errWithContext)
56+
57+
// A deadline exceeded is expected if lnd is temporarily slow
58+
// to respond (e.g. due to database load). We just skip this
59+
// scrape cycle and let Prometheus retry on the next interval.
60+
if !IsDeadlineExceeded(err) {
61+
c.errChan <- errWithContext
62+
}
63+
5564
return
5665
}
5766

‎collectors/wallet_collector.go‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
115115
context.Background(), 0, math.MaxInt32,
116116
)
117117
if err != nil {
118-
u.errChan <- fmt.Errorf("WalletCollector ListUnspent failed "+
119-
"with: %v", err)
118+
errWithContext := fmt.Errorf("WalletCollector ListUnspent failed "+
119+
"with: %w", err)
120+
Logger.Error(errWithContext)
121+
122+
if !IsDeadlineExceeded(err) {
123+
u.errChan <- errWithContext
124+
}
125+
120126
return
121127
}
122128

@@ -171,8 +177,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
171177
// balance at this instance.
172178
walletBal, err := u.lnd.Client.WalletBalance(context.Background())
173179
if err != nil {
174-
u.errChan <- fmt.Errorf("WalletCollector WalletBalance "+
175-
"failed with: %v", err)
180+
errWithContext := fmt.Errorf("WalletCollector WalletBalance "+
181+
"failed with: %w", err)
182+
Logger.Error(errWithContext)
183+
184+
if !IsDeadlineExceeded(err) {
185+
u.errChan <- errWithContext
186+
}
187+
176188
return
177189
}
178190

@@ -187,8 +199,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
187199

188200
accounts, err := u.lnd.WalletKit.ListAccounts(context.Background(), "", 0)
189201
if err != nil {
190-
u.errChan <- fmt.Errorf("WalletCollector ListAccounts"+
191-
"failed with: %v", err)
202+
errWithContext := fmt.Errorf("WalletCollector ListAccounts "+
203+
"failed with: %w", err)
204+
Logger.Error(errWithContext)
205+
206+
if !IsDeadlineExceeded(err) {
207+
u.errChan <- errWithContext
208+
}
209+
192210
return
193211
}
194212

‎collectors/wt_client_collector.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,14 @@ func (c *WtClientCollector) Collect(ch chan<- prometheus.Metric) {
7575
return
7676
}
7777

78-
c.errChan <- fmt.Errorf("WtClientCollector ListTowers failed "+
79-
"with: %v", err)
78+
errWithContext := fmt.Errorf("WtClientCollector ListTowers failed "+
79+
"with: %w", err)
80+
Logger.Error(errWithContext)
81+
82+
if !IsDeadlineExceeded(err) {
83+
c.errChan <- errWithContext
84+
}
85+
8086
return
8187
}
8288

0 commit comments

Comments
 (0)