Skip to content

Commit 0a2570a

Browse files
ziggie1984ziggie1984
committed
collectors: tolerate DeadlineExceeded in all scrape-cycle RPCs
GetInfo and other RPC calls are known to occasionally take longer than expected due to database interactions on the lnd side. A transient DeadlineExceeded during a Prometheus scrape should not cause lndmon to exit. All collectors now log the error and skip the current scrape cycle instead of propagating to errChan, mirroring the existing behaviour in ChainCollector and PeerCollector. Affected collectors: InfoCollector, ChannelsCollector, WalletCollector, GraphCollector, WtClientCollector.
1 parent 79d4936 commit 0a2570a

5 files changed

Lines changed: 97 additions & 23 deletions

File tree

‎collectors/channels_collector.go‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,12 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error,
202202
for {
203203
err := collector.refreshClosedChannelsCache()
204204
if err != nil {
205-
errChan <- err
205+
Logger.Errorf("ChannelsCollector refreshClosedChannelsCache "+
206+
"failed with: %v", err)
207+
208+
if !IsDeadlineExceeded(err) {
209+
errChan <- err
210+
}
206211
}
207212

208213
select {
@@ -293,8 +298,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
293298
// pending channel balances.
294299
chanBalResp, err := c.lnd.ChannelBalance(context.Background())
295300
if err != nil {
296-
c.errChan <- fmt.Errorf("ChannelsCollector ChannelBalance "+
297-
"failed with: %v", err)
301+
errWithContext := fmt.Errorf("ChannelsCollector ChannelBalance "+
302+
"failed with: %w", err)
303+
Logger.Error(errWithContext)
304+
305+
if !IsDeadlineExceeded(err) {
306+
c.errChan <- errWithContext
307+
}
308+
298309
return
299310
}
300311

@@ -341,8 +352,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
341352
// as well as the number of pending HTLCs.
342353
listChannelsResp, err := c.lnd.ListChannels(context.Background(), false, false)
343354
if err != nil {
344-
c.errChan <- fmt.Errorf("ChannelsCollector ListChannels "+
345-
"failed with: %v", err)
355+
errWithContext := fmt.Errorf("ChannelsCollector ListChannels "+
356+
"failed with: %w", err)
357+
Logger.Error(errWithContext)
358+
359+
if !IsDeadlineExceeded(err) {
360+
c.errChan <- errWithContext
361+
}
362+
346363
return
347364
}
348365

@@ -452,8 +469,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
452469
// Get the list of pending channels
453470
pendingChannelsResp, err := c.lnd.PendingChannels(context.Background())
454471
if err != nil {
455-
c.errChan <- fmt.Errorf("ChannelsCollector PendingChannels "+
456-
"failed with: %v", err)
472+
errWithContext := fmt.Errorf("ChannelsCollector PendingChannels "+
473+
"failed with: %w", err)
474+
Logger.Error(errWithContext)
475+
476+
if !IsDeadlineExceeded(err) {
477+
c.errChan <- errWithContext
478+
}
479+
457480
return
458481
}
459482
ch <- prometheus.MustNewConstMetric(
@@ -539,8 +562,14 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
539562
// Get all remote policies
540563
remotePolicies, err := c.getRemotePolicies(getInfoResp.IdentityPubkey)
541564
if err != nil {
542-
c.errChan <- fmt.Errorf("ChannelsCollector getRemotePolicies "+
543-
"failed with: %v", err)
565+
errWithContext := fmt.Errorf("ChannelsCollector getRemotePolicies "+
566+
"failed with: %w", err)
567+
Logger.Error(errWithContext)
568+
569+
if !IsDeadlineExceeded(err) {
570+
c.errChan <- errWithContext
571+
}
572+
544573
return
545574
}
546575

‎collectors/graph_collector.go‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,14 @@ func (g *GraphCollector) Describe(ch chan<- *prometheus.Desc) {
322322
func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) {
323323
resp, err := g.lnd.DescribeGraph(context.Background(), false)
324324
if err != nil {
325-
g.errChan <- fmt.Errorf("GraphCollector DescribeGraph failed "+
326-
"with: %v", err)
325+
errWithContext := fmt.Errorf("GraphCollector DescribeGraph failed "+
326+
"with: %w", err)
327+
Logger.Error(errWithContext)
328+
329+
if !IsDeadlineExceeded(err) {
330+
g.errChan <- errWithContext
331+
}
332+
327333
return
328334
}
329335

@@ -340,8 +346,14 @@ func (g *GraphCollector) Collect(ch chan<- prometheus.Metric) {
340346

341347
networkInfo, err := g.lnd.NetworkInfo(context.Background())
342348
if err != nil {
343-
g.errChan <- fmt.Errorf("GraphCollector NetworkInfo failed "+
344-
"with: %v", err)
349+
errWithContext := fmt.Errorf("GraphCollector NetworkInfo failed "+
350+
"with: %w", err)
351+
Logger.Error(errWithContext)
352+
353+
if !IsDeadlineExceeded(err) {
354+
g.errChan <- errWithContext
355+
}
356+
345357
return
346358
}
347359

‎collectors/info_collector.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,17 @@ func (c *InfoCollector) Describe(ch chan<- *prometheus.Desc) {
5050
func (c *InfoCollector) Collect(ch chan<- prometheus.Metric) {
5151
resp, err := c.lnd.GetInfo(context.Background())
5252
if err != nil {
53-
c.errChan <- fmt.Errorf("InfoCollector GetInfo failed with: "+
54-
"%v", err)
53+
errWithContext := fmt.Errorf("InfoCollector GetInfo failed with: "+
54+
"%w", err)
55+
Logger.Error(errWithContext)
56+
57+
// A deadline exceeded is expected if lnd is temporarily slow
58+
// to respond (e.g. due to database load). We just skip this
59+
// scrape cycle and let Prometheus retry on the next interval.
60+
if !IsDeadlineExceeded(err) {
61+
c.errChan <- errWithContext
62+
}
63+
5564
return
5665
}
5766

‎collectors/wallet_collector.go‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
115115
context.Background(), 0, math.MaxInt32,
116116
)
117117
if err != nil {
118-
u.errChan <- fmt.Errorf("WalletCollector ListUnspent failed "+
119-
"with: %v", err)
118+
errWithContext := fmt.Errorf("WalletCollector ListUnspent failed "+
119+
"with: %w", err)
120+
Logger.Error(errWithContext)
121+
122+
if !IsDeadlineExceeded(err) {
123+
u.errChan <- errWithContext
124+
}
125+
120126
return
121127
}
122128

@@ -171,8 +177,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
171177
// balance at this instance.
172178
walletBal, err := u.lnd.Client.WalletBalance(context.Background())
173179
if err != nil {
174-
u.errChan <- fmt.Errorf("WalletCollector WalletBalance "+
175-
"failed with: %v", err)
180+
errWithContext := fmt.Errorf("WalletCollector WalletBalance "+
181+
"failed with: %w", err)
182+
Logger.Error(errWithContext)
183+
184+
if !IsDeadlineExceeded(err) {
185+
u.errChan <- errWithContext
186+
}
187+
176188
return
177189
}
178190

@@ -187,8 +199,14 @@ func (u *WalletCollector) Collect(ch chan<- prometheus.Metric) {
187199

188200
accounts, err := u.lnd.WalletKit.ListAccounts(context.Background(), "", 0)
189201
if err != nil {
190-
u.errChan <- fmt.Errorf("WalletCollector ListAccounts"+
191-
"failed with: %v", err)
202+
errWithContext := fmt.Errorf("WalletCollector ListAccounts "+
203+
"failed with: %w", err)
204+
Logger.Error(errWithContext)
205+
206+
if !IsDeadlineExceeded(err) {
207+
u.errChan <- errWithContext
208+
}
209+
192210
return
193211
}
194212

‎collectors/wt_client_collector.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,14 @@ func (c *WtClientCollector) Collect(ch chan<- prometheus.Metric) {
7575
return
7676
}
7777

78-
c.errChan <- fmt.Errorf("WtClientCollector ListTowers failed "+
79-
"with: %v", err)
78+
errWithContext := fmt.Errorf("WtClientCollector ListTowers failed "+
79+
"with: %w", err)
80+
Logger.Error(errWithContext)
81+
82+
if !IsDeadlineExceeded(err) {
83+
c.errChan <- errWithContext
84+
}
85+
8086
return
8187
}
8288

0 commit comments

Comments
 (0)