Add Critical Logging for Low Disk Space Condition

healthcheck/fdcheck.go

@@ -0,0 +1,26 @@
+package healthcheck
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"syscall"
+)
+
+// CheckFileDescriptors checks if there are any free file descriptors available.
+// It returns an error if no free file descriptors are available or if an unexpected error occurs.
+func CheckFileDescriptors() error {
+	// Attempt to open /dev/null to test for available file descriptors
+	fd, err := os.OpenFile(os.DevNull, os.O_RDONLY, 0)
+	if err != nil {
+		// Check if the error is due to "too many open files"
+		if errors.Is(err, syscall.EMFILE) {
+			return fmt.Errorf("no free file descriptors available")
+		}
+
+		return fmt.Errorf("error checking file descriptors: %w", err)
+	}
+
+	fd.Close()
+	return nil
+}

lncfg/healthcheck.go

@@ -29,6 +29,8 @@ type HealthCheckConfig struct {
 
 	DiskCheck *DiskCheckConfig `group:"diskspace" namespace:"diskspace"`
 
+	FileDescriptorCheck *DiskCheckConfig `group:"file_descriptor" namespace:"file_descriptor"`
+
 	TLSCheck *CheckConfig `group:"tls" namespace:"tls"`
 
 	TorConnection *CheckConfig `group:"torconnection" namespace:"torconnection"`
@@ -48,6 +50,10 @@ func (h *HealthCheckConfig) Validate() error {
 		return err
 	}
 
+	if err := h.FileDescriptorCheck.validate("file descriptor"); err != nil {
+		return err
+	}
+
 	if err := h.TLSCheck.validate("tls"); err != nil {
 		return err
 	}

server.go

@@ -2022,6 +2022,14 @@ func (s *server) createLivenessMonitor(cfg *Config, cc *chainreg.ChainControl,
 				return nil
 			}
 
+			// Define the critical threshold (e.g., 5%)
+			const criticalThreshold = 0.05
+
+			// If free space is lesser than critical value, log a warning
+			if free < criticalThreshold {
+				srvrLog.Errorf("Disk space low: %.1f%% free space remaining", free*100)
+			}
+
 			return fmt.Errorf("require: %v free space, got: %v",
 				cfg.HealthChecks.DiskCheck.RequiredRemaining,
 				free)
@@ -2032,6 +2040,22 @@ func (s *server) createLivenessMonitor(cfg *Config, cc *chainreg.ChainControl,
 		cfg.HealthChecks.DiskCheck.Attempts,
 	)
 
+	// Add file descriptor check
+	fdCheck := healthcheck.NewObservation(
+		"file descriptors",
+		func() error {
+			if err := healthcheck.CheckFileDescriptors(); err != nil {
+				srvrLog.Criticalf("CRITICAL: No free file descriptors available: %v", err)
+				return err
+			}
+			return nil
+		},
+		cfg.HealthChecks.FileDescriptorCheck.Interval,
+		cfg.HealthChecks.FileDescriptorCheck.Timeout,
+		cfg.HealthChecks.FileDescriptorCheck.Backoff,
+		cfg.HealthChecks.FileDescriptorCheck.Attempts,
+	)
+
 	tlsHealthCheck := healthcheck.NewObservation(
 		"tls",
 		func() error {
@@ -2057,7 +2081,7 @@ func (s *server) createLivenessMonitor(cfg *Config, cc *chainreg.ChainControl,
 	)
 
 	checks := []*healthcheck.Observation{
-		chainHealthCheck, diskCheck, tlsHealthCheck,
+		chainHealthCheck, diskCheck, fdCheck, tlsHealthCheck,
 	}
 
 	// If Tor is enabled, add the healthcheck for tor connection.