monit-nb: add module

nix-bitcoin monitoring via monit

Author: nixbitcoin

README.md

@@ -94,6 +94,7 @@ NixOS modules ([src](modules/modules.nix))
   * [netns-isolation](modules/netns-isolation.nix): isolates applications on the network-level via network namespaces
   * [nodeinfo](modules/nodeinfo.nix): script which prints info about the node's services
   * [backups](modules/backups.nix): duplicity backups of all your node's important files
+  * [monitoring](modules/monit-nb.nix): pro-active monitoring for nix-bitcoin
   * [operator](modules/operator.nix): adds non-root user `operator` who has access to client tools (e.g. `bitcoin-cli`, `lightning-cli`)
 
 Security

examples/configuration.nix

@@ -236,6 +236,13 @@
   # and electrs data directory, enable
   # services.backups.with-bulk-data = true;
 
+  ### Monitoring
+  # Set this to enable monit-nb, nix-bitcoin's own monitoring service. It will
+  # create a user `monitmail` with password $secretsDir/monitmail-password,
+  # which you can use to connect over IMAP using the monit-nb onion service and
+  # your favorite mail client with Tor proxy support.
+  # services.monit-nb.enable = true;
+
   ### netns-isolation (EXPERIMENTAL)
   # Enable this module to use Network Namespace Isolation. This feature places
   # every service in its own network namespace and only allows truly necessary

modules/modules.nix

@@ -35,6 +35,7 @@
     ./netns-isolation.nix
     ./nodeinfo.nix
     ./backups.nix
+    ./monit-nb.nix
   ];
 
   disabledModules = [ "services/networking/bitcoind.nix" ];

modules/monit-nb.nix

@@ -0,0 +1,180 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  options.services.monit-nb = {
+    enable = mkEnableOption "nix-bitcoin monitoring via monit";
+    address = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = "Dovecot server address.";
+    };
+    port = mkOption {
+      type = types.port;
+      default = 143;
+      description = "Dovecot server port.";
+    };
+  };
+
+  cfg = config.services.monit-nb;
+  nbLib = config.nix-bitcoin.lib;
+  secretsDir = config.nix-bitcoin.secretsDir;
+
+  checkStatus = pkgs.writeScriptBin "checkStatus" ''
+    #! /bin/sh
+    if [[ $(systemctl show --property=ActiveState $1) == *"=active"* ]];
+      then exit 0
+      else exit 1
+    fi
+  '';
+
+  configFile = ''
+    set alert monitmail@localhost
+    set daemon 120 with start delay 60
+    set mailserver
+      localhost
+
+    set httpd unixsocket /var/run/monit.sock
+      uid root
+      gid root
+      permission 660
+      # Placeholder username & password
+      # Secured through unix socket permissions
+      allow admin:obwjoawijerfoijsiwfj29jf2f2jd
+
+    check filesystem root with path /
+      if space usage > 80% then alert
+      if inode usage > 80% then alert
+
+    check system $HOST
+      if cpu usage > 95% for 10 cycles then alert
+      if memory usage > 75% for 5 cycles then alert
+      if swap usage > 20% for 10 cycles then alert
+      if loadavg (1min) > 90 for 15 cycles then alert
+      if loadavg (5min) > 80 for 10 cycles then alert
+      if loadavg (15min) > 70 for 8 cycles then alert
+
+    check program duplicity path "${pkgs.systemd}/bin/systemctl is-failed duplicity"
+      if changed status then alert
+
+    check program bitcoind path "${checkStatus}/bin/checkStatus bitcoind"
+      if changed status then alert
+
+    check program bitcoind-import-banlist path "${pkgs.systemd}/bin/systemctl is-failed bitcoind-import-banlist"
+      if changed status then alert
+
+    ${optionalString config.services.btcpayserver.enable ''
+      check program nbxplorer path "${checkStatus}/bin/checkStatus nbxplorer"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.btcpayserver.enable ''
+      check program btcpayserver path "${checkStatus}/bin/checkStatus nbxplorer"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.charge-lnd.enable ''
+      check program charge-lnd path "${checkStatus}/bin/checkStatus charge-lnd"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.clightning.enable ''
+      check program clightning path "${checkStatus}/bin/checkStatus clightning"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.electrs.enable ''
+      check program electrs path "${checkStatus}/bin/checkStatus electrs"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.joinmarket.enable ''
+      check program joinmarket path "${checkStatus}/bin/checkStatus joinmarket"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.joinmarket.yieldgenerator.enable ''
+      check program joinmarket-yieldgenerator path "${checkStatus}/bin/checkStatus joinmarket-yieldgenerator"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.joinmarket-ob-watcher.enable ''
+      check program joinmarket-ob-watcher path "${checkStatus}/bin/checkStatus joinmarket-ob-watcher"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.lightning-loop.enable ''
+      check program lightning-loop path "${checkStatus}/bin/checkStatus lightning-loop"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.lightning-pool.enable ''
+      check program lightning-pool path "${checkStatus}/bin/checkStatus lightning-pool"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.liquidd.enable ''
+      check program liquidd path "${checkStatus}/bin/checkStatus liquidd"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.lnd.enable ''
+      check program lnd path "${checkStatus}/bin/checkStatus lnd"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.clightning-rest.enable ''
+      check program clightning-rest path "${checkStatus}/bin/checkStatus clightning-rest"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.rtl.enable ''
+      check program rtl path "${checkStatus}/bin/checkStatus rtl"
+        if changed status then alert
+    ''}
+
+    ${optionalString config.services.spark-wallet.enable ''
+      check program spark-wallet path "${checkStatus}/bin/checkStatus spark-wallet"
+        if changed status then alert
+    ''}
+  '';
+
+in {
+  inherit options;
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.monit ];
+
+    environment.etc.monitrc = {
+      text = "${configFile}";
+      mode = "0400";
+    };
+
+    systemd.services.monit-nb = {
+      description = "Pro-active monitoring utility for nix-bitcoin";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.monit}/bin/monit -I -c /etc/monitrc";
+        ExecStop = "${pkgs.monit}/bin/monit -c /etc/monitrc quit";
+        ExecReload = "${pkgs.monit}/bin/monit -c /etc/monitrc reload";
+        KillMode = "process";
+        Restart = "always";
+      } // nbLib.allowLocalIPAddresses;
+      restartTriggers = [ config.environment.etc.monitrc.source ];
+    };
+
+    services.postfix.enable = true;
+    services.dovecot2.enable = true;
+    users.users.monitmail = {
+      passwordFile = "${secretsDir}/monitmail-password";
+      isNormalUser = true;
+    };
+
+    nix-bitcoin.secrets.monitmail-password.user = "root";
+    nix-bitcoin.generateSecretsCmds.monitmail = ''
+      makePasswordSecret monitmail-password
+    '';
+  };
+}
+

modules/nodeinfo.nix

@@ -34,6 +34,7 @@ let
     liquidd = mkInfo "";
     joinmarket-ob-watcher = mkInfo "";
     rtl = mkInfo "";
+    monit-nb = mkInfo "";
     # Only add sshd when it has an onion service
     sshd = name: cfg: mkIfOnionPort "sshd" (onionPort: ''
       add_service("sshd", """set_onion_address(info, "sshd", ${onionPort})""")

modules/presets/enable-tor.nix

@@ -49,5 +49,6 @@ in {
     spark-wallet.enable = defaultTrue;
     joinmarket-ob-watcher.enable = defaultTrue;
     rtl.enable = defaultTrue;
+    monit-nb.enable = defaultTrue;
   };
 }

test/tests.nix

@@ -23,7 +23,7 @@ let
         nix-bitcoin.secretsDir = "/secrets";
         nix-bitcoin.generateSecrets = true;
         nix-bitcoin.operator.enable = true;
-        environment.systemPackages = with pkgs; [ jq ];
+        environment.systemPackages = with pkgs; [ jq mailutils ];
       }
     ];
 
@@ -116,6 +116,8 @@ let
 
       tests.backups = cfg.backups.enable;
 
+      tests.monit-nb = cfg.monit-nb.enable;
+
       # To test that unused secrets are made inaccessible by 'setup-secrets'
       systemd.services.setup-secrets.preStart = mkIfTest "security" ''
         install -D -o nobody -g nogroup -m777 <(:) /secrets/dummy
@@ -179,6 +181,7 @@ let
       services.joinmarket.enable = true;
       services.joinmarket-ob-watcher.enable = true;
       services.backups.enable = true;
+      services.monit-nb.enable = true;
 
       nix-bitcoin.nodeinfo.enable = true;
 

test/tests.py

@@ -262,6 +262,17 @@ def _():
     info = json.loads(json_info)
     assert info["bitcoind"]["local_address"]
 
+@test("monit-nb")
+def _():
+    assert_running("monit-nb")
+    machine.wait_until_succeeds(
+      log_has_string("monit-nb", "Monit 5.29.0 started")
+    )
+    assert_matches(
+      "runuser -u monitmail -- mail -H",
+      "monit alert",
+    )
+
 @test("secure-node")
 def _():
     assert_running("onion-addresses")