fix alerting per cluster on slack and pushover

Author: emmaloubersac

cluster/cluster.go

@@ -20,13 +20,15 @@ import (
 	"time"
 
 	"github.com/BurntSushi/toml"
+	"github.com/bluele/logrus_slack"
 	"github.com/signal18/replication-manager/cluster/configurator"
 	"github.com/signal18/replication-manager/cluster/nbc"
 	"github.com/signal18/replication-manager/config"
 	v3 "github.com/signal18/replication-manager/repmanv3"
 	"github.com/signal18/replication-manager/router/maxscale"
 	"github.com/signal18/replication-manager/utils/cron"
 	"github.com/signal18/replication-manager/utils/dbhelper"
+	"github.com/signal18/replication-manager/utils/logrus/hooks/pushover"
 	"github.com/signal18/replication-manager/utils/s18log"
 	"github.com/signal18/replication-manager/utils/state"
 	log "github.com/sirupsen/logrus"
@@ -95,7 +97,9 @@ type Cluster struct {
 	DBIndexSize                   int64                       `json:"dbIndexSize"`
 	Connections                   int                         `json:"connections"`
 	QPS                           int64                       `json:"qps"`
+	LogVault                      *log.Logger                 `json:"-"`
 	Log                           s18log.HttpLog              `json:"log"`
+	LogSlack                      *log.Logger                 `json:"-"`
 	JobResults                    map[string]*JobResult       `json:"jobResults"`
 	Grants                        map[string]string           `json:"-"`
 	tlog                          *s18log.TermLog             `json:"-"`
@@ -252,7 +256,7 @@ const (
 )
 
 // Init initial cluster definition
-func (cluster *Cluster) Init(conf config.Config, cfgGroup string, tlog *s18log.TermLog, log *s18log.HttpLog, termlength int, runUUID string, repmgrVersion string, repmgrHostname string, key []byte) error {
+func (cluster *Cluster) Init(conf config.Config, cfgGroup string, tlog *s18log.TermLog, loghttp *s18log.HttpLog, termlength int, runUUID string, repmgrVersion string, repmgrHostname string, key []byte) error {
 	cluster.SqlErrorLog = logsql.New()
 	cluster.SqlGeneralLog = logsql.New()
 	cluster.crcTable = crc64.MakeTable(crc64.ECMA) // http://golang.org/pkg/hash/crc64/#pkg-constants
@@ -270,7 +274,7 @@ func (cluster *Cluster) Init(conf config.Config, cfgGroup string, tlog *s18log.T
 	cluster.testStopCluster = true
 	cluster.testStartCluster = true
 	cluster.tlog = tlog
-	cluster.htlog = log
+	cluster.htlog = loghttp
 	cluster.termlength = termlength
 	cluster.Name = cfgGroup
 	cluster.WorkingDir = conf.WorkingDir + "/" + cluster.Name
@@ -307,6 +311,29 @@ func (cluster *Cluster) Init(conf config.Config, cfgGroup string, tlog *s18log.T
 		os.MkdirAll(cluster.Conf.WorkingDir+"/"+cluster.Name, os.ModePerm)
 	}
 
+	cluster.LogVault = log.New()
+
+	if cluster.Conf.PushoverAppToken != "" && cluster.Conf.PushoverUserToken != "" {
+		cluster.LogVault.AddHook(
+			pushover.NewHook(cluster.Conf.PushoverAppToken, cluster.Conf.PushoverUserToken),
+		)
+		cluster.LogVault.SetLevel(log.WarnLevel)
+	}
+
+	cluster.LogSlack = log.New()
+
+	if cluster.Conf.SlackURL != "" {
+		cluster.LogSlack.AddHook(&logrus_slack.SlackHook{
+			HookURL:        cluster.Conf.SlackURL,
+			AcceptedLevels: logrus_slack.LevelThreshold(log.WarnLevel),
+			Channel:        cluster.Conf.SlackChannel,
+			IconEmoji:      ":ghost:",
+			Username:       cluster.Conf.SlackUser,
+			Timeout:        5 * time.Second, // request timeout for calling slack api
+		})
+	}
+	cluster.LogPrintf("ALERT", "Replication manager init cluster version : %s", cluster.Conf.Version)
+
 	hookerr, err := s18log.NewRotateFileHook(s18log.RotateFileConfig{
 		Filename:   cluster.WorkingDir + "/sql_error.log",
 		MaxSize:    cluster.Conf.LogRotateMaxSize,

cluster/cluster_log.go

@@ -171,18 +171,32 @@ func (cluster *Cluster) LogPrintf(level string, format string, args ...interface
 		switch level {
 		case "ERROR":
 			log.WithField("cluster", cluster.Name).Errorf(cliformat, args...)
+			if cluster.Conf.SlackURL != "" {
+				cluster.LogSlack.WithFields(log.Fields{"cluster": cluster.Name, "type": "alert"}).Errorf(cliformat, args...)
+			}
+
 		case "INFO":
 			log.WithField("cluster", cluster.Name).Infof(cliformat, args...)
 		case "DEBUG":
 			log.WithField("cluster", cluster.Name).Debugf(cliformat, args...)
 		case "WARN":
 			log.WithField("cluster", cluster.Name).Warnf(cliformat, args...)
+			if cluster.Conf.SlackURL != "" {
+				cluster.LogSlack.WithFields(log.Fields{"cluster": cluster.Name, "type": "alert"}).Errorf(cliformat, args...)
+			}
 		case "TEST":
 			log.WithFields(log.Fields{"cluster": cluster.Name, "type": "test"}).Infof(cliformat, args...)
 		case "BENCH":
 			log.WithFields(log.Fields{"cluster": cluster.Name, "type": "benchmark"}).Infof(cliformat, args...)
 		case "ALERT":
-			log.WithFields(log.Fields{"cluster": cluster.Name, "type": "alert"}).Warnf(cliformat, args...)
+			log.WithFields(log.Fields{"cluster": cluster.Name, "type": "alert"}).Errorf(cliformat, args...)
+			if cluster.Conf.SlackURL != "" {
+				cluster.LogSlack.WithFields(log.Fields{"cluster": cluster.Name, "type": "alert"}).Errorf(cliformat, args...)
+			}
+			if cluster.Conf.PushoverAppToken != "" && cluster.Conf.PushoverUserToken != "" {
+				cluster.LogVault.WithFields(log.Fields{"cluster": cluster.Name, "type": "alert"}).Errorf(cliformat, args...)
+
+			}
 		case "STATE":
 			status := cliformat[0:6]
 			code := cliformat[7:15]