Add alarm rule profile monitoring

Author: jwping

cmd/prometheus/main.go

@@ -434,6 +434,8 @@ func main() {
 		conntrack.DialWithTracing(),
 	)
 
+	cMonitoring := config.NewCMonitoring()
+
 	reloaders := []func(cfg *config.Config) error{
 		remoteStorage.ApplyConfig,
 		webHandler.ApplyConfig,
@@ -479,6 +481,7 @@ func main() {
 				}
 				files = append(files, fs...)
 			}
+			go cMonitoring.JudgeChange(files)
 			return ruleManager.Update(
 				time.Duration(cfg.GlobalConfig.EvaluationInterval),
 				files,
@@ -762,8 +765,9 @@ func main() {
 		)
 	}
 
-	// 添加配置文件监听器
-	if cfg.monitorConfig {
+	{
+		// 添加配置文件监听器
+		// if cfg.monitorConfig {
 		cancel := make(chan struct{})
 		g.Add(
 			func() error {
@@ -773,10 +777,6 @@ func main() {
 					return errors.New("Profile listener creation failed: " + err.Error())
 				}
 				defer watch.Close()
-				//添加要监控的对象，文件或文件夹
-				if err = watch.Add(cfg.configFile); err != nil {
-					return errors.New("Failed to listen to profile err: " + err.Error())
-				}
 
 				level.Info(logger).Log("msg", "Profile listening on...")
 				for {
@@ -797,15 +797,30 @@ func main() {
 								level.Error(logger).Log("Error reloading config err: " + err.Error())
 							}
 
-							if err = watch.Add(cfg.configFile); err != nil {
-								return errors.New("Failed to listen to profile err: " + err.Error())
-							}
-
 						}
 					case err := <-watch.Errors:
 						{
 							return errors.New("Listener exits abnormally err: " + err.Error())
 						}
+					case <-cMonitoring.Channel:
+						{
+							for _, file := range cMonitoring.Removefiles {
+								if err := watch.Remove(file); err != nil {
+									return errors.New("File listening removal failed: " + err.Error())
+								}
+							}
+
+							//添加要监控的对象，文件或文件夹
+							if err = watch.Add(cfg.configFile); err != nil {
+								return errors.New("Failed to listen to profile err: " + err.Error())
+							}
+
+							for _, file := range cMonitoring.Rules {
+								if err := watch.Add(file); err != nil {
+									return errors.New("Failed to listen to profile err: " + err.Error())
+								}
+							}
+						}
 					case <-cancel:
 						{
 							return errors.New("Listener receives interrupt processing signal")
@@ -817,6 +832,7 @@ func main() {
 				close(cancel)
 			},
 		)
+		// }
 	}
 
 	if err := g.Run(); err != nil {

cmd/prometheus/prometheus.yml

@@ -9,12 +9,11 @@ alerting:
   alertmanagers:
   - static_configs:
     - targets:
-      # - alertmanager:9093
+      # - http://127.0.0.1:9093
 
 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
-  # - "first_rules.yml"
-  # - "second_rules.yml"
+  - "rules/first_rules.yml"
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.

cmd/prometheus/rules/first_rules.yml

@@ -0,0 +1,10 @@
+groups:
+- name: InstanceException
+  rules:
+  - alert: 心跳失败
+    expr: up == 0
+    for: 1m
+    labels:
+      severity: critical
+    annotations:
+      summary: "连通性检测失败，请排查处理"

config/monitoring.go

@@ -0,0 +1,32 @@
+package config
+
+type CMonitoring struct {
+	Rules       []string
+	Removefiles []string
+
+	Channel chan struct{}
+}
+
+func NewCMonitoring() *CMonitoring {
+	return &CMonitoring{Channel: make(chan struct{})}
+}
+
+func (c *CMonitoring) JudgeChange(files []string) {
+	c.Removefiles = []string{}
+
+	for _, cfile := range c.Rules {
+		in := false
+		for _, file := range files {
+			if file == cfile {
+				in = true
+				break
+			}
+		}
+		if in == false {
+			c.Removefiles = append(c.Removefiles, cfile)
+		}
+	}
+	c.Rules = files
+	c.Channel <- struct{}{}
+	return
+}