-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathmain.go
More file actions
261 lines (241 loc) · 6.56 KB
/
Copy pathmain.go
File metadata and controls
261 lines (241 loc) · 6.56 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
// Gjallar is a KISS monitoring service: one binary, one YAML config file,
// one SQLite file.
package main
import (
"context"
"flag"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
"gjallar/internal/alert"
"gjallar/internal/check"
"gjallar/internal/config"
"gjallar/internal/store"
"gjallar/internal/web"
)
// version is injected at build time via -ldflags "-X main.version=..."
var version = "dev"
func main() {
configPath := flag.String("config", "gjallar.yaml", "path to YAML configuration")
showVersion := flag.Bool("version", false, "print version and exit")
checkOnly := flag.Bool("check", false, "validate the configuration (including checkers and alert URLs) and exit")
flag.Parse()
if *showVersion {
fmt.Println("gjallar", version)
return
}
if *checkOnly {
cfg, err := config.Load(*configPath)
if err == nil {
_, err = prepare(cfg)
}
if err != nil {
fmt.Fprintln(os.Stderr, "gjallar:", err)
os.Exit(1)
}
fmt.Printf("config OK: %d monitors, %d alert channels\n", len(cfg.Monitors), len(cfg.Alerts))
return
}
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
slog.Info("starting gjallar", "version", version)
if err := run(*configPath); err != nil {
fmt.Fprintln(os.Stderr, "gjallar:", err)
os.Exit(1)
}
}
func run(configPath string) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
hup := make(chan os.Signal, 1)
signal.Notify(hup, syscall.SIGHUP)
cfg, err := config.Load(configPath)
if err != nil {
return err
}
p, err := prepare(cfg)
if err != nil {
return err
}
inst, err := start(cfg, p)
if err != nil {
return err
}
for {
select {
case <-ctx.Done():
slog.Info("shutting down")
inst.stop()
return nil
case <-hup:
// Validate everything we can before touching the running
// instance: a broken config must not take the service down.
newCfg, err := config.Load(configPath)
if err != nil {
slog.Error("reload failed, keeping current config", "error", err)
continue
}
newP, err := prepare(newCfg)
if err != nil {
slog.Error("reload failed, keeping current config", "error", err)
continue
}
inst.stop() // releases the listen port and the SQLite file
if inst, err = start(newCfg, newP); err != nil {
return fmt.Errorf("restarting after reload: %w", err)
}
slog.Info("configuration reloaded")
}
}
}
// prepared holds the parts of an instance that can be built (and fail)
// without owning the listen port or the database.
type prepared struct {
checkers []check.Checker
notifiers map[string]alert.Notifier
}
func prepare(cfg *config.Config) (*prepared, error) {
p := &prepared{checkers: make([]check.Checker, len(cfg.Monitors))}
var err error
for i, m := range cfg.Monitors {
if !m.IsEnabled() {
continue // disabled monitors are listed but never built or run
}
if p.checkers[i], err = check.New(m); err != nil {
return nil, err
}
}
if err := pingSelfTest(cfg); err != nil {
return nil, err
}
if p.notifiers, err = alert.BuildNotifiers(cfg.Alerts); err != nil {
return nil, err
}
return p, nil
}
// instance is one running incarnation of the service; SIGHUP stops it and
// starts a fresh one.
type instance struct {
cancel context.CancelFunc
runners sync.WaitGroup
consumer sync.WaitGroup
httpSrv *http.Server
st *store.Store
}
func start(cfg *config.Config, p *prepared) (*instance, error) {
st, err := store.Open(cfg.Database)
if err != nil {
return nil, err
}
// Drop history and any lingering (possibly open) incidents for monitors
// removed from the config, so a decommissioned host stops appearing.
names := make([]string, len(cfg.Monitors))
for i, m := range cfg.Monitors {
names[i] = m.Name
}
if n, err := st.PruneOrphans(names); err != nil {
slog.Error("pruning removed monitors", "error", err)
} else if n > 0 {
slog.Info("pruned data for removed monitors", "rows", n)
}
engine, err := alert.NewEngine(cfg, st, p.notifiers)
if err != nil {
st.Close()
return nil, err
}
srv, err := web.New(cfg, st)
if err != nil {
st.Close()
return nil, err
}
ln, err := net.Listen("tcp", cfg.Listen)
if err != nil {
st.Close()
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
inst := &instance{cancel: cancel, st: st}
// Pipeline: one goroutine per monitor -> results channel -> single
// consumer (one SQLite writer, lock-free alert state machine).
results := make(chan check.Result, len(cfg.Monitors))
for i, m := range cfg.Monitors {
if !m.IsEnabled() {
continue // disabled: shown on the page, never scheduled
}
inst.runners.Add(1)
go func(m config.Monitor, c check.Checker) {
defer inst.runners.Done()
check.Run(ctx, m, c, results)
}(m, p.checkers[i])
}
go func() {
inst.runners.Wait()
close(results)
}()
inst.consumer.Add(1)
go func() {
defer inst.consumer.Done()
for r := range results {
slog.Info("check", "monitor", r.Monitor, "ok", r.OK, "latency", r.Latency, "message", r.Message)
if err := st.InsertResult(r.Monitor, r.Time, r.OK, r.Latency, r.Message); err != nil {
slog.Error("storing result", "monitor", r.Monitor, "error", err)
}
engine.Process(r)
}
}()
go pruneLoop(ctx, st, cfg.Retention.D())
inst.httpSrv = &http.Server{Handler: srv.Handler()}
go func() {
slog.Info("status page listening", "addr", cfg.Listen)
if err := inst.httpSrv.Serve(ln); err != nil && err != http.ErrServerClosed {
slog.Error("http server", "error", err)
}
}()
return inst, nil
}
// stop tears the instance down in order: stop check loops, drain the
// pipeline, stop HTTP, then close the DB.
func (i *instance) stop() {
i.cancel()
i.runners.Wait()
i.consumer.Wait()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := i.httpSrv.Shutdown(ctx); err != nil {
slog.Error("http shutdown", "error", err)
}
if err := i.st.Close(); err != nil {
slog.Error("closing store", "error", err)
}
}
func pingSelfTest(cfg *config.Config) error {
tested := map[bool]bool{}
for _, m := range cfg.Monitors {
if m.Type == "ping" && m.IsEnabled() && !tested[m.Privileged] {
tested[m.Privileged] = true
if err := check.SelfTestPing(m.Privileged); err != nil {
return err
}
}
}
return nil
}
func pruneLoop(ctx context.Context, st *store.Store, retention time.Duration) {
ticker := time.NewTicker(time.Hour)
defer ticker.Stop()
for {
if err := st.Prune(time.Now().Add(-retention)); err != nil {
slog.Error("pruning", "error", err)
}
select {
case <-ticker.C:
case <-ctx.Done():
return
}
}
}