Add runtime reload for global settings (#214)

## Summary
- compare global options when reloading config
- rebuild scheduler and job middlewares on global change
- add `ResetMiddlewares` to core
- test global reload of log-level and save settings
- document which settings are hot-reloaded

## Testing
- `go vet ./...`
- `go test ./...`

------
https://chatgpt.com/codex/tasks/task_b_686961668de88333ad17ed6741cfb032

Author: CybotTM

README.md

@@ -127,6 +127,11 @@ Ofelia merges options from multiple sources in the following order. Values from
 
 The daemon watches `config.ini` and reloads it automatically when the file changes.
 
+Job definitions and most `[global]` middleware options (`slack-*`, `save-*`,
+`mail-*`, `log-level`, `max-runtime`) are applied on reload. Options that start
+servers (`enable-web`, `web-address`, `enable-pprof`, `pprof-address`) and all
+`[docker]` settings require restarting the daemon.
+
 ## Configuration
 
 ### Jobs

cli/config.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"reflect"
 	"sort"
 	"strings"
 	"time"
@@ -257,6 +258,7 @@ type jobConfig interface {
 	Hash() (string, error)
 	GetJobSource() JobSource
 	SetJobSource(JobSource)
+	ResetMiddlewares(...core.Middleware)
 }
 
 // syncJobMap updates the scheduler and the provided job map based on the parsed
@@ -400,9 +402,30 @@ func (c *Config) iniConfigUpdate() error {
 	if err != nil {
 		return err
 	}
+	globalChanged := !reflect.DeepEqual(parsed.Global, c.Global)
 	c.configFiles = files
 	c.configModTime = latest
 	c.logger.Debugf("applied config files from %s", strings.Join(files, ", "))
+	if globalChanged {
+		c.Global = parsed.Global
+		c.sh.ResetMiddlewares()
+		c.buildSchedulerMiddlewares(c.sh)
+		for _, j := range c.sh.Jobs {
+			if jc, ok := j.(jobConfig); ok {
+				jc.ResetMiddlewares()
+				jc.buildMiddlewares()
+				j.Use(c.sh.Middlewares()...)
+			}
+		}
+		for _, j := range c.sh.Disabled {
+			if jc, ok := j.(jobConfig); ok {
+				jc.ResetMiddlewares()
+				jc.buildMiddlewares()
+				j.Use(c.sh.Middlewares()...)
+			}
+		}
+		ApplyLogLevel(c.Global.LogLevel)
+	}
 
 	execPrep := func(name string, j *ExecJobConfig) {
 		defaults.Set(j)

cli/config_extra_test.go

@@ -3,13 +3,17 @@ package cli
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"time"
 
 	defaults "github.com/creasty/defaults"
 	"github.com/netresearch/ofelia/core"
+	"github.com/netresearch/ofelia/middlewares"
+
+	"github.com/sirupsen/logrus"
 
 	. "gopkg.in/check.v1"
 )
@@ -345,6 +349,55 @@ func (s *SuiteConfig) TestIniConfigUpdateGlob(c *C) {
 	c.Assert(cfg.RunJobs["foo"].GetSchedule(), Equals, "@every 10s")
 }
 
+// TestIniConfigUpdateGlobalChange verifies global middleware options and log
+// level are reloaded.
+func (s *SuiteConfig) TestIniConfigUpdateGlobalChange(c *C) {
+	tmp, err := ioutil.TempFile("", "ofelia_*.ini")
+	c.Assert(err, IsNil)
+	defer os.Remove(tmp.Name())
+
+	dir := c.MkDir()
+	content1 := fmt.Sprintf("[global]\nlog-level = INFO\nsave-folder = %s\n",
+		dir)
+	content1 += "save-only-on-error = false\n"
+	content1 += "[job-run \"foo\"]\nschedule = @every 5s\nimage = busybox\ncommand = echo foo\n"
+	_, err = tmp.WriteString(content1)
+	c.Assert(err, IsNil)
+	tmp.Close()
+
+	logrus.SetLevel(logrus.InfoLevel)
+
+	cfg, err := BuildFromFile(tmp.Name(), &TestLogger{})
+	c.Assert(err, IsNil)
+	cfg.logger = &TestLogger{}
+	cfg.dockerHandler = &DockerHandler{}
+	cfg.sh = core.NewScheduler(&TestLogger{})
+	cfg.buildSchedulerMiddlewares(cfg.sh)
+
+	ApplyLogLevel(cfg.Global.LogLevel)
+	ms := cfg.sh.Middlewares()
+	c.Assert(ms, HasLen, 1)
+	saveMw := ms[0].(*middlewares.Save)
+	c.Assert(saveMw.SaveOnlyOnError, Equals, false)
+	c.Assert(logrus.GetLevel(), Equals, logrus.InfoLevel)
+
+	oldTime := cfg.configModTime
+	content2 := fmt.Sprintf("[global]\nlog-level = DEBUG\nsave-folder = %s\nsave-only-on-error = true\n", dir)
+	content2 += "[job-run \"foo\"]\nschedule = @every 5s\nimage = busybox\ncommand = echo foo\n"
+	err = os.WriteFile(tmp.Name(), []byte(content2), 0o644)
+	c.Assert(err, IsNil)
+	c.Assert(waitForModTimeChange(tmp.Name(), oldTime), IsNil)
+
+	err = cfg.iniConfigUpdate()
+	c.Assert(err, IsNil)
+	c.Assert(cfg.Global.LogLevel, Equals, "DEBUG")
+	ms = cfg.sh.Middlewares()
+	c.Assert(ms, HasLen, 1)
+	saveMw = ms[0].(*middlewares.Save)
+	c.Assert(saveMw.SaveOnlyOnError, Equals, true)
+	c.Assert(logrus.GetLevel(), Equals, logrus.DebugLevel)
+}
+
 func waitForModTimeChange(path string, after time.Time) error {
 	info, err := os.Stat(path)
 	if err != nil {

core/common.go

@@ -221,6 +221,12 @@ func (c *middlewareContainer) Use(ms ...Middleware) {
 	}
 }
 
+func (c *middlewareContainer) ResetMiddlewares(ms ...Middleware) {
+	c.m = nil
+	c.order = nil
+	c.Use(ms...)
+}
+
 func (c *middlewareContainer) Middlewares() []Middleware {
 	var ms []Middleware
 	for _, t := range c.order {

core/common_test.go

@@ -273,6 +273,19 @@ func (s *SuiteCommon) TestMiddlewareContainerUseOder(c *C) {
 	c.Assert(ms[1], Equals, mA)
 }
 
+func (s *SuiteCommon) TestMiddlewareContainerReset(c *C) {
+	mA := &TestMiddleware{}
+	mB := &TestMiddlewareAltA{}
+
+	container := &middlewareContainer{}
+	container.Use(mA)
+	container.ResetMiddlewares(mB)
+
+	ms := container.Middlewares()
+	c.Assert(ms, HasLen, 1)
+	c.Assert(ms[0], Equals, mB)
+}
+
 type TestMiddleware struct {
 	Called int
 	Nested bool