refactor configuration and fix SMTP mailer (#254)

* refactor configuration and fix SMTP mailer (BREAKING CHANGE)

* fix go version in lint job

* remove go 1.16

Author: Hsn723

.github/workflows/main.yml

@@ -12,6 +12,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
       - name: Lint
         uses: pre-commit/[email protected]
         with:
@@ -45,7 +49,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.16', '1.17' ]
+        go: [ '1.17' ]
     steps:
       - uses: actions/checkout@v2
       - name: Setup go

.gitignore

@@ -3,3 +3,4 @@ artifacts/
 bin/
 dist/
 ct-monitor
+*.out

Makefile

@@ -19,7 +19,7 @@ lint:
 
 .PHONY: test
 test:
-	go test -race -v $$(go list ./... | grep -v test)
+	go test -coverprofile cover.out -count=1 -race -p 4 -v ./...
 
 .PHONY: setup-container-structure-test
 setup-container-structure-test:
@@ -33,8 +33,8 @@ container-structure-test: setup-container-structure-test
 
 .PHONY: setup-kind
 setup-kind:
-	curl -sSLf -O https://storage.googleapis.com/kubernetes-release/release/v$(KUBERNETES_VERSION)/bin/linux/amd64/kubectl
-	sudo install kubectl /usr/local/bin/kubectl
+	curl -sSLf -o /tmp/kubectl -O https://storage.googleapis.com/kubernetes-release/release/v$(KUBERNETES_VERSION)/bin/linux/amd64/kubectl
+	sudo install /tmp/kubectl /usr/local/bin/kubectl
 	go install sigs.k8s.io/kind@v$(KIND_VERSION)
 
 .PHONY: start-kind
@@ -47,7 +47,7 @@ stop-kind:
 
 .PHONY: kindtest
 kindtest: clean stop-kind start-kind build
-	go test -race -v ./test
+	go test --tags=e2e -count=1 -coverprofile e2e.out -race -p 4 -v ./...
 
 .PHONY: verify
 verify:

README.md

@@ -11,25 +11,21 @@ Usage:
 
 Flags:
   -c, --config string     path to configuration file (default "/etc/ct-monitor/config.toml")
-  -d, --domains strings   domains to query certspotter for issuances
-  -e, --endpoint string   API endpoint (default "https://api.certspotter.com/v1/issuances")
   -h, --help              help for ct-monitor
-  -p, --position string   path to position file (default "/var/log/ct-monitor/positions.toml")
-  -s, --subdomains        include subdomains (default true)
-  -t, --token string      API token
-  -w, --wildcard          match wildcards (default true)
 ```
 
 ## Example config
 ```toml
 [alert_config]
-    from = "[email protected]"
-    recipient = "[email protected]"
     mailer_config = "sendgrid"
 
 [sendgrid]
+    from = "[email protected]"
+    to = "[email protected]"
     apiKey = "your-api-key"
 
 [position_config]
     filename = "/var/log/ct-monitor/positions.toml"
 ```
+
+For more details, check the documentation.

VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0

cmd/root.go

@@ -8,59 +8,29 @@ import (
 	"strings"
 
 	"github.com/Hsn723/certspotter-client/api"
+	"github.com/Hsn723/ct-monitor/config"
 	"github.com/Hsn723/ct-monitor/mailer"
 	"github.com/cybozu-go/log"
-	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
-const (
-	positionFileConfigKey     = "position_config.filename"
-	mailFromConfigKey         = "alert_config.from"
-	mailToConfigKey           = "alert_config.recipient"
-	mailerConfigKey           = "alert_config.mailer_config"
-	domainsConfigKey          = "domains"
-	endpointConfigKey         = "endpoint"
-	certspotterTokenConfigKey = "certspotter_token"
-	sendgridTokenConfigKey    = "sendgrid.apiKey"
-
-	defaultEndpoint     = "https://api.certspotter.com/v1/issuances"
-	defaultConfigFile   = "/etc/ct-monitor/config.toml"
-	defaultPositionFile = "/var/log/ct-monitor/positions.toml"
-	tokenEnv            = "CERTSPOTTER_TOKEN"
-	sendgridTokenEnv    = "SENDGRID_TOKEN"
-)
-
 var (
 	rootCmd = &cobra.Command{
 		Use:   "ct-monitor",
 		Short: "ct-monitor queries the certspotter API for new certificate issuances",
 		RunE:  runRoot,
 	}
 	position = viper.New()
-	config   = viper.New()
 
-	configFile        string
-	endpoint          string
-	matchWildcards    bool
-	includeSubdomains bool
-	mailSender        mailer.Mailer
+	configFile string
 
 	version string
 	commit  string
 	date    string
 	builtBy string
 )
 
-func getPositionFilePath() string {
-	positionFile := config.GetString(positionFileConfigKey)
-	if positionFile == "" {
-		return defaultPositionFile
-	}
-	return positionFile
-}
-
 func createFile(path string) error {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
 		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
@@ -75,35 +45,9 @@ func createFile(path string) error {
 	return nil
 }
 
-func initConfig() {
-	_ = log.Info("ct-monitor", map[string]interface{}{
-		"version":  version,
-		"commit":   commit,
-		"date":     date,
-		"built_by": builtBy,
-	})
-	config.SetConfigFile(configFile)
-	if err := createFile(configFile); err != nil {
-		_ = log.Critical(err.Error(), nil)
-	}
-	if err := config.ReadInConfig(); err != nil {
-		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
-			_ = log.Critical(err.Error(), nil)
-		}
-	}
-	_ = log.Info("using config file", map[string]interface{}{
-		"path": config.ConfigFileUsed(),
-	})
-	config.WatchConfig()
-
-	initPosition()
-	initMailer()
-}
-
-func initPosition() {
-	positionFile := getPositionFilePath()
-	position.SetConfigFile(positionFile)
-	if err := createFile(positionFile); err != nil {
+func initPosition(pc config.PositionConfig) {
+	position.SetConfigFile(pc.Filename)
+	if err := createFile(pc.Filename); err != nil {
 		_ = log.Critical(err.Error(), nil)
 	}
 	if err := position.ReadInConfig(); err != nil {
@@ -117,74 +61,15 @@ func initPosition() {
 	position.WatchConfig()
 }
 
-func initMailer() {
-	from := config.GetString(mailFromConfigKey)
-	to := config.GetString(mailToConfigKey)
-	if to == "" {
-		_ = log.Warn("alert mail recipient missing", nil)
-		return
-	}
-	if from == "" {
-		_ = log.Warn("alert mail from address missing", nil)
-		return
-	}
-	mailerName := config.GetString(mailerConfigKey)
-	switch mailerName {
-	case "smtp":
-		mailSender = &mailer.SMTPMailer{}
-	case "sendgrid":
-		mailSender = &mailer.SendgridMailer{
-			APIKey: config.GetString(sendgridTokenConfigKey),
-			Logger: log.DefaultLogger(),
-		}
-	case "amazonses":
-		mailSender = &mailer.AmazonSESMailer{
-			Logger: log.DefaultLogger(),
-		}
-	default:
-		_ = log.Warn("no mailer configured, email report will not be sent", nil)
-		return
-	}
-
-	senderConf := config.GetStringMap(mailerName)
-	if err := mapstructure.Decode(senderConf, &mailSender); err != nil {
-		_ = log.Critical(err.Error(), nil)
-	}
-
-	if err := mailSender.Init(from, to); err != nil {
-		_ = log.Critical(err.Error(), nil)
-	}
-}
-
 func init() {
-	cobra.OnInitialize(initConfig)
-	rootCmd.Flags().StringVarP(&configFile, "config", "c", defaultConfigFile, "path to configuration file")
-	rootCmd.Flags().StringVarP(&endpoint, "endpoint", "e", defaultEndpoint, "API endpoint")
-
-	rootCmd.Flags().StringP("position", "p", defaultPositionFile, "path to position file")
-	rootCmd.Flags().StringP("token", "t", "", "API token")
-	rootCmd.Flags().StringSliceP("domains", "d", []string{}, "domains to query certspotter for issuances")
-
-	rootCmd.Flags().BoolVarP(&matchWildcards, "wildcard", "w", true, "match wildcards")
-	rootCmd.Flags().BoolVarP(&includeSubdomains, "subdomains", "s", true, "include subdomains")
-
-	_ = config.BindEnv(certspotterTokenConfigKey, tokenEnv)
-	_ = config.BindEnv(sendgridTokenConfigKey, sendgridTokenEnv)
-
-	_ = config.BindPFlag(certspotterTokenConfigKey, rootCmd.Flags().Lookup("token"))
-	_ = config.BindPFlag(domainsConfigKey, rootCmd.Flags().Lookup(domainsConfigKey))
-	_ = config.BindPFlag(positionFileConfigKey, rootCmd.Flags().Lookup("position"))
-	_ = config.BindPFlag(endpointConfigKey, rootCmd.Flags().Lookup("endpoint"))
+	rootCmd.Flags().StringVarP(&configFile, "config", "c", config.DefaultConfigFile, "path to configuration file")
 }
 
 func getDomainConfigName(domain string) string {
 	return strings.ReplaceAll(domain, ".", "-")
 }
 
-func sendMail(domain string, issuances []api.Issuance) error {
-	if mailSender == nil {
-		return nil
-	}
+func sendMail(mailSender mailer.Mailer, domain string, issuances []api.Issuance) error {
 	subject := fmt.Sprintf("Certificate Transparency Notification for %s", domain)
 	body := fmt.Sprintf("ct-monitor has observed the issuance of the following certificate(s) for the %s domain:\n", domain)
 	for _, i := range issuances {
@@ -196,7 +81,7 @@ func sendMail(domain string, issuances []api.Issuance) error {
 	return mailSender.Send(subject, body)
 }
 
-func checkIssuances(domain string, wildcards, subdomains bool, c api.CertspotterClient) error {
+func checkIssuances(domain string, wildcards, subdomains bool, c api.CertspotterClient, mailSender mailer.Mailer) error {
 	key := getDomainConfigName(domain)
 	lastIssuance := position.GetUint64(key)
 	issuances, err := c.GetIssuances(domain, wildcards, subdomains, lastIssuance)
@@ -217,7 +102,7 @@ func checkIssuances(domain string, wildcards, subdomains bool, c api.Certspotter
 			"sha256": issuance.Cert.SHA256,
 		})
 	}
-	if err := sendMail(domain, issuances); err != nil {
+	if err := sendMail(mailSender, domain, issuances); err != nil {
 		return err
 	}
 	position.Set(key, lastIssuance)
@@ -227,9 +112,8 @@ func checkIssuances(domain string, wildcards, subdomains bool, c api.Certspotter
 	return nil
 }
 
-func atomicWritePosition() error {
-	positionFile := getPositionFilePath()
-	tmpFile, err := ioutil.TempFile(filepath.Dir(positionFile), "position.*.toml")
+func atomicWritePosition(pc config.PositionConfig) error {
+	tmpFile, err := ioutil.TempFile(filepath.Dir(pc.Filename), "position.*.toml")
 	if err != nil {
 		return err
 	}
@@ -243,23 +127,41 @@ func atomicWritePosition() error {
 	if fi.Size() == 0 {
 		return nil
 	}
-	return os.Rename(tmpFile.Name(), positionFile)
+	return os.Rename(tmpFile.Name(), pc.Filename)
 }
 
 func runRoot(cmd *cobra.Command, args []string) error {
+	_ = log.Info("ct-monitor", map[string]interface{}{
+		"version":  version,
+		"commit":   commit,
+		"date":     date,
+		"built_by": builtBy,
+	})
+	conf, err := config.Load(configFile)
+	if err != nil {
+		return err
+	}
+	_ = log.Info("loaded configuration", map[string]interface{}{
+		"config": configFile,
+	})
+	initPosition(conf.PositionConfig)
+	mailSender := conf.GetMailer()
+	if err := mailSender.Init(); err != nil {
+		return err
+	}
 	csp := api.CertspotterClient{
-		Endpoint: endpoint,
-		Token:    config.GetString(certspotterTokenConfigKey),
+		Endpoint: conf.Endpoint,
+		Token:    conf.Token,
 	}
-	for _, domain := range config.GetStringSlice(domainsConfigKey) {
-		if err := checkIssuances(domain, matchWildcards, includeSubdomains, csp); err != nil {
+	for _, domain := range conf.Domains {
+		if err := checkIssuances(domain.Name, domain.MatchWildcards, domain.IncludeSubdomains, csp, mailSender); err != nil {
 			_ = log.Error(err.Error(), map[string]interface{}{
-				"domain": domain,
+				"domain": domain.Name,
 			})
 		}
 	}
 
-	return atomicWritePosition()
+	return atomicWritePosition(conf.PositionConfig)
 }
 
 // Execute runs the root command.