Merge pull request #3 from systemli/configurable-expiration-duration

make expiration duration for pads configurable

Author: 0x46616c6b

README.md

@@ -73,7 +73,8 @@ Usage:
 
 Flags:
   -h, --help                 help for metrics
-      --listen.addr string    (default ":9012")
+      --listen.addr string   Address on which to expose metrics. (default ":9012")
+      --suffixes string      Suffixes to group the pads. (default "keep,temp")
 ```
 
 ### Move Pad
@@ -91,19 +92,28 @@ Flags:
 
 ### Purge
 
-The command checks every Pad if the last edited date is older than the defined limit. Older Pads will be deleted.
+The command checks every Pad for it’s last edited date. If it is older than the defined limit, the pad will be deleted.
 
-Pads without any changes (revisions) will be deleted.
-Pads without a suffix will be deleted after 30 days of inactivity.  
-Pads with the suffix "-temp" will be deleted after 24 hours of inactivity.  
-Pads with the suffix "-keep" will be deleted after 365 days of inactivity.  
+Pads without any changes (revisions) will be deleted. This can happen when no content was changed in the pad
+(e.g. a person misspelles a pad).
+Pads will grouped by the pre-defined suffixes. Every suffix has a defined expiration time. If the pad is older than the
+defined expiration time, the pad will be deleted.
+
+Example:
+
+`etherpad-toolkit purge --expiration "default:720h,temp:24h,keep:8760h"`
+
+This configuration will group the pads in three clusters: default (expiration: 30 days, suffix is required!),
+temp (expiration: 24 hours), keep (expiration: 365 days). If pads in the clusters older than the given expiration the
+pads will be deleted.
 
 ```
 Usage:
   etherpad-toolkit purge [flags]
 
 Flags:
-      --concurrency int   Concurrency for the purge process (default 4)
-      --dry-run           Enable dry-run
-  -h, --help              help for purge
+      --concurrency int     Concurrency for the purge process (default 4)
+      --dry-run             Enable dry-run
+      --expiration string   Configuration for pad expiration duration. Example: "default:720h,temp:24h,keep:8760h"
+  -h, --help                help for purge
 ```

cmd/metrics.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -13,6 +14,7 @@ import (
 
 var (
 	listenAddr string
+	suffixes   string
 
 	metricsCmd = &cobra.Command{
 		Use:   "metrics",
@@ -23,14 +25,15 @@ var (
 )
 
 func init() {
-	metricsCmd.Flags().StringVar(&listenAddr, "listen.addr", ":9012", "")
+	metricsCmd.Flags().StringVar(&listenAddr, "listen.addr", ":9012", "Address on which to expose metrics.")
+	metricsCmd.Flags().StringVar(&suffixes, "suffixes", "keep,temp", "Suffixes to group the pads.")
 
 	rootCmd.AddCommand(metricsCmd)
 }
 
 func runMetrics(cmd *cobra.Command, args []string) {
 	etherpad := pkg.NewEtherpadClient(etherpadUrl, etherpadApiKey)
-	prometheus.MustRegister(metrics.NewPadCollector(etherpad))
+	prometheus.MustRegister(metrics.NewPadCollector(etherpad, strings.Split(suffixes, ",")))
 
 	http.Handle("/metrics", promhttp.Handler())
 	log.Fatal(http.ListenAndServe(listenAddr, nil))

cmd/purge.go

@@ -11,14 +11,23 @@ import (
 var (
 	concurrency int
 	dryRun      bool
+	expiration  string
 
 	longDescription = `
-The command checks every Pad if the last edited date is older than the defined limit. Older Pads will be deleted.
+The command checks every Pad for it’s last edited date. If it is older than the defined limit, the pad will be deleted.
 
-Pads without any changes (revisions) will be deleted.
-Pads without a suffix will be deleted after 30 days of inactivity.
-Pads with the suffix "-temp" will be deleted after 24 hours of inactivity.
-Pads with the suffix "-keep" will be deleted after 365 days of inactivity.
+Pads without any changes (revisions) will be deleted. This can happen when no content was changed in the pad 
+(e.g. a person misspelles a pad).
+Pads will grouped by the pre-defined suffixes. Every suffix has a defined expiration time. If the pad is older than the 
+defined expiration time, the pad will be deleted.
+
+Example:
+
+etherpad-toolkit purge --expiration "default:720h,temp:24h,keep:8760h"
+
+This configuration will group the pads in three clusters: default (expiration: 30 days, suffix is required!), 
+temp (expiration: 24 hours), keep (expiration: 365 days). If pads in the clusters older than the given expiration the 
+pads will be deleted.
 `
 
 	purgeCmd = &cobra.Command{
@@ -30,6 +39,7 @@ Pads with the suffix "-keep" will be deleted after 365 days of inactivity.
 )
 
 func init() {
+	purgeCmd.Flags().StringVar(&expiration, "expiration", "", "Configuration for pad expiration duration. Example: \"default:720h,temp:24h,keep:8760h\"")
 	purgeCmd.Flags().IntVar(&concurrency, "concurrency", 4, "Concurrency for the purge process")
 	purgeCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Enable dry-run")
 
@@ -38,14 +48,11 @@ func init() {
 
 func runPurger(cmd *cobra.Command, args []string) {
 	etherpad := pkg.NewEtherpadClient(etherpadUrl, etherpadApiKey)
-	purger := purge.NewPurger(etherpad, dryRun)
-
-	pads, err := etherpad.ListAllPads()
+	exp, err := helper.ParsePadExpiration(expiration)
 	if err != nil {
-		log.WithError(err).Error("failed to fetch pads")
+		log.WithError(err).Error("failed to parse expiration string")
 		return
 	}
-	sorted := helper.SortPads(pads)
-
-	purger.PurgePads(sorted, concurrency)
+	purger := purge.NewPurger(etherpad, exp, dryRun)
+	purger.PurgePads(concurrency)
 }

pkg/helper/expiration.go

@@ -0,0 +1,89 @@
+package helper
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const DefaultSuffix = "default"
+
+type PadExpiration map[string]time.Duration
+
+// ParsePadExpiration splits a string with format "default:30d,temp:24h,keep:365d" and returns a PadExpiration type.
+// The key "default:<duration>" is mandatory in the input string.
+func ParsePadExpiration(s string) (PadExpiration, error) {
+	exp := make(PadExpiration)
+
+	if s == "" {
+		return exp, errors.New("input string is empty")
+	}
+
+	for _, str := range strings.Split(s, ",") {
+		split := strings.Split(str, ":")
+		if len(split) != 2 {
+			log.WithField("string", str).Error("string is not valid")
+			continue
+		}
+		duration, err := time.ParseDuration(split[1])
+		if err != nil {
+			log.WithError(err).WithField("duration", split[1]).Error("unable to parse the duration")
+			continue
+		}
+
+		exp[split[0]] = duration
+	}
+
+	if _, ok := exp[DefaultSuffix]; !ok {
+		return exp, errors.New("missing default expiration duration")
+	}
+
+	return exp, nil
+}
+
+// GetDuration tries to get the Duration by pad name, returns the default duration if no suffix matches.
+func (pe *PadExpiration) GetDuration(pad string) time.Duration {
+	for suffix, duration := range *pe {
+		if strings.HasSuffix(pad, fmt.Sprintf("-%s", suffix)) {
+			return -duration
+		}
+	}
+
+	return -(*pe)[DefaultSuffix]
+}
+
+// GroupPadsByExpiration sorts pads for the given expiration and returns a map with string keys and string slices.
+func GroupPadsByExpiration(pads []string, expiration PadExpiration) map[string][]string {
+	var suffixes []string
+	for suffix := range expiration {
+		if suffix == DefaultSuffix {
+			continue
+		}
+		suffixes = append(suffixes, suffix)
+	}
+
+	return GroupPadsBySuffixes(pads, suffixes)
+}
+
+// GroupPadsBySuffixes sorts pads for the given suffixes and returns a map with string keys and string slices.
+func GroupPadsBySuffixes(pads, suffixes []string) map[string][]string {
+	sorted := make(map[string][]string)
+
+	for _, pad := range pads {
+		found := false
+		for _, suffix := range suffixes {
+			if strings.HasSuffix(pad, fmt.Sprintf("-%s", suffix)) {
+				sorted[suffix] = append(sorted[suffix], pad)
+				found = true
+			}
+		}
+		if !found {
+			sorted[DefaultSuffix] = append(sorted[DefaultSuffix], pad)
+		}
+	}
+
+	return sorted
+}

pkg/helper/expiration_test.go

@@ -0,0 +1,77 @@
+package helper
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParsePadExpiration(t *testing.T) {
+	s := "default:720h,temp:24h,keep:262800h"
+	exp, err := ParsePadExpiration(s)
+	assert.Nil(t, err)
+	assert.Equal(t, time.Duration(2592000000000000), exp[DefaultSuffix])
+	assert.Equal(t, time.Duration(946080000000000000), exp["keep"])
+	assert.Equal(t, time.Duration(86400000000000), exp["temp"])
+
+	s = "wrong:1d"
+
+	_, err = ParsePadExpiration(s)
+	assert.Error(t, err)
+	assert.Equal(t, "missing default expiration duration", err.Error())
+
+	s = "wrong:1h:2h"
+	_, err = ParsePadExpiration(s)
+	assert.Error(t, err)
+	assert.Equal(t, "missing default expiration duration", err.Error())
+
+	s = ""
+	_, err = ParsePadExpiration(s)
+	assert.Error(t, err)
+	assert.Equal(t, "input string is empty", err.Error())
+}
+
+func TestPadExpiration_GetDuration(t *testing.T) {
+	s := "default:24h"
+	exp, err := ParsePadExpiration(s)
+	assert.Nil(t, err)
+
+	dur := exp.GetDuration("pad")
+	assert.Equal(t, "-24h0m0s", dur.String())
+
+	s = "default:24h,temp:10m"
+	exp, err = ParsePadExpiration(s)
+	assert.Nil(t, err)
+
+	dur = exp.GetDuration("pad")
+	assert.Equal(t, "-24h0m0s", dur.String())
+
+	dur = exp.GetDuration("pad-temp")
+	assert.Equal(t, "-10m0s", dur.String())
+}
+
+func TestGroupPadsByExpiration(t *testing.T) {
+	s := "default:720h,temp:24h,keep:262800h"
+	pads := []string{"pad", "pad2", "pad-keep", "pad-temp"}
+	exp, err := ParsePadExpiration(s)
+	assert.Nil(t, err)
+
+	sorted := GroupPadsByExpiration(pads, exp)
+
+	if _, ok := sorted[DefaultSuffix]; !ok {
+		t.Fail()
+	}
+
+	if _, ok := sorted["keep"]; !ok {
+		t.Fail()
+	}
+
+	if _, ok := sorted["temp"]; !ok {
+		t.Fail()
+	}
+
+	assert.Equal(t, []string{"pad", "pad2"}, sorted[DefaultSuffix])
+	assert.Equal(t, []string{"pad-keep"}, sorted["keep"])
+	assert.Equal(t, []string{"pad-temp"}, sorted["temp"])
+}

pkg/helper/sort.go

@@ -9,12 +9,14 @@ import (
 
 type PadCollector struct {
 	etherpad     *pkg.Etherpad
+	suffixes     []string
 	PadGaugeDesc *prometheus.Desc
 }
 
-func NewPadCollector(etherpad *pkg.Etherpad) *PadCollector {
+func NewPadCollector(etherpad *pkg.Etherpad, suffixes []string) *PadCollector {
 	return &PadCollector{
 		etherpad:     etherpad,
+		suffixes:     suffixes,
 		PadGaugeDesc: prometheus.NewDesc("etherpad_toolkit_pads", "The current number of pads", []string{"suffix"}, nil),
 	}
 }
@@ -30,7 +32,7 @@ func (pc *PadCollector) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
-	sorted := helper.SortPads(allPads)
+	sorted := helper.GroupPadsBySuffixes(allPads, pc.suffixes)
 
 	for suffix, pads := range sorted {
 		ch <- prometheus.MustNewConstMetric(