Refactor cache

internal/mdathome/main.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/lflare/mdathome-golang/pkg/diskcache"
+	"github.com/lflare/mdathome-golang/pkg/cache/boltdb"
 )
 
 var clientSettings = ClientSettings{
@@ -32,7 +32,7 @@ var clientSettings = ClientSettings{
 	VerifyImageIntegrity:       false,    // Default to not verify image integrity
 }
 var serverResponse ServerResponse
-var cache *diskcache.Cache
+var cache *boltdb.BoltCache
 var timeLastRequest time.Time
 var running = true
 var client *http.Client
@@ -227,9 +227,9 @@ func ShrinkDatabase() {
 	loadClientSettings()
 	saveClientSettings()
 
-	// Prepare diskcache
+	// Prepare boltdb
 	log.Println("Preparing database...")
-	cache = diskcache.New(clientSettings.CacheDirectory, 0, 0, 0, 0)
+	cache = boltdb.New(clientSettings.CacheDirectory, 0, 0, 0, 0)
 	defer cache.Close()
 
 	// Attempts to start cache shrinking
@@ -246,8 +246,8 @@ func StartServer() {
 	loadClientSettings()
 	saveClientSettings()
 
-	// Prepare diskcache
-	cache = diskcache.New(
+	// Prepare boltdb
+	cache = boltdb.New(
 		clientSettings.CacheDirectory,
 		clientSettings.MaxCacheSizeInMebibytes*1024*1024,
 		clientSettings.CacheScanIntervalInSeconds,

pkg/diskcache/diskcache.go → pkg/cache/boltdb/cache.go

@@ -1,4 +1,4 @@
-package diskcache
+package boltdb
 
 import (
 	"crypto/md5"
@@ -10,10 +10,23 @@ import (
 	"os"
 	"sort"
 	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/lflare/mdathome-golang/pkg/cache"
 )
 
+// BoltCache is a struct that represents a cache object
+type BoltCache struct {
+	directory         string
+	cacheLimit        int
+	cacheScanInterval int
+	cacheRefreshAge   int
+	maxCacheScanTime  int
+	database          *bolt.DB
+}
+
 // DeleteFile takes an absolute path to a file and deletes it
-func (c *Cache) DeleteFile(file string) error {
+func (c *BoltCache) DeleteFile(file string) error {
 	dir, file := file[0:2]+"/"+file[2:4]+"/"+file[4:6], file
 
 	// Delete file off disk
@@ -35,7 +48,7 @@ func (c *Cache) DeleteFile(file string) error {
 }
 
 // Get takes a key, hashes it, and returns the corresponding file in the directory
-func (c *Cache) Get(key string) (resp []byte, err error) {
+func (c *BoltCache) Get(key string) (resp []byte, err error) {
 	// Check for empty cache key
 	if len(key) == 0 {
 		return nil, fmt.Errorf("Empty cache key")
@@ -64,7 +77,7 @@ func (c *Cache) Get(key string) (resp []byte, err error) {
 		if err != nil {
 			size := len(file)
 			timestamp := time.Now().Unix()
-			keyPair = KeyPair{key, timestamp, size}
+			keyPair = cache.KeyPair{Key: key, Timestamp: timestamp, Size: size}
 		}
 
 		// Update timestamp
@@ -83,7 +96,7 @@ func (c *Cache) Get(key string) (resp []byte, err error) {
 }
 
 // Set takes a key, hashes it, and saves the `resp` bytearray into the corresponding file
-func (c *Cache) Set(key string, resp []byte) error {
+func (c *BoltCache) Set(key string, resp []byte) error {
 	// Check for empty cache key
 	if len(key) == 0 {
 		return fmt.Errorf("Empty cache key")
@@ -109,7 +122,7 @@ func (c *Cache) Set(key string, resp []byte) error {
 	// Update database
 	size := len(resp)
 	timestamp := time.Now().Unix()
-	keyPair := KeyPair{key, timestamp, size}
+	keyPair := cache.KeyPair{Key: key, Timestamp: timestamp, Size: size}
 
 	// Set database entry
 	err = c.setEntry(keyPair)
@@ -123,23 +136,23 @@ func (c *Cache) Set(key string, resp []byte) error {
 }
 
 // UpdateCacheLimit allows for updating of cache limit=
-func (c *Cache) UpdateCacheLimit(cacheLimit int) {
+func (c *BoltCache) UpdateCacheLimit(cacheLimit int) {
 	c.cacheLimit = cacheLimit
 }
 
 // UpdateCacheScanInterval allows for updating of cache scanning interval
-func (c *Cache) UpdateCacheScanInterval(cacheScanInterval int) {
+func (c *BoltCache) UpdateCacheScanInterval(cacheScanInterval int) {
 	c.cacheScanInterval = cacheScanInterval
 }
 
 // UpdateCacheRefreshAge allows for updating of cache refresh age
-func (c *Cache) UpdateCacheRefreshAge(cacheRefreshAge int) {
+func (c *BoltCache) UpdateCacheRefreshAge(cacheRefreshAge int) {
 	c.cacheRefreshAge = cacheRefreshAge
 }
 
 // StartBackgroundThread starts a background thread that automatically scans the directory and removes older files
 // when cache exceeds size limits
-func (c *Cache) StartBackgroundThread() {
+func (c *BoltCache) StartBackgroundThread() {
 	for {
 		// Retrieve cache information
 		size, keys, err := c.loadCacheInfo()
@@ -148,12 +161,12 @@ func (c *Cache) StartBackgroundThread() {
 		}
 
 		// Log
-		log.Printf("Current diskcache size: %s, limit: %s", ByteCountIEC(size), ByteCountIEC(c.cacheLimit))
+		log.Printf("Current diskcache size: %s, limit: %s", cache.ByteCountIEC(size), cache.ByteCountIEC(c.cacheLimit))
 
 		// If size is bigger than configured byte limit, keep deleting last recently used files
 		if size > c.cacheLimit {
 			// Get ready to shrink cache
-			log.Printf("Shrinking diskcache size: %s, limit: %s", ByteCountIEC(size), ByteCountIEC(c.cacheLimit))
+			log.Printf("Shrinking diskcache size: %s, limit: %s", cache.ByteCountIEC(size), cache.ByteCountIEC(c.cacheLimit))
 			deletedSize := 0
 			deletedItems := 0
 
@@ -185,7 +198,7 @@ func (c *Cache) StartBackgroundThread() {
 			}
 
 			// Log success
-			log.Printf("Successfully shrunk diskcache by: %s, %d items", ByteCountIEC(deletedSize), deletedItems)
+			log.Printf("Successfully shrunk diskcache by: %s, %d items", cache.ByteCountIEC(deletedSize), deletedItems)
 		}
 
 		// Sleep till next execution
@@ -194,7 +207,7 @@ func (c *Cache) StartBackgroundThread() {
 }
 
 // loadCacheInfo
-func (c *Cache) loadCacheInfo() (int, []KeyPair, error) {
+func (c *BoltCache) loadCacheInfo() (int, []cache.KeyPair, error) {
 	// Create running variables
 	totalSize := 0
 
@@ -210,14 +223,14 @@ func (c *Cache) loadCacheInfo() (int, []KeyPair, error) {
 	}
 
 	// Sort cache by access time
-	sort.Sort(ByTimestamp(keyPairs))
+	sort.Sort(cache.ByTimestamp(keyPairs))
 
 	// Return running variables
 	return totalSize, keyPairs, err
 }
 
 // Close closes the database
-func (c *Cache) Close() {
+func (c *BoltCache) Close() {
 	c.database.Close()
 }
 
@@ -236,8 +249,8 @@ func getCacheKey(key string) (string, string) {
 }
 
 // New returns a new Cache that will store files in basePath
-func New(directory string, cacheLimit int, cacheScanInterval int, cacheRefreshAge int, maxCacheScanTime int) *Cache {
-	cache := Cache{
+func New(directory string, cacheLimit int, cacheScanInterval int, cacheRefreshAge int, maxCacheScanTime int) *BoltCache {
+	cache := BoltCache{
 		directory:         directory,
 		cacheLimit:        cacheLimit,
 		cacheScanInterval: cacheScanInterval,

pkg/diskcache/database.go → pkg/cache/boltdb/database.go

@@ -1,4 +1,4 @@
-package diskcache
+package boltdb
 
 import (
 	"encoding/json"
@@ -10,10 +10,11 @@ import (
 	"time"
 
 	"github.com/boltdb/bolt"
+	"github.com/lflare/mdathome-golang/pkg/cache"
 )
 
 // setEntry adds or modifies an entry in the database from a keyPair
-func (c *Cache) setEntry(keyPair KeyPair) error {
+func (c *BoltCache) setEntry(keyPair cache.KeyPair) error {
 	// Marshal keyPair struct into bytes
 	keyPairBytes, err := json.Marshal(keyPair)
 	if err != nil {
@@ -34,7 +35,7 @@ func (c *Cache) setEntry(keyPair KeyPair) error {
 }
 
 // deleteEntry deletes an entry from database from a key
-func (c *Cache) deleteEntry(key string) error {
+func (c *BoltCache) deleteEntry(key string) error {
 	// Update database and delete entry
 	err := c.database.Update(func(tx *bolt.Tx) error {
 		err := tx.Bucket([]byte("KEYS")).Delete([]byte(key))
@@ -49,9 +50,9 @@ func (c *Cache) deleteEntry(key string) error {
 }
 
 // getEntry retrieves an entry from the database from a key
-func (c *Cache) getEntry(key string) (KeyPair, error) {
+func (c *BoltCache) getEntry(key string) (cache.KeyPair, error) {
 	// Prepare empty keyPair variable
-	var keyPair KeyPair
+	var keyPair cache.KeyPair
 
 	// Retrieve entry from database
 	err := c.database.View(func(tx *bolt.Tx) error {
@@ -62,7 +63,7 @@ func (c *Cache) getEntry(key string) (KeyPair, error) {
 		}
 
 		// Unmarshal keyPairBytes into previously declared keyPair
-		err := json.Unmarshal(keyPairBytes, &keyPair)
+		err := keyPair.FromJSON(keyPairBytes)
 		if err != nil {
 			return err
 		}
@@ -75,17 +76,17 @@ func (c *Cache) getEntry(key string) (KeyPair, error) {
 }
 
 // getAllKeys returns a full slice of keyPairs from the database
-func (c *Cache) getAllKeys() ([]KeyPair, error) {
+func (c *BoltCache) getAllKeys() ([]cache.KeyPair, error) {
 	// Prepare empty keyPairs reference
-	var keyPairs []KeyPair
+	var keyPairs []cache.KeyPair
 
 	// Retrieve all entries from database, unmarshaling and appending to []keyPair slice
 	err := c.database.View(func(tx *bolt.Tx) error {
 		// Get bucket
 		b := tx.Bucket([]byte("KEYS"))
 
 		// Create slice of keypairs of size of bucket
-		keyPairs = make([]KeyPair, b.Stats().KeyN)
+		keyPairs = make([]cache.KeyPair, b.Stats().KeyN)
 		index := 0
 
 		// Prepare timer
@@ -95,10 +96,10 @@ func (c *Cache) getAllKeys() ([]KeyPair, error) {
 		cur := b.Cursor()
 		for key, keyPairBytes := cur.First(); key != nil; key, keyPairBytes = cur.Next() {
 			// Prepare empty keyPair struct
-			var keyPair KeyPair
+			var keyPair cache.KeyPair
 
 			// Unmarshal bytes
-			err := json.Unmarshal(keyPairBytes, &keyPair)
+			err := keyPair.FromJSON(keyPairBytes)
 			if err != nil {
 				return err
 			}
@@ -121,7 +122,7 @@ func (c *Cache) getAllKeys() ([]KeyPair, error) {
 }
 
 // ShrinkDatabase manually re-creates the cache.db file and effectively shrinks it
-func (c *Cache) ShrinkDatabase() error {
+func (c *BoltCache) ShrinkDatabase() error {
 	// Hook on to SIGTERM
 	sigtermChannel := make(chan os.Signal)
 	signal.Notify(sigtermChannel, os.Interrupt, syscall.SIGTERM)
@@ -164,7 +165,7 @@ func (c *Cache) ShrinkDatabase() error {
 	return nil
 }
 
-func (c *Cache) openDB() error {
+func (c *BoltCache) openDB() error {
 	// Open BoltDB database
 	options := c.getOptions()
 	database, err := bolt.Open(c.directory+"/cache.db", 0600, options)
@@ -178,7 +179,7 @@ func (c *Cache) openDB() error {
 }
 
 // setupDB initialises the BoltDB database
-func (c *Cache) setupDB() error {
+func (c *BoltCache) setupDB() error {
 	// Create cache directory if not exists
 	err := os.MkdirAll(c.directory, os.ModePerm)
 	if err != nil {

pkg/diskcache/options.go → pkg/cache/boltdb/options.go

@@ -1,12 +1,12 @@
 // +build !linux
 
-package diskcache
+package boltdb
 
 import (
 	"github.com/boltdb/bolt"
 )
 
-func (c *Cache) getOptions() *bolt.Options {
+func (c *BoltCache) getOptions() *bolt.Options {
 	options := &bolt.Options{}
 	return options
 }

pkg/diskcache/options_linux.go → pkg/cache/boltdb/options_linux.go

@@ -1,4 +1,4 @@
-package diskcache
+package boltdb
 
 import (
 	"syscall"

pkg/diskcache/shrink.go → pkg/cache/boltdb/shrink.go

@@ -1,4 +1,4 @@
-package diskcache
+package boltdb
 
 import (
 	"log"

pkg/cache/interfaces.go

@@ -0,0 +1,33 @@
+package cache
+
+import (
+	"time"
+)
+
+// ChapterCache caches a chapter locally
+type ChapterCache interface {
+	Get(key string) ([]byte, error)
+	Put(key string, resp []byte) error
+	GetCacheLimit() int
+	SetCacheLimit(limit int)
+}
+
+// SelfCleaningCache periodically cleans itself
+type SelfCleaningCache interface {
+	GetCleanInterval() time.Duration
+	SetCleanInterval(interval time.Duration)
+
+	Clean() error
+	BackgroundThread()
+}
+
+// ExpiringCache automatically refreshes its keys from upstream when its TTL expires
+type ExpiringCache interface {
+	GetTTL() time.Duration
+	SetTTL(time.Duration)
+}
+
+// FSCache is located at a filesystem path
+type FSCache interface {
+	GetDirectory() string
+}