diff --git a/.gitattributes b/.gitattributes old mode 100755 new mode 100644 diff --git a/CHANGELOG.md b/CHANGELOG.md old mode 100755 new mode 100644 diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 diff --git a/README.md b/README.md old mode 100755 new mode 100644 diff --git a/internal/mdathome/main.go b/internal/mdathome/main.go index 7e3cf49..cda4088 100644 --- a/internal/mdathome/main.go +++ b/internal/mdathome/main.go @@ -14,7 +14,7 @@ import ( "time" "github.com/gorilla/mux" - "github.com/lflare/mdathome-golang/pkg/diskcache" + "github.com/lflare/mdathome-golang/pkg/cache/boltdb" ) var clientSettings = ClientSettings{ @@ -32,7 +32,7 @@ var clientSettings = ClientSettings{ VerifyImageIntegrity: false, // Default to not verify image integrity } var serverResponse ServerResponse -var cache *diskcache.Cache +var cache *boltdb.BoltCache var timeLastRequest time.Time var running = true var client *http.Client @@ -227,9 +227,9 @@ func ShrinkDatabase() { loadClientSettings() saveClientSettings() - // Prepare diskcache + // Prepare boltdb log.Println("Preparing database...") - cache = diskcache.New(clientSettings.CacheDirectory, 0, 0, 0, 0) + cache = boltdb.New(clientSettings.CacheDirectory, 0, 0, 0, 0) defer cache.Close() // Attempts to start cache shrinking @@ -246,8 +246,8 @@ func StartServer() { loadClientSettings() saveClientSettings() - // Prepare diskcache - cache = diskcache.New( + // Prepare boltdb + cache = boltdb.New( clientSettings.CacheDirectory, clientSettings.MaxCacheSizeInMebibytes*1024*1024, clientSettings.CacheScanIntervalInSeconds, diff --git a/pkg/diskcache/diskcache.go b/pkg/cache/boltdb/cache.go similarity index 80% rename from pkg/diskcache/diskcache.go rename to pkg/cache/boltdb/cache.go index 8ec3c76..2edc874 100644 --- a/pkg/diskcache/diskcache.go +++ b/pkg/cache/boltdb/cache.go @@ -1,4 +1,4 @@ -package diskcache +package boltdb import ( "crypto/md5" @@ -10,10 +10,23 @@ import ( "os" "sort" "time" + + "github.com/boltdb/bolt" + "github.com/lflare/mdathome-golang/pkg/cache" ) +// BoltCache is a struct that represents a cache object +type BoltCache struct { + directory string + cacheLimit int + cacheScanInterval int + cacheRefreshAge int + maxCacheScanTime int + database *bolt.DB +} + // DeleteFile takes an absolute path to a file and deletes it -func (c *Cache) DeleteFile(file string) error { +func (c *BoltCache) DeleteFile(file string) error { dir, file := file[0:2]+"/"+file[2:4]+"/"+file[4:6], file // Delete file off disk @@ -35,7 +48,7 @@ func (c *Cache) DeleteFile(file string) error { } // Get takes a key, hashes it, and returns the corresponding file in the directory -func (c *Cache) Get(key string) (resp []byte, err error) { +func (c *BoltCache) Get(key string) (resp []byte, err error) { // Check for empty cache key if len(key) == 0 { return nil, fmt.Errorf("Empty cache key") @@ -64,7 +77,7 @@ func (c *Cache) Get(key string) (resp []byte, err error) { if err != nil { size := len(file) timestamp := time.Now().Unix() - keyPair = KeyPair{key, timestamp, size} + keyPair = cache.KeyPair{Key: key, Timestamp: timestamp, Size: size} } // Update timestamp @@ -83,7 +96,7 @@ func (c *Cache) Get(key string) (resp []byte, err error) { } // Set takes a key, hashes it, and saves the `resp` bytearray into the corresponding file -func (c *Cache) Set(key string, resp []byte) error { +func (c *BoltCache) Set(key string, resp []byte) error { // Check for empty cache key if len(key) == 0 { return fmt.Errorf("Empty cache key") @@ -109,7 +122,7 @@ func (c *Cache) Set(key string, resp []byte) error { // Update database size := len(resp) timestamp := time.Now().Unix() - keyPair := KeyPair{key, timestamp, size} + keyPair := cache.KeyPair{Key: key, Timestamp: timestamp, Size: size} // Set database entry err = c.setEntry(keyPair) @@ -123,23 +136,23 @@ func (c *Cache) Set(key string, resp []byte) error { } // UpdateCacheLimit allows for updating of cache limit= -func (c *Cache) UpdateCacheLimit(cacheLimit int) { +func (c *BoltCache) UpdateCacheLimit(cacheLimit int) { c.cacheLimit = cacheLimit } // UpdateCacheScanInterval allows for updating of cache scanning interval -func (c *Cache) UpdateCacheScanInterval(cacheScanInterval int) { +func (c *BoltCache) UpdateCacheScanInterval(cacheScanInterval int) { c.cacheScanInterval = cacheScanInterval } // UpdateCacheRefreshAge allows for updating of cache refresh age -func (c *Cache) UpdateCacheRefreshAge(cacheRefreshAge int) { +func (c *BoltCache) UpdateCacheRefreshAge(cacheRefreshAge int) { c.cacheRefreshAge = cacheRefreshAge } // StartBackgroundThread starts a background thread that automatically scans the directory and removes older files // when cache exceeds size limits -func (c *Cache) StartBackgroundThread() { +func (c *BoltCache) StartBackgroundThread() { for { // Retrieve cache information size, keys, err := c.loadCacheInfo() @@ -148,12 +161,12 @@ func (c *Cache) StartBackgroundThread() { } // Log - log.Printf("Current diskcache size: %s, limit: %s", ByteCountIEC(size), ByteCountIEC(c.cacheLimit)) + log.Printf("Current diskcache size: %s, limit: %s", cache.ByteCountIEC(size), cache.ByteCountIEC(c.cacheLimit)) // If size is bigger than configured byte limit, keep deleting last recently used files if size > c.cacheLimit { // Get ready to shrink cache - log.Printf("Shrinking diskcache size: %s, limit: %s", ByteCountIEC(size), ByteCountIEC(c.cacheLimit)) + log.Printf("Shrinking diskcache size: %s, limit: %s", cache.ByteCountIEC(size), cache.ByteCountIEC(c.cacheLimit)) deletedSize := 0 deletedItems := 0 @@ -185,7 +198,7 @@ func (c *Cache) StartBackgroundThread() { } // Log success - log.Printf("Successfully shrunk diskcache by: %s, %d items", ByteCountIEC(deletedSize), deletedItems) + log.Printf("Successfully shrunk diskcache by: %s, %d items", cache.ByteCountIEC(deletedSize), deletedItems) } // Sleep till next execution @@ -194,7 +207,7 @@ func (c *Cache) StartBackgroundThread() { } // loadCacheInfo -func (c *Cache) loadCacheInfo() (int, []KeyPair, error) { +func (c *BoltCache) loadCacheInfo() (int, []cache.KeyPair, error) { // Create running variables totalSize := 0 @@ -210,14 +223,14 @@ func (c *Cache) loadCacheInfo() (int, []KeyPair, error) { } // Sort cache by access time - sort.Sort(ByTimestamp(keyPairs)) + sort.Sort(cache.ByTimestamp(keyPairs)) // Return running variables return totalSize, keyPairs, err } // Close closes the database -func (c *Cache) Close() { +func (c *BoltCache) Close() { c.database.Close() } @@ -236,8 +249,8 @@ func getCacheKey(key string) (string, string) { } // New returns a new Cache that will store files in basePath -func New(directory string, cacheLimit int, cacheScanInterval int, cacheRefreshAge int, maxCacheScanTime int) *Cache { - cache := Cache{ +func New(directory string, cacheLimit int, cacheScanInterval int, cacheRefreshAge int, maxCacheScanTime int) *BoltCache { + cache := BoltCache{ directory: directory, cacheLimit: cacheLimit, cacheScanInterval: cacheScanInterval, diff --git a/pkg/diskcache/database.go b/pkg/cache/boltdb/database.go similarity index 87% rename from pkg/diskcache/database.go rename to pkg/cache/boltdb/database.go index 839eb1a..41e88cb 100644 --- a/pkg/diskcache/database.go +++ b/pkg/cache/boltdb/database.go @@ -1,4 +1,4 @@ -package diskcache +package boltdb import ( "encoding/json" @@ -10,10 +10,11 @@ import ( "time" "github.com/boltdb/bolt" + "github.com/lflare/mdathome-golang/pkg/cache" ) // setEntry adds or modifies an entry in the database from a keyPair -func (c *Cache) setEntry(keyPair KeyPair) error { +func (c *BoltCache) setEntry(keyPair cache.KeyPair) error { // Marshal keyPair struct into bytes keyPairBytes, err := json.Marshal(keyPair) if err != nil { @@ -34,7 +35,7 @@ func (c *Cache) setEntry(keyPair KeyPair) error { } // deleteEntry deletes an entry from database from a key -func (c *Cache) deleteEntry(key string) error { +func (c *BoltCache) deleteEntry(key string) error { // Update database and delete entry err := c.database.Update(func(tx *bolt.Tx) error { err := tx.Bucket([]byte("KEYS")).Delete([]byte(key)) @@ -49,9 +50,9 @@ func (c *Cache) deleteEntry(key string) error { } // getEntry retrieves an entry from the database from a key -func (c *Cache) getEntry(key string) (KeyPair, error) { +func (c *BoltCache) getEntry(key string) (cache.KeyPair, error) { // Prepare empty keyPair variable - var keyPair KeyPair + var keyPair cache.KeyPair // Retrieve entry from database err := c.database.View(func(tx *bolt.Tx) error { @@ -62,7 +63,7 @@ func (c *Cache) getEntry(key string) (KeyPair, error) { } // Unmarshal keyPairBytes into previously declared keyPair - err := json.Unmarshal(keyPairBytes, &keyPair) + err := keyPair.FromJSON(keyPairBytes) if err != nil { return err } @@ -75,9 +76,9 @@ func (c *Cache) getEntry(key string) (KeyPair, error) { } // getAllKeys returns a full slice of keyPairs from the database -func (c *Cache) getAllKeys() ([]KeyPair, error) { +func (c *BoltCache) getAllKeys() ([]cache.KeyPair, error) { // Prepare empty keyPairs reference - var keyPairs []KeyPair + var keyPairs []cache.KeyPair // Retrieve all entries from database, unmarshaling and appending to []keyPair slice err := c.database.View(func(tx *bolt.Tx) error { @@ -85,7 +86,7 @@ func (c *Cache) getAllKeys() ([]KeyPair, error) { b := tx.Bucket([]byte("KEYS")) // Create slice of keypairs of size of bucket - keyPairs = make([]KeyPair, b.Stats().KeyN) + keyPairs = make([]cache.KeyPair, b.Stats().KeyN) index := 0 // Prepare timer @@ -95,10 +96,10 @@ func (c *Cache) getAllKeys() ([]KeyPair, error) { cur := b.Cursor() for key, keyPairBytes := cur.First(); key != nil; key, keyPairBytes = cur.Next() { // Prepare empty keyPair struct - var keyPair KeyPair + var keyPair cache.KeyPair // Unmarshal bytes - err := json.Unmarshal(keyPairBytes, &keyPair) + err := keyPair.FromJSON(keyPairBytes) if err != nil { return err } @@ -121,7 +122,7 @@ func (c *Cache) getAllKeys() ([]KeyPair, error) { } // ShrinkDatabase manually re-creates the cache.db file and effectively shrinks it -func (c *Cache) ShrinkDatabase() error { +func (c *BoltCache) ShrinkDatabase() error { // Hook on to SIGTERM sigtermChannel := make(chan os.Signal) signal.Notify(sigtermChannel, os.Interrupt, syscall.SIGTERM) @@ -164,7 +165,7 @@ func (c *Cache) ShrinkDatabase() error { return nil } -func (c *Cache) openDB() error { +func (c *BoltCache) openDB() error { // Open BoltDB database options := c.getOptions() database, err := bolt.Open(c.directory+"/cache.db", 0600, options) @@ -178,7 +179,7 @@ func (c *Cache) openDB() error { } // setupDB initialises the BoltDB database -func (c *Cache) setupDB() error { +func (c *BoltCache) setupDB() error { // Create cache directory if not exists err := os.MkdirAll(c.directory, os.ModePerm) if err != nil { diff --git a/pkg/diskcache/options.go b/pkg/cache/boltdb/options.go similarity index 61% rename from pkg/diskcache/options.go rename to pkg/cache/boltdb/options.go index 3ccffad..a65d5ec 100644 --- a/pkg/diskcache/options.go +++ b/pkg/cache/boltdb/options.go @@ -1,12 +1,12 @@ // +build !linux -package diskcache +package boltdb import ( "github.com/boltdb/bolt" ) -func (c *Cache) getOptions() *bolt.Options { +func (c *BoltCache) getOptions() *bolt.Options { options := &bolt.Options{} return options } diff --git a/pkg/diskcache/options_linux.go b/pkg/cache/boltdb/options_linux.go similarity index 90% rename from pkg/diskcache/options_linux.go rename to pkg/cache/boltdb/options_linux.go index e3b8d6e..81557ad 100644 --- a/pkg/diskcache/options_linux.go +++ b/pkg/cache/boltdb/options_linux.go @@ -1,4 +1,4 @@ -package diskcache +package boltdb import ( "syscall" diff --git a/pkg/diskcache/shrink.go b/pkg/cache/boltdb/shrink.go similarity index 99% rename from pkg/diskcache/shrink.go rename to pkg/cache/boltdb/shrink.go index f626ffb..99f001d 100644 --- a/pkg/diskcache/shrink.go +++ b/pkg/cache/boltdb/shrink.go @@ -1,4 +1,4 @@ -package diskcache +package boltdb import ( "log" diff --git a/pkg/cache/interfaces.go b/pkg/cache/interfaces.go new file mode 100644 index 0000000..349d25d --- /dev/null +++ b/pkg/cache/interfaces.go @@ -0,0 +1,33 @@ +package cache + +import ( + "time" +) + +// ChapterCache caches a chapter locally +type ChapterCache interface { + Get(key string) ([]byte, error) + Put(key string, resp []byte) error + GetCacheLimit() int + SetCacheLimit(limit int) +} + +// SelfCleaningCache periodically cleans itself +type SelfCleaningCache interface { + GetCleanInterval() time.Duration + SetCleanInterval(interval time.Duration) + + Clean() error + BackgroundThread() +} + +// ExpiringCache automatically refreshes its keys from upstream when its TTL expires +type ExpiringCache interface { + GetTTL() time.Duration + SetTTL(time.Duration) +} + +// FSCache is located at a filesystem path +type FSCache interface { + GetDirectory() string +} diff --git a/pkg/diskcache/structs.go b/pkg/cache/keypair.go similarity index 66% rename from pkg/diskcache/structs.go rename to pkg/cache/keypair.go index db561de..521ad73 100644 --- a/pkg/diskcache/structs.go +++ b/pkg/cache/keypair.go @@ -1,21 +1,10 @@ -package diskcache +package cache import ( + "encoding/json" "time" - - "github.com/boltdb/bolt" ) -// Cache is a struct that represents a cache object -type Cache struct { - directory string - cacheLimit int - cacheScanInterval int - cacheRefreshAge int - maxCacheScanTime int - database *bolt.DB -} - // KeyPair is a struct that represents a cache key in database type KeyPair struct { Key string @@ -26,6 +15,17 @@ type KeyPair struct { // UpdateTimestamp allows for updating of a KeyPair timestamp field func (a *KeyPair) UpdateTimestamp() { a.Timestamp = time.Now().Unix() } +// ToJSON marshals a KeyPair into a JSON byte slice +func (a *KeyPair) ToJSON() ([]byte, error) { + return json.Marshal(a) +} + +// FromJSON unmarshals a KeyPair from a JSON byte slice +func (a *KeyPair) FromJSON(data []byte) error { + err := json.Unmarshal(data, &a) + return err +} + // ByTimestamp is a sortable slice of KeyPair based off timestamp type ByTimestamp []KeyPair diff --git a/pkg/diskcache/utils.go b/pkg/cache/utils.go similarity index 95% rename from pkg/diskcache/utils.go rename to pkg/cache/utils.go index 4cdda82..2b8df55 100644 --- a/pkg/diskcache/utils.go +++ b/pkg/cache/utils.go @@ -1,4 +1,4 @@ -package diskcache +package cache import ( "fmt"