Merge pull request #44 from SpectoLabs/develop

Moving from Redis to BoltDB for persistance

Author: Karolis Rusenas

Vagrantfile

@@ -14,16 +14,16 @@ sudo service redis-server restart
 
 # Download a pre-built hoverfly binary to avoid dealing with Go dependencies
 
-wget https://storage.googleapis.com/hoverfly-binaries/latest/hoverfly_v0.4_linux_amd64
-mv hoverfly_v0.4_linux_amd64 hoverfly && chmod +x hoverfly
+wget https://github.com/SpectoLabs/hoverfly/releases/download/v0.4.2/hoverfly_v0.4.2_linux_amd64
+mv hoverfly_v0.4.2_linux_amd64 hoverfly && chmod +x hoverfly
 
 # Symlink the webUI static files
 
 ln -s /vagrant/static /home/vagrant/static
 
-# Start hoverfly in the background in "virtualize" mode, pass logging output 
-# to hoverfly.log and save the hoverfly PID into a file so you can kill the process 
-# with 'kill -SIGTERM $(cat hoverfly_pid)' 
+# Start hoverfly in the background in "virtualize" mode, pass logging output
+# to hoverfly.log and save the hoverfly PID into a file so you can kill the process
+# with 'kill -SIGTERM $(cat hoverfly_pid)'
 
 nohup ./hoverfly > hoverfly.log 2>&1 & echo $! > hoverfly_pid &
 

admin.go

@@ -35,8 +35,7 @@ func (d *DBClient) startAdminInterface() {
 
 	// admin interface starting message
 	log.WithFields(log.Fields{
-		"RedisAddress": AppConfig.redisAddress,
-		"AdminPort":    AppConfig.adminInterface,
+		"AdminPort": AppConfig.adminInterface,
 	}).Info("Admin interface is starting...")
 
 	n.Run(AppConfig.adminInterface)
@@ -60,7 +59,7 @@ func getBoneRouter(d DBClient) *bone.Mux {
 
 // AllRecordsHandler returns JSON content type http response
 func (d *DBClient) AllRecordsHandler(w http.ResponseWriter, req *http.Request) {
-	records, err := d.getAllRecords()
+	records, err := d.cache.GetAllRequests()
 
 	if err == nil {
 
@@ -79,8 +78,7 @@ func (d *DBClient) AllRecordsHandler(w http.ResponseWriter, req *http.Request) {
 		}
 	} else {
 		log.WithFields(log.Fields{
-			"Error":        err.Error(),
-			"PasswordUsed": AppConfig.redisPassword,
+			"Error": err.Error(),
 		}).Error("Failed to get data from cache!")
 
 		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
@@ -128,7 +126,7 @@ func (d *DBClient) ImportRecordsHandler(w http.ResponseWriter, req *http.Request
 			} else {
 				// recalculating request hash and storing it in database
 				r := request{details: pl.Request}
-				d.cache.set(r.hash(), bts)
+				d.cache.Set([]byte(r.hash()), bts)
 			}
 		}
 		response.Message = fmt.Sprintf("%d requests imported successfully", len(payloads))
@@ -143,7 +141,7 @@ func (d *DBClient) ImportRecordsHandler(w http.ResponseWriter, req *http.Request
 }
 
 func (d *DBClient) DeleteAllRecordsHandler(w http.ResponseWriter, req *http.Request) {
-	err := d.deleteAllRecords()
+	err := d.cache.DeleteBucket(d.cache.requestsBucket)
 
 	w.Header().Set("Content-Type", "application/json")
 

cache.go

@@ -1,169 +1,99 @@
+// redis cache backend implementation for persistent storage
+
 package main
 
 import (
-	"bytes"
+	"encoding/json"
 	"fmt"
-	"os"
-	"strconv"
 
 	log "github.com/Sirupsen/logrus"
-	"github.com/garyburd/redigo/redis"
+	"github.com/boltdb/bolt"
 )
 
-const prefix = "genproxy:"
+const requestsBucketName = "rqbucket"
 
 type Cache struct {
-	pool   *redis.Pool
-	prefix string
+	db             *bolt.DB
+	requestsBucket []byte
 }
 
-// set records a key in cache (redis)
-func (c *Cache) set(key string, value interface{}) error {
-
-	if c.prefix == "" {
-		c.prefix = prefix
-	}
-
-	client := c.pool.Get()
-	defer client.Close()
-
-	_, err := client.Do("SET", fmt.Sprintf(c.prefix+key), value)
-
+func getDB(name string) *bolt.DB {
+	db, err := bolt.Open(name, 0600, nil)
 	if err != nil {
-		// getting payload for debugging
-		var payload string
-		switch v := value.(type) {
-		case bytes.Buffer: // return as is
-			payload = v.String() // Here v is of type bytes.Buffer
-		}
-		log.WithFields(log.Fields{
-			"error":   err.Error(),
-			"key":     fmt.Sprintf(c.prefix + key),
-			"payload": payload,
-		}).Error("Failed to SET key...")
-	} else {
-		log.WithFields(log.Fields{
-			"key": fmt.Sprintf(c.prefix + key),
-		}).Info("Key/value SET successfuly!")
-	}
-
-	return err
-}
-
-// getAllKeys returns all keys for specified (or default) prefix
-func (c *Cache) getAllKeys() ([]string, error) {
-
-	client := c.pool.Get()
-	defer client.Close()
-
-	values, err := redis.Strings(client.Do("KEYS", fmt.Sprintf(c.prefix+"*")))
-
-	return values, err
-}
-
-// getAllValues returns values for specified keys
-func (c *Cache) getAllValues(keys []string) ([]string, error) {
-
-	log.WithFields(log.Fields{
-		"keys": keys,
-	}).Debug("Getting all supplied values")
-
-	client := c.pool.Get()
-	defer client.Close()
-
-	// preparing keys
-	var args []interface{}
-	for _, k := range keys {
-		args = append(args, k)
+		log.Fatal(err)
 	}
 
-	jsonStr, err := redis.Strings(client.Do("MGET", args...))
-
-	log.WithFields(log.Fields{
-		"keys": keys,
-	}).Debug("Returning supplied values")
-
-	return jsonStr, err
-
+	return db
 }
 
-// get returns key from cache
-func (c *Cache) get(key string) (interface{}, error) {
-
-	client := c.pool.Get()
-	defer client.Close()
-
-	value, err := client.Do("GET", fmt.Sprintf(c.prefix+key))
-
-	if err != nil {
-		log.WithFields(log.Fields{
-			"error": err.Error(),
-			"key":   fmt.Sprintf(c.prefix + key),
-		}).Error("Failed to GET key...")
-	} else {
-		log.WithFields(log.Fields{
-			"key": fmt.Sprintf(c.prefix + key),
-		}).Debug("Key found!")
-	}
+func (c *Cache) Set(key, value []byte) error {
+	err := c.db.Update(func(tx *bolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(c.requestsBucket)
+		if err != nil {
+			return err
+		}
+		err = bucket.Put(key, value)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
 
-	return value, err
+	return err
 }
 
-// delete removes specified entry from cache
-func (c *Cache) delete(key string) error {
-	if c.prefix == "" {
-		c.prefix = prefix
-	}
-
-	client := c.pool.Get()
-	defer client.Close()
-
-	_, err := client.Do("DEL", key)
-
-	return err
+func (c *Cache) Get(key []byte) (value []byte, err error) {
+	err = c.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(c.requestsBucket)
+		if bucket == nil {
+			return fmt.Errorf("Bucket %q not found!", c.requestsBucket)
+		}
+		value = bucket.Get(key)
+		return nil
+	})
 
+	return
 }
 
-// getRedisPool returns thread safe Redis connection pool
-func getRedisPool() *redis.Pool {
-
-	// getting redis connection
-	maxConnections := 10
-	mc := os.Getenv("MaxConnections")
-	if mc != "" {
-		maxCons, err := strconv.Atoi(mc)
-		if err != nil {
-			maxConnections = 10
-		} else {
-			maxConnections = maxCons
+func (c *Cache) GetAllRequests() (payloads []Payload, err error) {
+	err = c.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(c.requestsBucket)
+		if b == nil {
+			// bucket doesn't exist
+			return nil
 		}
-	}
+		c := b.Cursor()
 
-	// getting redis client for state storing
-	redisPool := redis.NewPool(func() (redis.Conn, error) {
-		c, err := redis.Dial("tcp", AppConfig.redisAddress)
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			var pl Payload
+			err = json.Unmarshal(v, &pl)
 
-		if err != nil {
-			log.WithFields(log.Fields{
-				"Error": err.Error(),
-			}).Warn("Failed to create Redis connection pool!")
-			return nil, err
-		}
-		if AppConfig.redisPassword != "" {
-			if _, err := c.Do("AUTH", AppConfig.redisPassword); err != nil {
+			if err != nil {
 				log.WithFields(log.Fields{
-					"Error":        err.Error(),
-					"PasswordUsed": AppConfig.redisPassword,
-				}).Warn("Failed to authenticate to Redis!")
-				c.Close()
-				return nil, err
+					"error": err.Error(),
+					"json":  v,
+				}).Warning("Failed to deserialize json")
 			} else {
-				log.Debug("Authenticated to Redis successfully! ")
+				payloads = append(payloads, pl)
 			}
 		}
+		return nil
+	})
+	return
+}
 
-		return c, err
-	}, maxConnections)
-
-	return redisPool
+func (c *Cache) DeleteBucket(name []byte) (err error) {
+	err = c.db.Update(func(tx *bolt.Tx) error {
+		err = tx.DeleteBucket(name)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err.Error(),
+				"name":  string(name),
+			}).Warning("Failed to delete bucket")
+			return err
+		} else {
+			return nil
+		}
+	})
+	return
 }