add: cache for remote images (#310)

* add: cache for remote image requests

* update: use default expiration for NoExpiration setting

* update: clean code

* fix: do not save empty etag

* fix: panic during type assertion

Author: HolgerHuo

config.json

@@ -10,5 +10,6 @@
   "ENABLE_EXTRA_PARAMS": false,
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
-  "DISABLE_KEEPALIVE": false
+  "DISABLE_KEEPALIVE": false,
+  "CACHE_TTL": 259200
 }

config/config.go

@@ -33,7 +33,8 @@ const (
   "ENABLE_EXTRA_PARAMS": false
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
-  "DISABLE_KEEPALIVE": false
+  "DISABLE_KEEPALIVE": false,
+  "CACHE_TTL": 259200,
 }`
 )
 
@@ -52,6 +53,7 @@ var (
 	RemoteRaw      = "./remote-raw"
 	Metadata       = "./metadata"
 	LocalHostAlias = "local"
+	RemoteCache    *cache.Cache
 )
 
 type MetaFile struct {
@@ -73,6 +75,7 @@ type WebpConfig struct {
 	ReadBufferSize    int               `json:"READ_BUFFER_SIZE"`
 	Concurrency       int               `json:"CONCURRENCY"`
 	DisableKeepalive  bool              `json:"DISABLE_KEEPALIVE"`
+	CacheTTL     	  int               `json:"CACHE_TTL"`
 }
 
 func NewWebPConfig() *WebpConfig {
@@ -89,6 +92,7 @@ func NewWebPConfig() *WebpConfig {
 		ReadBufferSize:    4096,
 		Concurrency:       262144,
 		DisableKeepalive:  false,
+		CacheTTL:          259200,
 	}
 }
 
@@ -184,6 +188,20 @@ func LoadConfig() {
 			log.Warnf("WEBP_DISABLE_KEEPALIVE is not a valid boolean, using value in config.json %t", Config.DisableKeepalive)
 		}
 	}
+	if os.Getenv("CACHE_TTL") != "" {
+		cacheTTL, err := strconv.Atoi(os.Getenv("CACHE_TTL"))
+		if err != nil {
+			log.Warnf("CACHE_TTL is not a valid integer, using value in config.json %d", Config.CacheTTL)
+		} else {
+			Config.CacheTTL = cacheTTL
+		}
+	}
+
+	if Config.CacheTTL == 0 {
+		RemoteCache = cache.New(cache.NoExpiration, 10*time.Minute)
+	} else {
+		RemoteCache = cache.New(time.Duration(Config.CacheTTL)*time.Minute, 10*time.Minute)
+	}
 
 	log.Debugln("Config init complete")
 	log.Debugln("Config", Config)

config/config_test.go

@@ -21,6 +21,7 @@ func TestLoadConfig(t *testing.T) {
 	assert.Equal(t, Config.ImgPath, "./pics")
 	assert.Equal(t, Config.ImageMap, map[string]string{})
 	assert.Equal(t, Config.ExhaustPath, "./exhaust")
+	assert.Equal(t, Config.CacheTTL, 259200)
 }
 
 func TestSwitchProxyMode(t *testing.T) {

handler/remote.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/h2non/filetype"
+	"github.com/patrickmn/go-cache"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -78,8 +79,27 @@ func downloadFile(filepath string, url string) {
 func fetchRemoteImg(url string, subdir string) config.MetaFile {
 	// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 	// How do we know if the remote img is changed? we're using hash(etag+length)
-	log.Infof("Remote Addr is %s, pinging for info...", url)
-	etag := pingURL(url)
+	var etag string
+	
+	cacheKey := subdir+":"+helper.HashString(url)
+	
+	if val, found := config.RemoteCache.Get(cacheKey); found {
+		if etagVal, ok := val.(string); ok {
+			log.Infof("Using cache for remote addr: %s", url)
+			etag = etagVal
+		} else {
+			config.RemoteCache.Delete(cacheKey)
+		}
+	} 
+	
+	if etag == "" {
+		log.Infof("Remote Addr is %s, pinging for info...", url)
+		etag = pingURL(url)
+		if etag != "" {
+			config.RemoteCache.Set(cacheKey, etag, cache.DefaultExpiration)
+		}
+	}
+	
 	metadata := helper.ReadMetadata(url, etag, subdir)
 	localRawImagePath := path.Join(config.RemoteRaw, subdir, metadata.Id)