Merge pull request #21 from myrunes/dev

add asset crawler

Author: zekroTJA

.github/workflows/main.yml

@@ -12,11 +12,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.13
+    - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.13
-      id: go
+        go-version: ^1.14
 
     - name: Check out code
       uses: actions/checkout@v1

.gitignore

@@ -43,4 +43,6 @@ yarn-error.log*
 *.sln
 *.sw?
 
-__debug_bin
+__debug_bin
+
+fsdata/

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine as build
+FROM golang:1.14-alpine as build
 ARG RELEASE=TRUE
 WORKDIR /var/myrunes
 ADD . .

Makefile

@@ -72,7 +72,7 @@ lint:
 PHONY += run
 run:
 	$(GO) run -v \
-		$(CURDIR)/cmd/server/*.go -c $(CONFIG)
+		$(CURDIR)/cmd/server/*.go -c $(CONFIG) -skipFetch
 
 PHONY += cleanup
 cleanup:

cmd/server/main.go

@@ -1,25 +1,95 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
+	"github.com/myrunes/backend/pkg/ddragon"
+	"github.com/myrunes/backend/pkg/lifecycletimer"
+
+	"github.com/myrunes/backend/internal/assets"
 	"github.com/myrunes/backend/internal/caching"
 	"github.com/myrunes/backend/internal/config"
 	"github.com/myrunes/backend/internal/database"
-	"github.com/myrunes/backend/internal/ddragon"
 	"github.com/myrunes/backend/internal/logger"
 	"github.com/myrunes/backend/internal/mailserver"
+	"github.com/myrunes/backend/internal/storage"
 	"github.com/myrunes/backend/internal/webserver"
 )
 
 var (
-	flagConfig = flag.String("c", "config.yml", "config file location")
+	flagConfig    = flag.String("c", "config.yml", "config file location")
+	flagSkipFetch = flag.Bool("skipFetch", false, "skip avatar asset fetching")
 )
 
+func initStorage(c *config.Main) (st storage.Middleware, err error) {
+	var cfg interface{}
+
+	switch c.Storage.Typ {
+	case "file", "fs":
+		st = new(storage.File)
+		cfg = *c.Storage.File
+	case "minio", "s3":
+		st = new(storage.Minio)
+		cfg = *c.Storage.Minio
+	default:
+		return nil, errors.New("invalid storage type")
+	}
+
+	if cfg == nil {
+		return nil, errors.New("invalid storage config")
+	}
+
+	err = st.Init(cfg)
+
+	return
+}
+
+func fetchAssets(a *assets.AvatarHandler) error {
+	if *flagSkipFetch {
+		return nil
+	}
+
+	cChamps := make(chan string)
+	cError := make(chan error)
+
+	go a.FetchAll(cChamps, cError)
+
+	go func() {
+		for _, c := range ddragon.DDragonInstance.Champions {
+			cChamps <- c.UID
+		}
+		close(cChamps)
+	}()
+
+	for err := range cError {
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func refetch(a *assets.AvatarHandler) {
+	var err error
+
+	logger.Info("DDRAGON :: refetch")
+	if ddragon.DDragonInstance, err = ddragon.Fetch("latest"); err != nil {
+		logger.Error("DDRAGON :: failed polling data from ddragon: %s", err.Error())
+	}
+
+	logger.Info("ASSETHANDLER :: refetch")
+	if err = fetchAssets(a); err != nil {
+		logger.Fatal("ASSETHANDLER :: failed fetching assets: %s", err.Error())
+	}
+}
+
 func main() {
 	flag.Parse()
 
@@ -79,14 +149,31 @@ func main() {
 		db.Close()
 	}()
 
-	logger.Info("MAILSERVER :: initialization")
-	ms, err := mailserver.NewMailServer(cfg.MailServer, "[email protected]", "myrunes")
+	logger.Info("STORAGE :: initialization")
+	st, err := initStorage(cfg)
 	if err != nil {
-		logger.Fatal("MAILSERVER :: failed connecting to mail account: %s", err.Error())
+		logger.Fatal("STORAGE :: failed initializing storage: %s", err.Error())
+	}
+
+	logger.Info("ASSETHANDLER :: initialization")
+	avatarAssetsHandler := assets.NewAvatarHandler(st)
+	if err = fetchAssets(avatarAssetsHandler); err != nil {
+		logger.Fatal("ASSETHANDLER :: failed fetching assets: %s", err.Error())
+	}
+
+	var ms *mailserver.MailServer
+	if cfg.MailServer != nil {
+		logger.Info("MAILSERVER :: initialization")
+		ms, err = mailserver.NewMailServer(cfg.MailServer, "[email protected]", "myrunes")
+		if err != nil {
+			logger.Fatal("MAILSERVER :: failed connecting to mail account: %s", err.Error())
+		}
+		logger.Info("MAILSERVER :: started")
+	} else {
+		logger.Warning("MAILSERVER :: mail server is disabled due to missing configuration")
 	}
-	logger.Info("MAILSERVER :: started")
 
-	var cache caching.Middleware
+	var cache caching.CacheMiddleware
 	if cfg.Redis != nil && cfg.Redis.Enabled {
 		cache = caching.NewRedis(cfg.Redis)
 	} else {
@@ -95,7 +182,7 @@ func main() {
 	cache.SetDatabase(db)
 
 	logger.Info("WEBSERVER :: initialization")
-	ws, err := webserver.NewWebServer(db, cache, ms, cfg.WebServer)
+	ws, err := webserver.NewWebServer(db, cache, ms, avatarAssetsHandler, cfg.WebServer)
 	if err != nil {
 		logger.Fatal("WEBSERVER :: failed creating web server: %s", err.Error())
 	}
@@ -106,17 +193,11 @@ func main() {
 	}()
 	logger.Info("WEBSERVER :: started")
 
-	// Lifecycle Timer was used to clean up expierd
-	// sessions which is no more necessary after
-	// implementation of JWT tokens.
-	// Just keeping this here in case of this may
-	// be needed some time later.
-	// lct := lifecycletimer.New(5 * time.Minute).
-	// 	Handle(func() {
-	// 	}).
-	// 	Start()
-	// defer lct.Stop()
-	// logger.Info("LIFECYCLETIMER :: started")
+	lct := lifecycletimer.New(24 * time.Hour).
+		Handle(func() { refetch(avatarAssetsHandler) }).
+		Start()
+	defer lct.Stop()
+	logger.Info("LIFECYCLETIMER :: started")
 
 	sc := make(chan os.Signal, 1)
 	signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, os.Kill)

cmd/testing/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"fmt"
 
-	"github.com/myrunes/backend/internal/ddragon"
+	"github.com/myrunes/backend/pkg/ddragon"
 )
 
 func main() {

go.mod

@@ -1,24 +1,25 @@
 module github.com/myrunes/backend
 
-go 1.13
+go 1.14
 
 require (
 	github.com/alexedwards/argon2id v0.0.0-20200420065805-90c52fcb498a
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/ghodss/yaml v1.0.0
+	github.com/go-ini/ini v1.60.2 // indirect
 	github.com/go-ozzo/ozzo-routing v2.1.4+incompatible // indirect
-	github.com/go-redis/redis v6.15.7+incompatible
-	github.com/golang/gddo v0.0.0-20200324184333-3c2cc9a6329d // indirect
-	github.com/onsi/ginkgo v1.12.0 // indirect
-	github.com/onsi/gomega v1.9.0 // indirect
+	github.com/go-redis/redis v6.15.9+incompatible
+	github.com/minio/minio-go v6.0.14+incompatible
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/qiangxue/fasthttp-routing v0.0.0-20160225050629-6ccdc2a18d87
-	github.com/valyala/fasthttp v1.12.0
+	github.com/valyala/fasthttp v1.15.1
 	github.com/zekroTJA/ratelimit v0.0.0-20190321090824-219ca33049a5
-	github.com/zekroTJA/timedmap v0.0.0-20200421070859-4302645f33f5
-	go.mongodb.org/mongo-driver v1.3.2
-	golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a
+	github.com/zekroTJA/shinpuru v0.0.0-20200829092918-bf62c98441a9
+	github.com/zekroTJA/timedmap v1.2.0
+	go.mongodb.org/mongo-driver v1.3.3
+	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )