Skip to content

Commit 67610e9

Browse files
man90esman90es
authored
Improve logging (#34)
* Implement better logging
1 parent 95990f2 commit 67610e9

File tree

9 files changed

+148
-18
lines changed

9 files changed

+148
-18
lines changed

‎config/config.go

+17-5
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
type config struct {
1313
cacheTTL time.Duration
1414
maintenanceTTL time.Duration
15+
mongoDB string
1516
mu sync.RWMutex
1617
port int
1718
proxyList []string
@@ -116,12 +117,23 @@ func GetRateLimit() int64 {
116117
return getInstance().rateLimit
117118
}
118119

119-
func PrintConfig() {
120-
fmt.Printf("Configuration:\n" +
121-
fmt.Sprintf("\tPort:\t\t%v\n", GetPort()) +
120+
func SetMongoDB(mongoDB string) {
121+
getInstance().mu.Lock()
122+
defer getInstance().mu.Unlock()
123+
getInstance().mongoDB = mongoDB
124+
}
125+
126+
func GetMongoDB() string {
127+
getInstance().mu.RLock()
128+
defer getInstance().mu.RUnlock()
129+
return getInstance().mongoDB
130+
}
131+
132+
func SprintfConfig() string {
133+
return fmt.Sprintf("\tPort:\t\t%v\n", GetPort()) +
122134
fmt.Sprintf("\tProxies:\t%v\n", GetProxyList()) +
123135
fmt.Sprintf("\tVerbosity:\t%v\n", GetVerbosity()) +
124136
fmt.Sprintf("\tCache TTL:\t%v\n", GetCacheTTL()) +
125-
fmt.Sprintf("\tRate limit:\t%v/min\n", GetRateLimit()),
126-
)
137+
fmt.Sprintf("\tRate limit:\t%v/min\n", GetRateLimit()) +
138+
fmt.Sprintf("\tMongoDB:\t%v", GetMongoDB())
127139
}

‎go.mod

+11-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/patrickmn/go-cache v2.1.0+incompatible
88
github.com/sa-/slicefunk v0.1.4
99
github.com/ulule/limiter/v3 v3.11.2
10+
go.mongodb.org/mongo-driver/v2 v2.0.0
1011
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
1112
)
1213

@@ -19,12 +20,20 @@ require (
1920
github.com/gobwas/glob v0.2.3 // indirect
2021
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
2122
github.com/golang/protobuf v1.5.3 // indirect
23+
github.com/golang/snappy v0.0.4 // indirect
2224
github.com/kennygrant/sanitize v1.2.4 // indirect
25+
github.com/klauspost/compress v1.16.7 // indirect
2326
github.com/pkg/errors v0.9.1 // indirect
2427
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
2528
github.com/temoto/robotstxt v1.1.2 // indirect
26-
golang.org/x/net v0.19.0 // indirect
27-
golang.org/x/text v0.14.0 // indirect
29+
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
30+
github.com/xdg-go/scram v1.1.2 // indirect
31+
github.com/xdg-go/stringprep v1.0.4 // indirect
32+
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
33+
golang.org/x/crypto v0.29.0 // indirect
34+
golang.org/x/net v0.21.0 // indirect
35+
golang.org/x/sync v0.9.0 // indirect
36+
golang.org/x/text v0.20.0 // indirect
2837
google.golang.org/appengine v1.6.8 // indirect
2938
google.golang.org/protobuf v1.31.0 // indirect
3039
)

‎go.sum

+22
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
5151
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
5252
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
5353
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
54+
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
55+
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
5456
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
5557
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
5658
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -61,6 +63,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
6163
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
6264
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
6365
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
66+
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
67+
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
6468
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
6569
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
6670
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -83,10 +87,22 @@ github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fx
8387
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
8488
github.com/ulule/limiter/v3 v3.11.2 h1:P4yOrxoEMJbOTfRJR2OzjL90oflzYPPmWg+dvwN2tHA=
8589
github.com/ulule/limiter/v3 v3.11.2/go.mod h1:QG5GnFOCV+k7lrL5Y8kgEeeflPH3+Cviqlqa8SVSQxI=
90+
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
91+
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
92+
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
93+
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
94+
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
95+
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
96+
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
97+
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
8698
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
99+
go.mongodb.org/mongo-driver/v2 v2.0.0 h1:Jfd7XpdZa9yk3eY774bO7SWVb30noLSirL9nKTpavhI=
100+
go.mongodb.org/mongo-driver/v2 v2.0.0/go.mod h1:nSjmNq4JUstE8IRZKTktLgMHM4F1fccL6HGX1yh+8RA=
87101
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
88102
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
89103
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
104+
golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
105+
golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
90106
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
91107
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
92108
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
@@ -115,12 +131,16 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
115131
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
116132
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
117133
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
134+
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
135+
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
118136
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
119137
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
120138
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
121139
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
122140
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
123141
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
142+
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
143+
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
124144
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
125145
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
126146
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -149,6 +169,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
149169
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
150170
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
151171
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
172+
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
173+
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
152174
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
153175
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
154176
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

‎handlers/ListenAndServe.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package handlers
22

33
import (
44
"fmt"
5-
"log"
65
"net/http"
76
"time"
87

98
"bdo-rest-api/config"
9+
"bdo-rest-api/logger"
1010
"bdo-rest-api/middleware"
1111
)
1212

@@ -26,13 +26,13 @@ func ListenAndServe() {
2626
middleware.GetRateLimitMiddleware(),
2727
)
2828

29-
log.Println("Listening for requests")
3029
srv := &http.Server{
3130
Addr: fmt.Sprintf("0.0.0.0:%v", config.GetPort()),
3231
Handler: middlewareStack(mux),
3332
IdleTimeout: 60 * time.Second,
3433
ReadTimeout: 15 * time.Second,
3534
WriteTimeout: 15 * time.Second,
3635
}
37-
log.Fatal(srv.ListenAndServe())
36+
37+
logger.Error(srv.ListenAndServe().Error())
3838
}

‎handlers/getStatus.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
)
1111

1212
var initTime = time.Now()
13-
var version = "1.9.4"
13+
var version = "1.9.5"
1414

1515
func getStatus(w http.ResponseWriter, r *http.Request) {
1616
json.NewEncoder(w).Encode(map[string]interface{}{

‎logger/Log.go

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package logger
2+
3+
import "time"
4+
5+
type Log struct {
6+
CreatedAt time.Time `json:"createdAt" bson:"createdAt"`
7+
Level string `json:"level" bson:"level"`
8+
Message string `json:"message" bson:"message"`
9+
}

‎logger/logger.go

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package logger
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
"os"
8+
"time"
9+
10+
"bdo-rest-api/config"
11+
12+
"go.mongodb.org/mongo-driver/v2/mongo"
13+
"go.mongodb.org/mongo-driver/v2/mongo/options"
14+
)
15+
16+
var (
17+
infoLogger *log.Logger
18+
errorLogger *log.Logger
19+
)
20+
var initialised = false
21+
var mongoCollection *mongo.Collection
22+
23+
func InitLogger() {
24+
mongoURI := config.GetMongoDB()
25+
26+
if len(mongoURI) > 0 {
27+
client, err := mongo.Connect(options.Client().ApplyURI(mongoURI))
28+
29+
if err != nil {
30+
panic(err)
31+
}
32+
33+
mongoCollection = client.Database("bdo-rest-api").Collection("logs")
34+
}
35+
36+
infoLogger = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime)
37+
errorLogger = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime)
38+
initialised = true
39+
40+
Info(fmt.Sprintf("API initialised, configuration loaded:\n%v", config.SprintfConfig()))
41+
}
42+
43+
func writeToMongo(level, message string) {
44+
if mongoCollection == nil {
45+
return
46+
}
47+
48+
mongoCollection.InsertOne(context.TODO(), Log{
49+
CreatedAt: time.Now(),
50+
Level: level,
51+
Message: message,
52+
})
53+
}
54+
55+
func Info(message string) {
56+
writeToMongo("INFO", message)
57+
58+
if !config.GetVerbosity() || !initialised {
59+
return
60+
}
61+
62+
infoLogger.Println(message)
63+
}
64+
65+
func Error(message string) {
66+
writeToMongo("ERROR", message)
67+
68+
if !config.GetVerbosity() || !initialised {
69+
return
70+
}
71+
72+
errorLogger.Println(message)
73+
}

‎main.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@ import (
99

1010
"bdo-rest-api/config"
1111
"bdo-rest-api/handlers"
12+
"bdo-rest-api/logger"
1213
)
1314

1415
func main() {
1516
flagCacheTTL := flag.Int("cachettl", 180, "Cache TTL in minutes")
1617
flagMaintenanceTTL := flag.Int("maintenancettl", 5, "Allows to limit how frequently scraper can check for maintenance end in minutes")
18+
flagMongo := flag.String("mongo", "", "MongoDB connection string for loggig")
1719
flagPort := flag.Int("port", 8001, "Port to catch requests on")
1820
flagProxy := flag.String("proxy", "", "Open proxy address to make requests to BDO servers")
1921
flagRateLimit := flag.Int64("ratelimit", 512, "Maximum number of requests per minute per IP")
@@ -42,9 +44,10 @@ func main() {
4244

4345
config.SetCacheTTL(time.Duration(*flagCacheTTL) * time.Minute)
4446
config.SetMaintenanceStatusTTL(time.Duration(*flagMaintenanceTTL) * time.Minute)
45-
config.SetVerbosity(*flagVerbose)
47+
config.SetMongoDB(*flagMongo)
4648
config.SetRateLimit(*flagRateLimit)
49+
config.SetVerbosity(*flagVerbose)
4750

48-
config.PrintConfig()
51+
logger.InitLogger()
4952
handlers.ListenAndServe()
5053
}

‎scrapers/scraper.go

+7-5
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ package scrapers
22

33
import (
44
"fmt"
5-
"log"
65
"time"
76

87
"bdo-rest-api/config"
8+
"bdo-rest-api/logger"
99

1010
colly "github.com/gocolly/colly/v2"
1111
)
@@ -26,10 +26,12 @@ func newScraper(region string) (s scraper) {
2626
s.c.SetProxyFunc(config.GetProxySwitcher())
2727
}
2828

29-
s.c.OnRequest(func(r *colly.Request) {
30-
if config.GetVerbosity() {
31-
log.Println("Visiting", r.URL)
32-
}
29+
s.c.OnError(func(r *colly.Response, err error) {
30+
logger.Error(fmt.Sprintf("%v", err))
31+
})
32+
33+
s.c.OnResponse(func(r *colly.Response) {
34+
logger.Info(fmt.Sprintf("Received response code for %v: %v", r.Request.URL, r.StatusCode))
3335
})
3436

3537
// Detect maintenance

0 commit comments

Comments
 (0)