Skip to content

Commit 1571316

Browse files
man90esman90es
authored
Add a route with a list of all cached requests (#22)
1 parent b56bc19 commit 1571316

10 files changed

+213
-7
lines changed

‎Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.21-alpine AS build
1+
FROM golang:1.22-alpine AS build
22
RUN apk add --no-cache git
33
WORKDIR /src/bdo-rest-api
44
COPY . .

‎README.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33
A scraper for Black Desert Online player in-game data with a REST API. It currently supports European, North American, and South American servers (Korean server support is in progress).
44

55
## Projects using this API
6-
- [BDO Leaderboards](https://bdo.hemlo.cc/leaderboards) ([Source](https://github.com/man90es/BDO-Leaderboards)): web-based leaderboards for Black Desert Online.
7-
- [Ikusa](https://ikusa.site) ([Source](https://github.com/sch-28/ikusa_api)): a powerful tool that allows you to analyze your game logs and gain valuable insights into your combat performance.
6+
- BDO Leaderboards ([Website](https://bdo.hemlo.cc/leaderboards), [sources](https://github.com/man90es/BDO-Leaderboards)): web-based leaderboards for Black Desert Online.
7+
- Ikusa ([Website](https://ikusa.site), [sources](https://github.com/sch-28/ikusa_api)): powerful tool that allows you to analyze your game logs and gain valuable insights into your combat performance.
8+
- GuildYapper ([Discord server](https://discord.gg/x2nKYuu2Z2)): Discord bot with various features for BDO guilds such as guild and player history logging, and automatic trial Discord management (more features TBA).
89

910
## How to start using it
1011
There are two ways to use this scraper for your needs:

‎cache/cache.go

+5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"time"
66

77
goCache "github.com/patrickmn/go-cache"
8+
"golang.org/x/exp/maps"
89

910
"bdo-rest-api/config"
1011
"bdo-rest-api/utils"
@@ -64,3 +65,7 @@ func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, ex
6465
func (c *cache[T]) GetItemCount() int {
6566
return c.internalCache.ItemCount()
6667
}
68+
69+
func (c *cache[T]) GetKeys() []string {
70+
return maps.Keys(c.internalCache.Items())
71+
}

‎docs/openapi.json

+135
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
"type": "number"
7878
},
7979
"/guild/search": {
80+
"deprecated": true,
8081
"type": "number"
8182
}
8283
}
@@ -96,6 +97,10 @@
9697
}
9798
}
9899
},
100+
"docs": {
101+
"type": "string",
102+
"example": "https://man90es.github.io/BDO-REST-API"
103+
},
99104
"proxies": {
100105
"type": "number"
101106
},
@@ -404,6 +409,7 @@
404409
"name": "page",
405410
"in": "query",
406411
"description": "This parameter is understood by the API, but you should either omit it or set to 1. Because of how search currently works, there is never more than one page.",
412+
"deprecated": true,
407413
"schema": {
408414
"type": "number",
409415
"default": 1
@@ -600,6 +606,7 @@
600606
"summary": "Search for a guild.",
601607
"description": "Search for a guild by combination of its region and name.",
602608
"operationId": "getGuildSearch",
609+
"deprecated": true,
603610
"parameters": [
604611
{
605612
"name": "query",
@@ -691,6 +698,134 @@
691698
}
692699
}
693700
}
701+
},
702+
"/v1/cache": {
703+
"get": {
704+
"summary": "Retrieve cached routes",
705+
"operationId": "getCache",
706+
"responses": {
707+
"200": {
708+
"description": "OK.",
709+
"content": {
710+
"application/json": {
711+
"schema": {
712+
"type": "object",
713+
"properties": {
714+
"/adventurer": {
715+
"type": "array",
716+
"items": {
717+
"type": "object",
718+
"properties": {
719+
"profileTarget": {
720+
"nullable": false,
721+
"type": "string",
722+
"example": "XXX"
723+
},
724+
"region": {
725+
"nullable": false,
726+
"type": "string",
727+
"enum": [
728+
"EU",
729+
"NA",
730+
"SA"
731+
]
732+
}
733+
}
734+
}
735+
},
736+
"/adventurer/search": {
737+
"type": "array",
738+
"items": {
739+
"type": "object",
740+
"properties": {
741+
"page": {
742+
"deprecated": true,
743+
"nullable": false,
744+
"type": "number",
745+
"example": 1
746+
},
747+
"query": {
748+
"nullable": false,
749+
"type": "string",
750+
"example": "Apple"
751+
},
752+
"region": {
753+
"nullable": false,
754+
"type": "string",
755+
"enum": [
756+
"EU",
757+
"NA",
758+
"SA"
759+
]
760+
},
761+
"searchType": {
762+
"nullable": false,
763+
"type": "string",
764+
"enum": [
765+
"familyName",
766+
"characterName"
767+
]
768+
}
769+
}
770+
}
771+
},
772+
"/guild": {
773+
"type": "array",
774+
"items": {
775+
"type": "object",
776+
"properties": {
777+
"guildName": {
778+
"nullable": false,
779+
"type": "string",
780+
"example": "TumblrGirls"
781+
},
782+
"region": {
783+
"nullable": false,
784+
"type": "string",
785+
"enum": [
786+
"EU",
787+
"NA",
788+
"SA"
789+
]
790+
}
791+
}
792+
}
793+
},
794+
"/guild/search": {
795+
"deprecated": true,
796+
"type": "array",
797+
"items": {
798+
"type": "object",
799+
"properties": {
800+
"page": {
801+
"nullable": false,
802+
"type": "number",
803+
"example": 1
804+
},
805+
"query": {
806+
"nullable": false,
807+
"type": "string",
808+
"example": "TumblrGirls"
809+
},
810+
"region": {
811+
"nullable": false,
812+
"type": "string",
813+
"enum": [
814+
"EU",
815+
"NA",
816+
"SA"
817+
]
818+
}
819+
}
820+
}
821+
}
822+
}
823+
}
824+
}
825+
}
826+
}
827+
}
828+
}
694829
}
695830
}
696831
}

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ require (
1919
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
2020
github.com/golang/protobuf v1.5.3 // indirect
2121
github.com/kennygrant/sanitize v1.2.4 // indirect
22+
github.com/sa-/slicefunk v0.1.4 // indirect
2223
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
2324
github.com/temoto/robotstxt v1.1.2 // indirect
25+
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
2426
golang.org/x/net v0.19.0 // indirect
2527
golang.org/x/text v0.14.0 // indirect
2628
google.golang.org/appengine v1.6.8 // indirect

‎go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK
6767
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
6868
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
6969
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
70+
github.com/sa-/slicefunk v0.1.4 h1:fCgDllo0nYVywdREyJm53BQ5rfMW8pin57yNVpyPxNU=
71+
github.com/sa-/slicefunk v0.1.4/go.mod h1:k0abNpV9EW8LIPl2+Hc9RiKsojKmsUhNNGFyMpjMTCI=
7072
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
7173
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
7274
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
@@ -82,6 +84,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
8284
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
8385
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
8486
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
87+
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
88+
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
8589
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
8690
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
8791
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=

‎handlers/GetAdventurerSearch.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) {
1818
page := validators.ValidatePageQueryParam(r.URL.Query()["page"])
1919
query, queryOk := validators.ValidateAdventurerNameQueryParam(r.URL.Query()["query"])
2020
region, regionOk := validators.ValidateRegionQueryParam(r.URL.Query()["region"])
21-
searchType := validators.ValidateSearchTypeQueryParam(r.URL.Query()["searchType"])
21+
searchTypeQueryParam := r.URL.Query()["searchType"]
22+
searchType := validators.ValidateSearchTypeQueryParam(searchTypeQueryParam)
2223

2324
if !queryOk || !regionOk {
2425
giveBadRequestResponse(w)
@@ -33,7 +34,7 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) {
3334
query = strings.ToLower(query)
3435

3536
// Look for cached data, then run the scraper if needed
36-
data, status, date, expires, found := profileSearchCache.GetRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)})
37+
data, status, date, expires, found := profileSearchCache.GetRecord([]string{region, query, searchTypeQueryParam[0], fmt.Sprint(page)})
3738
if !found {
3839
data, status = scrapers.ScrapeAdventurerSearch(region, query, searchType, page)
3940

@@ -46,7 +47,7 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) {
4647
return
4748
}
4849

49-
date, expires = profileSearchCache.AddRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)}, data, status)
50+
date, expires = profileSearchCache.AddRecord([]string{region, query, searchTypeQueryParam[0], fmt.Sprint(page)}, data, status)
5051
}
5152

5253
w.Header().Set("Date", date)

‎handlers/GetCache.go

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package handlers
2+
3+
import (
4+
"encoding/json"
5+
"net/http"
6+
"strconv"
7+
"strings"
8+
9+
sf "github.com/sa-/slicefunk"
10+
)
11+
12+
func getParseCacheKey(cacheType string) func(string) map[string]interface{} {
13+
return func(key string) map[string]interface{} {
14+
parts := strings.Split(key, ",")
15+
16+
switch cacheType {
17+
case "/adventurer":
18+
return map[string]interface{}{
19+
"region": parts[0],
20+
"profileTarget": parts[1],
21+
}
22+
case "/adventurer/search":
23+
page, _ := strconv.Atoi(parts[3])
24+
25+
return map[string]interface{}{
26+
"region": parts[0],
27+
"query": parts[1],
28+
"searhType": parts[2],
29+
"page": page,
30+
}
31+
case "/guild":
32+
return map[string]interface{}{
33+
"region": parts[0],
34+
"guildName": parts[1],
35+
}
36+
case "/guild/search":
37+
page, _ := strconv.Atoi(parts[2])
38+
39+
return map[string]interface{}{
40+
"region": parts[0],
41+
"query": parts[1],
42+
"page": page,
43+
}
44+
default:
45+
return nil
46+
}
47+
}
48+
}
49+
50+
func GetCache(w http.ResponseWriter, r *http.Request) {
51+
json.NewEncoder(w).Encode(map[string]interface{}{
52+
"/adventurer": sf.Map(profilesCache.GetKeys(), getParseCacheKey("/adventurer")),
53+
"/adventurer/search": sf.Map(profileSearchCache.GetKeys(), getParseCacheKey("/adventurer/search")),
54+
"/guild": sf.Map(guildProfilesCache.GetKeys(), getParseCacheKey("/guild")),
55+
"/guild/search": sf.Map(guildSearchCache.GetKeys(), getParseCacheKey("/guild/search")),
56+
})
57+
}

‎handlers/GetStatus.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
)
1111

1212
var initTime = time.Now()
13-
var version = "1.8.4"
13+
var version = "1.9.0"
1414

1515
func GetStatus(w http.ResponseWriter, r *http.Request) {
1616
json.NewEncoder(w).Encode(map[string]interface{}{

‎httpServer/BuildServer.go

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ func BuildServer() *http.Server {
1616
"/v1": handlers.GetStatus,
1717
"/v1/adventurer": handlers.GetAdventurer,
1818
"/v1/adventurer/search": handlers.GetAdventurerSearch,
19+
"/v1/cache": handlers.GetCache,
1920
"/v1/guild": handlers.GetGuild,
2021
"/v1/guild/search": handlers.GetGuildSearch,
2122
}, handlers.Catchall)

0 commit comments

Comments
 (0)