11package utils
22
33import (
4+ "bytes"
5+ "encoding/json"
46 "errors"
7+ "io"
58 "net/http"
69 "razor/cache"
710 "razor/core"
11+ "regexp"
812 "time"
913
14+ "razor/core/types"
15+
1016 "github.com/PaesslerAG/jsonpath"
1117 "github.com/avast/retry-go"
1218 "github.com/gocolly/colly"
1319)
1420
15- func ( * UtilsStruct ) GetDataFromAPI (url string , localCache * cache.LocalCache ) ([]byte , error ) {
21+ func GetDataFromAPI (dataSourceURLStruct types. DataSourceURL , localCache * cache.LocalCache ) ([]byte , error ) {
1622 client := http.Client {
1723 Timeout : time .Duration (HTTPTimeout ) * time .Second ,
1824 }
19- cachedData , err := localCache .Read (url )
20- if err != nil {
21- var body []byte
22- err := retry .Do (
23- func () error {
24- response , err := client .Get (url )
25- if err != nil {
26- return err
27- }
28- defer response .Body .Close ()
29- if response .StatusCode != 200 {
30- log .Errorf ("API: %s responded with status code %d" , url , response .StatusCode )
31- return errors .New ("unable to reach API" )
32- }
33- body , err = IOInterface .ReadAll (response .Body )
34- if err != nil {
35- return err
36- }
37- return nil
38- }, retry .Attempts (2 ), retry .Delay (time .Second * 2 ))
39- if err != nil {
40- return nil , err
25+ cachedData , cachedErr := localCache .Read (dataSourceURLStruct .URL )
26+ if cachedErr != nil {
27+ var response []byte
28+ switch dataSourceURLStruct .Type {
29+ case "GET" :
30+ err := retry .Do (
31+ func () error {
32+ responseBody , err := ProcessRequest (client , dataSourceURLStruct , nil )
33+ if err != nil {
34+ log .Error ("Error in processing GET request: " , err )
35+ return err
36+ }
37+ response = responseBody
38+ return nil
39+ }, retry .Attempts (core .ProcessRequestRetryAttempts ), retry .Delay (time .Second * time .Duration (core .ProcessRequestRetryDelay )))
40+ if err != nil {
41+ return nil , err
42+ }
43+ case "POST" :
44+ postBody , err := json .Marshal (dataSourceURLStruct .Body )
45+ if err != nil {
46+ log .Errorf ("Error in marshalling body of a POST request URL %s: %v" , dataSourceURLStruct .URL , err )
47+ return nil , err
48+ }
49+ requestBody := bytes .NewBuffer (postBody )
50+ err = retry .Do (
51+ func () error {
52+ responseBody , err := ProcessRequest (client , dataSourceURLStruct , requestBody )
53+ if err != nil {
54+ log .Error ("Error in processing POST request: " , err )
55+ return err
56+ }
57+ response = responseBody
58+ return nil
59+ }, retry .Attempts (core .ProcessRequestRetryAttempts ), retry .Delay (time .Second * time .Duration (core .ProcessRequestRetryDelay )))
60+ if err != nil {
61+ return nil , err
62+ }
63+ default :
64+ return nil , errors .New ("invalid request type" )
4165 }
4266 //Storing the data into cache
43- localCache .Update (body , url , time .Now ().Add (time .Second * time .Duration (core .StateLength )).Unix ())
44- return body , nil
67+ localCache .Update (response , dataSourceURLStruct . URL , time .Now ().Add (time .Second * time .Duration (core .StateLength )).Unix ())
68+ return response , nil
4569 }
46- log .Debugf ("Getting Data for URL %s from local cache..." , url )
70+ log .Debugf ("Getting Data for URL %s from local cache..." , dataSourceURLStruct . URL )
4771 return cachedData , nil
4872}
4973
50- func ( * UtilsStruct ) GetDataFromJSON (jsonObject map [string ]interface {}, selector string ) (interface {}, error ) {
74+ func GetDataFromJSON (jsonObject map [string ]interface {}, selector string ) (interface {}, error ) {
5175 if selector [0 ] == '[' {
5276 selector = "$" + selector
5377 } else {
@@ -56,15 +80,55 @@ func (*UtilsStruct) GetDataFromJSON(jsonObject map[string]interface{}, selector
5680 return jsonpath .Get (selector , jsonObject )
5781}
5882
59- func ( * UtilsStruct ) GetDataFromXHTML (url string , selector string ) (string , error ) {
83+ func GetDataFromXHTML (dataSourceURLStruct types. DataSourceURL , selector string ) (string , error ) {
6084 c := colly .NewCollector ()
6185 var priceData string
6286 c .OnXML (selector , func (e * colly.XMLElement ) {
6387 priceData = e .Text
6488 })
65- err := c .Visit (url )
89+ err := c .Visit (dataSourceURLStruct . URL )
6690 if err != nil {
6791 return "" , err
6892 }
6993 return priceData , nil
7094}
95+
96+ func AddHeaderToRequest (request * http.Request , headerMap map [string ]string ) (* http.Request , error ) {
97+ for key , value := range headerMap {
98+ re := regexp .MustCompile (core .APIKeyRegex )
99+ isAPIKeyRequired := re .MatchString (value )
100+ if isAPIKeyRequired {
101+ value = ReplaceValueWithDataFromENVFile (re , value )
102+ }
103+ log .Debugf ("Adding key: %s, value: %s pair to header" , key , value )
104+ request .Header .Add (key , value )
105+ }
106+ return request , nil
107+ }
108+
109+ func ProcessRequest (client http.Client , dataSourceURLStruct types.DataSourceURL , requestBody io.Reader ) ([]byte , error ) {
110+ request , err := http .NewRequest (dataSourceURLStruct .Type , dataSourceURLStruct .URL , requestBody )
111+ if err != nil {
112+ return nil , err
113+ }
114+ requestWithHeader , err := AddHeaderToRequest (request , dataSourceURLStruct .Header )
115+ if err != nil {
116+ log .Errorf ("Error in adding header to %s request: %v" , dataSourceURLStruct .Type , err )
117+ return nil , err
118+ }
119+ response , err := client .Do (requestWithHeader )
120+ if err != nil {
121+ log .Errorf ("Error sending %s request URL %s: %v" , dataSourceURLStruct .Type , dataSourceURLStruct .URL , err )
122+ return nil , err
123+ }
124+ defer response .Body .Close ()
125+ if response .StatusCode != 200 {
126+ log .Errorf ("API: %s responded with status code %d" , dataSourceURLStruct .URL , response .StatusCode )
127+ return nil , errors .New ("unable to reach API" )
128+ }
129+ responseBody , err := io .ReadAll (response .Body )
130+ if err != nil {
131+ return nil , err
132+ }
133+ return responseBody , nil
134+ }
0 commit comments