Skip to content

Commit 29e5586

Browse files
franciscovalentecastrofranciscovalentecastro
authored
[exporter/collector/logs] Create Int32OrString and Int64OrString to handle integer fields in gcp.http_request. (#1091)
* Create `Int32_Or_String` and `Int64_Or_String` to handle unmarshalling of integer fields in `gcp.http_request`. * Remove underscores in variable names. * Return `integer` value from `strconv.ParseInt` when an error is found.
1 parent 86523a9 commit 29e5586

2 files changed

Lines changed: 159 additions & 31 deletions

File tree

‎exporter/collector/logs.go‎

Lines changed: 51 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -585,48 +585,68 @@ func (l logMapper) logToSplitEntries(
585585
return entries, nil
586586
}
587587

588-
type IntOrString int32
589-
590-
// Used to unmarhall JSON fields that can be either provided as a JSON number
591-
// or a string containing an integer.
592-
func (f *IntOrString) UnmarshalJSON(data []byte) error {
593-
var number int32
588+
func unmarshalJSONToIntByBitSize(data []byte, bitSize int) (int64, error) {
589+
var number int64
594590
if err := json.Unmarshal(data, &number); err == nil {
595-
*f = IntOrString(number)
596-
return nil
591+
return number, nil
597592
}
598593

599594
var str string
600595
if err := json.Unmarshal(data, &str); err == nil {
601-
integer, err := strconv.ParseInt(strings.TrimSpace(str), 10, 32)
596+
integer, err := strconv.ParseInt(strings.TrimSpace(str), 10, bitSize)
602597
if err != nil {
603-
return fmt.Errorf("failed to convert string number to int32: %w", err)
598+
return integer, fmt.Errorf("failed to convert string number to integer: %w", err)
604599
}
605-
*f = IntOrString(integer)
606-
return nil
600+
return integer, nil
607601
}
608602

609-
return fmt.Errorf("field must be a JSON number or a string containing an integer: %s", string(data))
603+
return 0, fmt.Errorf("field must be a JSON number or a string containing an integer: %s", string(data))
604+
}
605+
606+
type Int32OrString int32
607+
608+
// Used to unmarhall JSON fields that can be either provided as a JSON number
609+
// or a string containing an integer.
610+
func (f *Int32OrString) UnmarshalJSON(data []byte) error {
611+
integer, err := unmarshalJSONToIntByBitSize(data, 32)
612+
if err != nil {
613+
return err
614+
}
615+
*f = Int32OrString(integer)
616+
return nil
617+
}
618+
619+
type Int64OrString int64
620+
621+
// Used to unmarhall JSON fields that can be either provided as a JSON number
622+
// or a string containing an integer.
623+
func (f *Int64OrString) UnmarshalJSON(data []byte) error {
624+
integer, err := unmarshalJSONToIntByBitSize(data, 64)
625+
if err != nil {
626+
return err
627+
}
628+
*f = Int64OrString(integer)
629+
return nil
610630
}
611631

612632
// JSON keys derived from:
613633
// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#httprequest
614634
type httpRequestLog struct {
615-
RemoteIP string `json:"remoteIp"`
616-
RequestURL string `json:"requestUrl"`
617-
Latency string `json:"latency"`
618-
Referer string `json:"referer"`
619-
ServerIP string `json:"serverIp"`
620-
UserAgent string `json:"userAgent"`
621-
RequestMethod string `json:"requestMethod"`
622-
Protocol string `json:"protocol"`
623-
ResponseSize int64 `json:"responseSize,string"`
624-
RequestSize int64 `json:"requestSize,string"`
625-
CacheFillBytes int64 `json:"cacheFillBytes,string"`
626-
Status IntOrString `json:"status"`
627-
CacheLookup bool `json:"cacheLookup"`
628-
CacheHit bool `json:"cacheHit"`
629-
CacheValidatedWithOriginServer bool `json:"cacheValidatedWithOriginServer"`
635+
RemoteIP string `json:"remoteIp"`
636+
RequestURL string `json:"requestUrl"`
637+
Latency string `json:"latency"`
638+
Referer string `json:"referer"`
639+
ServerIP string `json:"serverIp"`
640+
UserAgent string `json:"userAgent"`
641+
RequestMethod string `json:"requestMethod"`
642+
Protocol string `json:"protocol"`
643+
ResponseSize Int64OrString `json:"responseSize"`
644+
RequestSize Int64OrString `json:"requestSize"`
645+
CacheFillBytes Int64OrString `json:"cacheFillBytes"`
646+
Status Int32OrString `json:"status"`
647+
CacheLookup bool `json:"cacheLookup"`
648+
CacheHit bool `json:"cacheHit"`
649+
CacheValidatedWithOriginServer bool `json:"cacheValidatedWithOriginServer"`
630650
}
631651

632652
func (l logMapper) parseHTTPRequest(httpRequestAttr pcommon.Value) (*logtypepb.HttpRequest, error) {
@@ -639,17 +659,17 @@ func (l logMapper) parseHTTPRequest(httpRequestAttr pcommon.Value) (*logtypepb.H
639659
pb := &logtypepb.HttpRequest{
640660
RequestMethod: parsedHTTPRequest.RequestMethod,
641661
RequestUrl: fixUTF8(parsedHTTPRequest.RequestURL),
642-
RequestSize: parsedHTTPRequest.RequestSize,
662+
RequestSize: int64(parsedHTTPRequest.RequestSize),
643663
Status: int32(parsedHTTPRequest.Status),
644-
ResponseSize: parsedHTTPRequest.ResponseSize,
664+
ResponseSize: int64(parsedHTTPRequest.ResponseSize),
645665
UserAgent: parsedHTTPRequest.UserAgent,
646666
ServerIp: parsedHTTPRequest.ServerIP,
647667
RemoteIp: parsedHTTPRequest.RemoteIP,
648668
Referer: parsedHTTPRequest.Referer,
649669
CacheHit: parsedHTTPRequest.CacheHit,
650670
CacheValidatedWithOriginServer: parsedHTTPRequest.CacheValidatedWithOriginServer,
651671
Protocol: "HTTP/1.1",
652-
CacheFillBytes: parsedHTTPRequest.CacheFillBytes,
672+
CacheFillBytes: int64(parsedHTTPRequest.CacheFillBytes),
653673
CacheLookup: parsedHTTPRequest.CacheLookup,
654674
}
655675
if parsedHTTPRequest.Latency != "" {

‎exporter/collector/logs_test.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ func TestLogMapping(t *testing.T) {
6262
testSpanID := pcommon.SpanID([8]byte{
6363
0, 0, 0, 0, 0, 0, 0, 1,
6464
})
65+
testInt64RequestSize := int64(4611686018427387904) // 2 ^ 62 + 1
66+
testInt64ResponseSize := int64(4611686018427387905) // 2 ^ 62 + 2
67+
testInt64CacheFillBytes := int64(4611686018427387906) // 2 ^ 62 + 3
68+
testStringRequestSize := "4611686018427387904" // 2 ^ 62 + 1
69+
testStringResponseSize := "4611686018427387905" // 2 ^ 62 + 2
70+
testStringCacheFillBytes := "4611686018427387906" // 2 ^ 62 + 3
6571
logName := "projects/fakeprojectid/logs/default-log"
6672

6773
testCases := []struct {
@@ -266,6 +272,108 @@ func TestLogMapping(t *testing.T) {
266272
},
267273
maxEntrySize: defaultMaxEntrySize,
268274
},
275+
{
276+
name: "log with json and httpRequest with integer values, empty monitoredresource",
277+
log: func() plog.LogRecord {
278+
log := plog.NewLogRecord()
279+
log.Body().SetEmptyMap().PutStr("message", "hello!")
280+
httpRequest := log.Attributes().PutEmptyMap(HTTPRequestAttributeKey)
281+
httpRequest.PutStr("requestMethod", "GET")
282+
httpRequest.PutStr("requestURL", "https://www.example.com")
283+
httpRequest.PutInt("requestSize", testInt64RequestSize)
284+
httpRequest.PutInt("status", 200)
285+
httpRequest.PutInt("responseSize", testInt64ResponseSize)
286+
httpRequest.PutStr("userAgent", "test")
287+
httpRequest.PutStr("remoteIP", "192.168.0.1")
288+
httpRequest.PutStr("serverIP", "192.168.0.2")
289+
httpRequest.PutStr("referer", "https://www.example2.com")
290+
httpRequest.PutBool("cacheHit", false)
291+
httpRequest.PutBool("cacheValidatedWithOriginServer", false)
292+
httpRequest.PutInt("cacheFillBytes", testInt64CacheFillBytes)
293+
httpRequest.PutStr("protocol", "HTTP/2")
294+
return log
295+
},
296+
mr: func() *monitoredrespb.MonitoredResource {
297+
return nil
298+
},
299+
expectedEntries: []*logpb.LogEntry{
300+
{
301+
LogName: logName,
302+
Timestamp: timestamppb.New(testObservedTime),
303+
Payload: &logpb.LogEntry_JsonPayload{JsonPayload: &structpb.Struct{Fields: map[string]*structpb.Value{
304+
"message": {Kind: &structpb.Value_StringValue{StringValue: "hello!"}},
305+
}}},
306+
HttpRequest: &logtypepb.HttpRequest{
307+
RequestMethod: "GET",
308+
UserAgent: "test",
309+
Referer: "https://www.example2.com",
310+
RequestUrl: "https://www.example.com",
311+
Protocol: "HTTP/1.1",
312+
RequestSize: testInt64RequestSize,
313+
Status: 200,
314+
ResponseSize: testInt64ResponseSize,
315+
ServerIp: "192.168.0.2",
316+
RemoteIp: "192.168.0.1",
317+
CacheHit: false,
318+
CacheValidatedWithOriginServer: false,
319+
CacheFillBytes: testInt64CacheFillBytes,
320+
CacheLookup: false,
321+
},
322+
},
323+
},
324+
maxEntrySize: defaultMaxEntrySize,
325+
},
326+
{
327+
name: "log with json and httpRequest with string values, empty monitoredresource",
328+
log: func() plog.LogRecord {
329+
log := plog.NewLogRecord()
330+
log.Body().SetEmptyMap().PutStr("message", "hello!")
331+
httpRequest := log.Attributes().PutEmptyMap(HTTPRequestAttributeKey)
332+
httpRequest.PutStr("requestMethod", "GET")
333+
httpRequest.PutStr("requestURL", "https://www.example.com")
334+
httpRequest.PutStr("requestSize", testStringRequestSize)
335+
httpRequest.PutInt("status", 200)
336+
httpRequest.PutStr("responseSize", testStringResponseSize)
337+
httpRequest.PutStr("userAgent", "test")
338+
httpRequest.PutStr("remoteIP", "192.168.0.1")
339+
httpRequest.PutStr("serverIP", "192.168.0.2")
340+
httpRequest.PutStr("referer", "https://www.example2.com")
341+
httpRequest.PutBool("cacheHit", false)
342+
httpRequest.PutBool("cacheValidatedWithOriginServer", false)
343+
httpRequest.PutStr("cacheFillBytes", testStringCacheFillBytes)
344+
httpRequest.PutStr("protocol", "HTTP/2")
345+
return log
346+
},
347+
mr: func() *monitoredrespb.MonitoredResource {
348+
return nil
349+
},
350+
expectedEntries: []*logpb.LogEntry{
351+
{
352+
LogName: logName,
353+
Timestamp: timestamppb.New(testObservedTime),
354+
Payload: &logpb.LogEntry_JsonPayload{JsonPayload: &structpb.Struct{Fields: map[string]*structpb.Value{
355+
"message": {Kind: &structpb.Value_StringValue{StringValue: "hello!"}},
356+
}}},
357+
HttpRequest: &logtypepb.HttpRequest{
358+
RequestMethod: "GET",
359+
UserAgent: "test",
360+
Referer: "https://www.example2.com",
361+
RequestUrl: "https://www.example.com",
362+
Protocol: "HTTP/1.1",
363+
RequestSize: testInt64RequestSize,
364+
Status: 200,
365+
ResponseSize: testInt64ResponseSize,
366+
ServerIp: "192.168.0.2",
367+
RemoteIp: "192.168.0.1",
368+
CacheHit: false,
369+
CacheValidatedWithOriginServer: false,
370+
CacheFillBytes: testInt64CacheFillBytes,
371+
CacheLookup: false,
372+
},
373+
},
374+
},
375+
maxEntrySize: defaultMaxEntrySize,
376+
},
269377
{
270378
name: "log with httpRequest attribute unsupported type",
271379
log: func() plog.LogRecord {

0 commit comments

Comments
 (0)