[pkg/translator/azurelogs] Support additional fields not in the common schema

.chloggen/46162.yaml

@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
+component: pkg/azurelogs
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: 'Add support for additional time and property fields in Azure Resource Logs'
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [46162]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Azure resource logs are not standard. This change expands support for different resource types by adding support
+  for additional time and property fields that are less commonly used in Azure resource logs.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

pkg/translator/azurelogs/resourcelogs_to_logs.go

@@ -103,9 +103,16 @@ type identity struct {
 // azureLogRecord represents a single Azure log following
 // the common schema:
 // https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/resource-logs-schema
+// Some services don't follow the common schema and may have different fields, but these are the most common ones across all services and are used for grouping and semantic conventions mapping.
+// Example:
+//   - Service Bus: https://learn.microsoft.com/en-us/azure/service-bus-messaging/monitor-service-bus-reference#operational-logs
+//   - Event Hub: https://learn.microsoft.com/en-us/azure/event-hubs/monitor-event-hubs-reference#archive-logs-schema
 type azureLogRecord struct {
 	Time              string          `json:"time"`
 	Timestamp         string          `json:"timeStamp"`
+	EventTimeString   string          `json:"EventTimeString"`
+	EventTimestamp    string          `json:"EventTimestamp"`
+	StartTime         string          `json:"startTime"`
 	ResourceID        string          `json:"resourceId"`
 	TenantID          *string         `json:"tenantId"`
 	OperationName     string          `json:"operationName"`
@@ -121,6 +128,16 @@ type azureLogRecord struct {
 	Level             *json.Number    `json:"Level"`
 	Location          *string         `json:"location"`
 	Properties        json.RawMessage `json:"properties"`
+	EventProperties   json.RawMessage `json:"EventProperties"`
+}
+
+// getProperties returns whichever properties field is populated,
+// preferring "properties" over "EventProperties".
+func (r *azureLogRecord) getProperties() json.RawMessage {
+	if len(r.Properties) > 0 {
+		return r.Properties
+	}
+	return r.EventProperties
 }
 
 var _ plog.Unmarshaler = (*ResourceLogsUnmarshaler)(nil)
@@ -182,7 +199,14 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) {
 
 		nanos, err := getTimestamp(log, r.TimeFormats...)
 		if err != nil {
-			r.Logger.Warn("Unable to convert timestamp from log", zap.String("timestamp", log.Time))
+			fields := []zap.Field{zap.String("timestamp", log.Time)}
+			if log.ResourceID != "" {
+				fields = append(fields, zap.String("resource_id", log.ResourceID))
+			}
+			if log.Category != "" {
+				fields = append(fields, zap.String("category", log.Category))
+			}
+			r.Logger.Warn("Unable to convert timestamp from log", fields...)
 			continue
 		}
 
@@ -199,7 +223,7 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) {
 			lr.SetSeverityText(log.Level.String())
 		}
 
-		err = addRecordAttributes(log.Category, log.Properties, lr)
+		err = addRecordAttributes(log.Category, log.getProperties(), lr)
 		if err != nil {
 			if errors.Is(err, errStillToImplement) || errors.Is(err, errUnsupportedCategory) {
 				// TODO @constanca-m This will be removed once the categories
@@ -247,13 +271,20 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) {
 }
 
 func getTimestamp(record *azureLogRecord, formats ...string) (pcommon.Timestamp, error) {
-	if record.Time != "" {
+	switch {
+	case record.Time != "":
 		return asTimestamp(record.Time, formats...)
-	} else if record.Timestamp != "" {
+	case record.Timestamp != "":
 		return asTimestamp(record.Timestamp, formats...)
+	case record.EventTimeString != "":
+		return asTimestamp(record.EventTimeString, formats...)
+	case record.EventTimestamp != "":
+		return asTimestamp(record.EventTimestamp, formats...)
+	case record.StartTime != "":
+		return asTimestamp(record.StartTime, formats...)
+	default:
+		return 0, errMissingTimestamp
 	}
-
-	return 0, errMissingTimestamp
 }
 
 // asTimestamp will parse an ISO8601 string into an OpenTelemetry
@@ -357,12 +388,13 @@ func extractRawAttributes(log *azureLogRecord, rawRecord json.RawMessage) map[st
 	attrs[azureOperationName] = log.OperationName
 	setIf(attrs, azureOperationVersion, log.OperationVersion)
 
-	if log.Properties != nil {
-		copyPropertiesAndApplySemanticConventions(log.Category, log.Properties, attrs)
+	properties := log.getProperties()
+	if properties != nil {
+		copyPropertiesAndApplySemanticConventions(log.Category, properties, attrs)
 	}
 
 	// The original log needs to be preserved for logs that don't have a properties field
-	if len(log.Properties) == 0 && len(rawRecord) > 0 {
+	if len(properties) == 0 && len(rawRecord) > 0 {
 		// Format the JSON with proper indentation to match expected output
 		var formattedJSON bytes.Buffer
 		if err := json.Indent(&formattedJSON, rawRecord, "", "\t"); err == nil {

pkg/translator/azurelogs/resourcelogs_to_logs_test.go

@@ -942,3 +942,239 @@ func TestUnmarshalLogs_StableGates(t *testing.T) {
 		})
 	}
 }
+
+func TestGetProperties(t *testing.T) {
+	tests := []struct {
+		name            string
+		properties      json.RawMessage
+		eventProperties json.RawMessage
+		expected        json.RawMessage
+	}{
+		{
+			name:            "properties set, eventProperties empty",
+			properties:      json.RawMessage(`{"key":"value"}`),
+			eventProperties: nil,
+			expected:        json.RawMessage(`{"key":"value"}`),
+		},
+		{
+			name:            "properties empty, eventProperties set",
+			properties:      nil,
+			eventProperties: json.RawMessage(`{"event":"data"}`),
+			expected:        json.RawMessage(`{"event":"data"}`),
+		},
+		{
+			name:            "both set, properties takes precedence",
+			properties:      json.RawMessage(`{"key":"value"}`),
+			eventProperties: json.RawMessage(`{"event":"data"}`),
+			expected:        json.RawMessage(`{"key":"value"}`),
+		},
+		{
+			name:            "both empty",
+			properties:      nil,
+			eventProperties: nil,
+			expected:        nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			record := &azureLogRecord{
+				Properties:      tt.properties,
+				EventProperties: tt.eventProperties,
+			}
+			result := record.getProperties()
+			assert.Equal(t, string(tt.expected), string(result))
+		})
+	}
+}
+
+func TestGetTimestamp(t *testing.T) {
+	tests := []struct {
+		name        string
+		record      *azureLogRecord
+		formats     []string
+		expectError bool
+	}{
+		{
+			name: "time field set",
+			record: &azureLogRecord{
+				Time: "2022-11-11T04:48:27.6767145Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "timestamp field set",
+			record: &azureLogRecord{
+				Timestamp: "2022-11-11T04:48:27.6767145Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "EventTimeString field set",
+			record: &azureLogRecord{
+				EventTimeString: "2022-11-11T04:48:27.6767145Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "time takes precedence over timestamp",
+			record: &azureLogRecord{
+				Time:      "2022-11-11T04:48:27.6767145Z",
+				Timestamp: "2023-01-01T00:00:00Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "timestamp takes precedence over EventTimeString",
+			record: &azureLogRecord{
+				Timestamp:       "2022-11-11T04:48:27.6767145Z",
+				EventTimeString: "2023-01-01T00:00:00Z",
+			},
+			expectError: false,
+		},
+		{
+			name:        "all empty returns error",
+			record:      &azureLogRecord{},
+			expectError: true,
+		},
+		{
+			name: "EventTimeString with custom format",
+			record: &azureLogRecord{
+				EventTimeString: "11/20/2024 13:57:18",
+			},
+			formats:     []string{"01/02/2006 15:04:05"},
+			expectError: false,
+		},
+		{
+			name: "startTime field set",
+			record: &azureLogRecord{
+				StartTime: "2022-11-11T04:48:27.6767145Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "EventTimeString takes precedence over startTime",
+			record: &azureLogRecord{
+				EventTimeString: "2022-11-11T04:48:27.6767145Z",
+				StartTime:       "2023-01-01T00:00:00Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "EventTimestamp field set",
+			record: &azureLogRecord{
+				EventTimestamp: "2022-11-11T04:48:27.6767145Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "EventTimestamp with custom format",
+			record: &azureLogRecord{
+				EventTimestamp: "11/20/2024 13:57:18",
+			},
+			formats:     []string{"01/02/2006 15:04:05"},
+			expectError: false,
+		},
+		{
+			name: "EventTimeString takes precedence over EventTimestamp",
+			record: &azureLogRecord{
+				EventTimeString: "2022-11-11T04:48:27.6767145Z",
+				EventTimestamp:  "2023-01-01T00:00:00Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "EventTimestamp takes precedence over startTime",
+			record: &azureLogRecord{
+				EventTimestamp: "2022-11-11T04:48:27.6767145Z",
+				StartTime:      "2023-01-01T00:00:00Z",
+			},
+			expectError: false,
+		},
+		{
+			name: "startTime with custom format",
+			record: &azureLogRecord{
+				StartTime: "11/20/2024 13:57:18",
+			},
+			formats:     []string{"01/02/2006 15:04:05"},
+			expectError: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			nanos, err := getTimestamp(tt.record, tt.formats...)
+			if tt.expectError {
+				assert.Error(t, err)
+				assert.Equal(t, pcommon.Timestamp(0), nanos)
+			} else {
+				assert.NoError(t, err)
+				assert.Less(t, pcommon.Timestamp(0), nanos)
+			}
+		})
+	}
+}
+
+func TestExtractRawAttributes_EventProperties(t *testing.T) {
+	tests := []struct {
+		name     string
+		log      *azureLogRecord
+		expected map[string]any
+	}{
+		{
+			name: "uses EventProperties when Properties is nil",
+			log: &azureLogRecord{
+				OperationName:   "operation.name",
+				Category:        "category",
+				EventProperties: json.RawMessage(`{"a":1,"b":"two"}`),
+			},
+			expected: map[string]any{
+				azureOperationName: "operation.name",
+				azureCategory:      "category",
+				azureProperties: map[string]any{
+					"a": float64(1),
+					"b": "two",
+				},
+			},
+		},
+		{
+			name: "Properties takes precedence over EventProperties",
+			log: &azureLogRecord{
+				OperationName:   "operation.name",
+				Category:        "category",
+				Properties:      json.RawMessage(`{"from":"properties"}`),
+				EventProperties: json.RawMessage(`{"from":"eventproperties"}`),
+			},
+			expected: map[string]any{
+				azureOperationName: "operation.name",
+				azureCategory:      "category",
+				azureProperties: map[string]any{
+					"from": "properties",
+				},
+			},
+		},
+		{
+			name: "rawRecord preserved when no properties",
+			log: &azureLogRecord{
+				OperationName: "operation.name",
+				Category:      "category",
+			},
+			expected: map[string]any{
+				azureOperationName:     "operation.name",
+				azureCategory:          "category",
+				attributeEventOriginal: "{\n\t\"test\": true\n}",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var rawRecord json.RawMessage
+			if tt.name == "rawRecord preserved when no properties" {
+				rawRecord = json.RawMessage(`{"test": true}`)
+			}
+			result := extractRawAttributes(tt.log, rawRecord)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}