[processor/unroll]: upstream unroll processor

.chloggen/feat_unroll-processor.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: unrollprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Adds a processor that generically takes a log body of slices and creates new entries from that slice.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [42491]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.codecov.yml

@@ -544,6 +544,10 @@ component_management:
     name: processor_transform
     paths:
     - processor/transformprocessor/**
+  - component_id: processor_unroll
+    name: processor_unroll
+    paths:
+    - processor/unrollprocessor/**
   - component_id: receiver_activedirectoryds
     name: receiver_activedirectoryds
     paths:

.github/CODEOWNERS

@@ -216,6 +216,7 @@ processor/spanprocessor/                                         @open-telemetry
 processor/sumologicprocessor/                                    @open-telemetry/collector-contrib-approvers @rnishtala-sumo @chan-tim-sumo @echlebek @amdprophet
 processor/tailsamplingprocessor/                                 @open-telemetry/collector-contrib-approvers @portertech
 processor/transformprocessor/                                    @open-telemetry/collector-contrib-approvers @TylerHelmuth @evan-bradley @edmocosta
+processor/unrollprocessor/                                       @open-telemetry/collector-contrib-approvers @axw @schmikei @rnishtala-sumo
 receiver/activedirectorydsreceiver/                              @open-telemetry/collector-contrib-approvers @pjanotti
 receiver/aerospikereceiver/                                      @open-telemetry/collector-contrib-approvers @antonblock
 receiver/apachereceiver/                                         @open-telemetry/collector-contrib-approvers @colelaven @ishleenk17

.github/ISSUE_TEMPLATE/beta_stability.yaml

@@ -217,6 +217,7 @@ body:
       - processor/sumologic
       - processor/tailsampling
       - processor/transform
+      - processor/unroll
       - receiver/activedirectoryds
       - receiver/aerospike
       - receiver/apache

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -220,6 +220,7 @@ body:
       - processor/sumologic
       - processor/tailsampling
       - processor/transform
+      - processor/unroll
       - receiver/activedirectoryds
       - receiver/aerospike
       - receiver/apache

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -214,6 +214,7 @@ body:
       - processor/sumologic
       - processor/tailsampling
       - processor/transform
+      - processor/unroll
       - receiver/activedirectoryds
       - receiver/aerospike
       - receiver/apache

.github/ISSUE_TEMPLATE/other.yaml

@@ -214,6 +214,7 @@ body:
       - processor/sumologic
       - processor/tailsampling
       - processor/transform
+      - processor/unroll
       - receiver/activedirectoryds
       - receiver/aerospike
       - receiver/apache

.github/ISSUE_TEMPLATE/unmaintained.yaml

@@ -219,6 +219,7 @@ body:
       - processor/sumologic
       - processor/tailsampling
       - processor/transform
+      - processor/unroll
       - receiver/activedirectoryds
       - receiver/aerospike
       - receiver/apache

.github/component_labels.txt

@@ -197,6 +197,7 @@ processor/spanprocessor processor/span
 processor/sumologicprocessor processor/sumologic
 processor/tailsamplingprocessor processor/tailsampling
 processor/transformprocessor processor/transform
+processor/unrollprocessor processor/unroll
 receiver/activedirectorydsreceiver receiver/activedirectoryds
 receiver/aerospikereceiver receiver/aerospike
 receiver/apachereceiver receiver/apache

internal/tidylist/tidylist.txt

@@ -225,6 +225,7 @@ processor/resourceprocessor
 processor/schemaprocessor
 processor/spanprocessor
 processor/sumologicprocessor
+processor/unrollprocessor
 receiver/activedirectorydsreceiver
 receiver/aerospikereceiver
 receiver/apachereceiver

processor/unrollprocessor/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

processor/unrollprocessor/README.md

@@ -0,0 +1,217 @@
+# Unroll Processor
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [alpha]: logs   |
+| Distributions | [contrib] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aprocessor%2Funroll%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aprocessor%2Funroll) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aprocessor%2Funroll%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aprocessor%2Funroll) |
+| Code coverage | [![codecov](https://codecov.io/github/open-telemetry/opentelemetry-collector-contrib/graph/main/badge.svg?component=processor_unroll)](https://app.codecov.io/gh/open-telemetry/opentelemetry-collector-contrib/tree/main/?components%5B0%5D=processor_unroll&displayType=list) |
+| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@axw](https://www.github.com/axw), [@schmikei](https://www.github.com/schmikei), [@rnishtala-sumo](https://www.github.com/rnishtala-sumo) |
+
+[alpha]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#alpha
+[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
+<!-- end autogenerated section -->
+
+The Unroll Processor takes log records with slice bodies and expands each element of the slice into its own log record. This allows for better processing and analysis of structured log data that contains arrays or lists.
+
+## Supported pipelines
+
+- Logs
+
+
+## How it works
+
+The Unroll Processor processes log records through the following steps:
+
+1. The processor examines each incoming log record to determine if the body contains a slice (array) structure
+2. For log records with slice bodies, each element of the slice is extracted and used to create a new individual log record
+3. Each new log record retains all the original metadata (timestamps, attributes, etc.) from the parent record
+4. When `recursive` is enabled, the processor will also unroll nested slices within slice elements
+
+## Config
+
+### General Config
+
+```yaml
+unroll:
+  recursive: false   # Whether to recursively unroll nested slices
+```
+
+| Field     | Type   | Default | Description                                                                                                |
+| --------- | ------ | ------- | ---------------------------------------------------------------------------------------------------------- |
+| recursive | bool   | false   | Whether to recursively unroll nested slices within slice elements                                         |
+
+### Example configuration
+
+```yaml
+unroll:
+  recursive: false
+```
+
+
+
+## Examples
+
+### Basic Usage
+
+The simplest configuration for the unroll processor:
+
+```yaml
+processors:
+  unroll:
+    recursive: false
+
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]
+      processors: [unroll]
+      exporters: [logging]
+```
+
+### Split a log record into multiple via a delimiter
+
+The following configuration utilizes the [transform processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/transformprocessor) to first split the original string body using a delimiter, and then the unroll processor creates multiple log records from the resulting slice.
+
+```yaml
+receivers:
+  filelog:
+    include: [ ./test.txt ]
+    start_at: beginning
+
+processors:
+  transform:
+    log_statements:
+      - context: log
+        statements:
+          - set(body, Split(body, ","))
+  unroll:
+    recursive: false
+
+exporters:
+  file:
+    path: ./test/output.json
+
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      processors: [transform, unroll]
+      exporters: [file]
+```
+
+#### Input and Output Example
+
+<details>
+<summary>Sample Input Data</summary>
+
+**Input file (test.txt):**
+```txt
+1,2,3
+```
+
+**After transform processor (before unroll):**
+The body becomes a slice: `["1", "2", "3"]`
+
+</details>
+
+<details>
+<summary>Final Output (after unroll)</summary>
+
+```json
+{
+  "resourceLogs": [
+    {
+      "resource": {},
+      "scopeLogs": [
+        {
+          "scope": {},
+          "logRecords": [
+            {
+              "observedTimeUnixNano": "1733240156591852000",
+              "body": { "stringValue": "1" },
+              "attributes": [
+                {
+                  "key": "log.file.name",
+                  "value": { "stringValue": "test.txt" }
+                }
+              ],
+              "traceId": "",
+              "spanId": ""
+            },
+            {
+              "observedTimeUnixNano": "1733240156591852000",
+              "body": { "stringValue": "2" },
+              "attributes": [
+                {
+                  "key": "log.file.name",
+                  "value": { "stringValue": "test.txt" }
+                }
+              ],
+              "traceId": "",
+              "spanId": ""
+            },
+            {
+              "observedTimeUnixNano": "1733240156591852000",
+              "body": { "stringValue": "3" },
+              "attributes": [
+                {
+                  "key": "log.file.name",
+                  "value": { "stringValue": "test.txt" }
+                }
+              ],
+              "traceId": "",
+              "spanId": ""
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
+```
+</details>
+
+### Recursive Unrolling
+
+When dealing with nested slices, you can enable recursive unrolling:
+
+```yaml
+processors:
+  unroll:
+    recursive: true
+
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]
+      processors: [unroll]
+      exporters: [logging]
+```
+
+This configuration will unroll nested slices within slice elements, creating individual log records for all nested elements.
+
+### Common Issues
+
+#### Log records not being unrolled
+- **Cause**: The log body is not a slice/array type
+- **Solution**: Ensure the log body contains a slice. You may need to use the [transform processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/transformprocessor) to convert string data to slices first
+
+#### Unexpected number of output records
+- **Cause**: Nested slices with `recursive: false` setting
+- **Solution**: Enable `recursive: true` if you want to unroll nested slices, or restructure your data to avoid nested arrays
+
+#### Performance issues with large slices
+- **Cause**: Very large slices being unrolled into many individual log records
+- **Solution**: Consider preprocessing the data to limit slice sizes or batch processing
+
+## Warnings
+
+The Unroll Processor modifies the structure and quantity of log records in your telemetry pipeline. Consider the following warnings:
+
+- **Data Volume**: Unrolling slices can significantly increase the number of log records, which may impact downstream processing performance and storage requirements.
+- **Resource Usage**: Large slices will consume more memory and CPU resources during the unrolling process.
+- **Downstream Compatibility**: Ensure that downstream processors and exporters can handle the increased volume of log records.
+- **Metadata Duplication**: Each unrolled log record retains the same metadata (timestamps, attributes, etc.) from the original record, which may result in data duplication.
+
+Use this processor carefully in production environments and monitor resource usage and performance impact.

processor/unrollprocessor/config.go

@@ -0,0 +1,17 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package unrollprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/unrollprocessor"
+
+// Config is the configuration for the unroll processor.
+type Config struct {
+	Recursive bool `mapstructure:"recursive"`
+
+	// prevent unkeyed literal initialization
+	_ struct{}
+}
+
+// Validate is a no-op for this as there's no configuration that is possibly invalid after unmarshalling
+func (*Config) Validate() error {
+	return nil
+}

processor/unrollprocessor/config_test.go

@@ -0,0 +1,34 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package unrollprocessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidate(t *testing.T) {
+	testCases := []struct {
+		desc        string
+		cfg         *Config
+		expectedErr string
+	}{
+		{
+			desc: "valid config",
+			cfg:  createDefaultConfig().(*Config),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			err := tc.cfg.Validate()
+			if tc.expectedErr != "" {
+				require.ErrorContains(t, err, tc.expectedErr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

processor/unrollprocessor/doc.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+// Package unrollprocessor contains the logic to unroll log based telemetry.
+package unrollprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/unrollprocessor"