[cmd/mdatagen] Add basic support for entities to metadata.yaml schema

.chloggen/mdatagen-entities-support.yaml

@@ -0,0 +1,28 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: cmd/mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "`metadata.yaml` now supports an optional `entities` section to organize resource attributes into logical entities with identity and description attributes"
+
+# One or more tracking issues or pull requests related to the change
+issues: [14051]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  When entities are defined, mdatagen generates `AssociateWith{EntityType}()` methods on ResourceBuilder
+  that associate resources with entity types using the entity refs API. The entities section is backward
+  compatible - existing metadata.yaml files without entities continue to work as before.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

cmd/mdatagen/go.mod

@@ -16,6 +16,7 @@ require (
 	go.opentelemetry.io/collector/consumer/consumertest v0.138.0
 	go.opentelemetry.io/collector/filter v0.138.0
 	go.opentelemetry.io/collector/pdata v1.44.0
+	go.opentelemetry.io/collector/pdata/xpdata v0.138.0
 	go.opentelemetry.io/collector/pipeline v1.44.0
 	go.opentelemetry.io/collector/processor v1.44.0
 	go.opentelemetry.io/collector/processor/processortest v0.138.0

cmd/mdatagen/internal/command.go

@@ -462,6 +462,7 @@ func validateYAMLKeyOrder(raw []byte) error {
 	}
 	for _, p := range [][]string{
 		{"resource_attributes"},
+		{"entities"},
 		{"attributes"},
 		{"metrics"},
 		{"events"},

cmd/mdatagen/internal/metadata.go

@@ -30,6 +30,8 @@ type Metadata struct {
 	SemConvVersion string `mapstructure:"sem_conv_version"`
 	// ResourceAttributes that can be emitted by the component.
 	ResourceAttributes map[AttributeName]Attribute `mapstructure:"resource_attributes"`
+	// Entities organizes resource attributes into logical entities.
+	Entities map[string]Entity `mapstructure:"entities"`
 	// Attributes emitted by one or more metrics.
 	Attributes map[AttributeName]Attribute `mapstructure:"attributes"`
 	// Metrics that can be emitted by the component.
@@ -56,6 +58,10 @@ func (md Metadata) GetCodeCovComponentID() string {
 	return strings.ReplaceAll(md.Status.Class+"_"+md.Type, "/", "_")
 }
 
+func (md Metadata) HasEntities() bool {
+	return len(md.Entities) > 0
+}
+
 func (md *Metadata) Validate() error {
 	var errs error
 	if err := md.validateType(); err != nil {
@@ -75,6 +81,10 @@ func (md *Metadata) Validate() error {
 		errs = errors.Join(errs, err)
 	}
 
+	if err := md.validateEntities(); err != nil {
+		errs = errors.Join(errs, err)
+	}
+
 	if err := md.validateMetricsAndEvents(); err != nil {
 		errs = errors.Join(errs, err)
 	}
@@ -122,6 +132,41 @@ func (md *Metadata) validateResourceAttributes() error {
 	return errs
 }
 
+func (md *Metadata) validateEntities() error {
+	var errs error
+	usedAttrs := make(map[AttributeName]string)
+
+	for entityType, entity := range md.Entities {
+		if entityType == "" {
+			errs = errors.Join(errs, errors.New("entity type cannot be empty"))
+		}
+		if len(entity.IDAttributes) == 0 {
+			errs = errors.Join(errs, fmt.Errorf(`entity "%v": id_attributes is required`, entityType))
+		}
+		for _, attrName := range entity.IDAttributes {
+			if _, ok := md.ResourceAttributes[attrName]; !ok {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": id_attributes refers to undefined resource attribute: %v`, entityType, attrName))
+			}
+			if otherEntity, used := usedAttrs[attrName]; used {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": attribute %v is already used by entity "%v"`, entityType, attrName, otherEntity))
+			} else {
+				usedAttrs[attrName] = entityType
+			}
+		}
+		for _, attrName := range entity.DescriptionAttributes {
+			if _, ok := md.ResourceAttributes[attrName]; !ok {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": description_attributes refers to undefined resource attribute: %v`, entityType, attrName))
+			}
+			if otherEntity, used := usedAttrs[attrName]; used {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": attribute %v is already used by entity "%v"`, entityType, attrName, otherEntity))
+			} else {
+				usedAttrs[attrName] = entityType
+			}
+		}
+	}
+	return errs
+}
+
 func (md *Metadata) validateMetricsAndEvents() error {
 	var errs error
 	usedAttrs := map[AttributeName]bool{}
@@ -388,3 +433,10 @@ func (s Signal) HasOptionalAttribute(attrs map[AttributeName]Attribute) bool {
 	}
 	return false
 }
+
+type Entity struct {
+	// IDAttributes are the resource attributes that uniquely identify the entity.
+	IDAttributes []AttributeName `mapstructure:"id_attributes"`
+	// DescriptionAttributes are the resource attributes that describe the entity.
+	DescriptionAttributes []AttributeName `mapstructure:"description_attributes"`
+}

cmd/mdatagen/internal/metadata_test.go

@@ -120,12 +120,28 @@ func TestValidate(t *testing.T) {
 			name:    "testdata/no_type_attr.yaml",
 			wantErr: "empty type for attribute: used_attr",
 		},
+		{
+			name:    "testdata/entity_undefined_id_attribute.yaml",
+			wantErr: `entity "host": id_attributes refers to undefined resource attribute: host.missing`,
+		},
+		{
+			name:    "testdata/entity_undefined_description_attribute.yaml",
+			wantErr: `entity "host": description_attributes refers to undefined resource attribute: host.missing`,
+		},
+		{
+			name:    "testdata/entity_empty_id_attributes.yaml",
+			wantErr: `entity "host": id_attributes is required`,
+		},
+		{
+			name:    "testdata/entity_duplicate_attributes.yaml",
+			wantErr: `attribute host.name is already used by entity`,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			_, err := LoadMetadata(tt.name)
 			require.Error(t, err)
-			require.EqualError(t, err, tt.wantErr)
+			require.ErrorContains(t, err, tt.wantErr)
 		})
 	}
 }

cmd/mdatagen/internal/sampleconnector/documentation.md

@@ -113,3 +113,15 @@ metrics:
 | string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true |
 | string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false |
 | string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true |
+
+## Entities
+
+The following entities are defined for this component:
+
+### test.entity
+
+**ID Attributes:**
+- `string.resource.attr`
+
+**Description Attributes:**
+- `map.resource.attr`

cmd/mdatagen/internal/sampleconnector/metadata.yaml

@@ -65,6 +65,13 @@ resource_attributes:
     warnings:
       if_enabled: This resource_attribute is deprecated and will be removed soon.
 
+entities:
+  test.entity:
+    id_attributes:
+      - string.resource.attr
+    description_attributes:
+      - map.resource.attr
+
 attributes:
   boolean_attr:
     description: Attribute with a boolean value.

cmd/mdatagen/internal/templates/documentation.md.tmpl

@@ -212,6 +212,36 @@ events:
 
 {{- end }}
 
+{{- if .Entities }}
+
+## Entities
+
+The following entities are defined for this component:
+
+{{- range $entityType, $entity := .Entities }}
+
+### {{ $entityType }}
+
+{{- if $entity.IDAttributes }}
+
+**ID Attributes:**
+{{- range $entity.IDAttributes }}
+- `{{ . }}`
+{{- end }}
+{{- end }}
+
+{{- if $entity.DescriptionAttributes }}
+
+**Description Attributes:**
+{{- range $entity.DescriptionAttributes }}
+- `{{ . }}`
+{{- end }}
+{{- end }}
+
+{{- end }}
+
+{{- end }}
+
 {{- if .Telemetry.Metrics }}
 
 ## Internal Telemetry

cmd/mdatagen/internal/templates/resource.go.tmpl

@@ -4,6 +4,9 @@ package {{ .Package }}
 
 import (
 	"go.opentelemetry.io/collector/pdata/pcommon"
+{{- if .HasEntities }}
+	"go.opentelemetry.io/collector/pdata/xpdata/entity"
+{{- end }}
 )
 
 // ResourceBuilder is a helper struct to build resources predefined in metadata.yaml.
@@ -43,6 +46,29 @@ func (rb *ResourceBuilder) Set{{ $name.Render }}(val {{ $attr.Type.Primitive }})
 {{- end }}
 {{ end }}
 
+{{- if .HasEntities }}
+
+{{ range $entityType, $entity := .Entities }}
+// AssociateWith{{ $entityType | publicVar }} associates the resource with entity type "{{ $entityType }}".
+func (rb *ResourceBuilder) AssociateWith{{ $entityType | publicVar }}() {
+	entityRef := entity.ResourceEntityRefs(rb.res).AppendEmpty()
+	entityRef.SetType("{{ $entityType }}")
+	{{- if $entity.IDAttributes }}
+	idKeys := entityRef.IdKeys()
+	{{- range $entity.IDAttributes }}
+	idKeys.Append("{{ . }}")
+	{{- end }}
+	{{- end }}
+	{{- if $entity.DescriptionAttributes }}
+	descKeys := entityRef.DescriptionKeys()
+	{{- range $entity.DescriptionAttributes }}
+	descKeys.Append("{{ . }}")
+	{{- end }}
+	{{- end }}
+}
+{{ end }}
+{{- end }}
+
 // Emit returns the built resource and resets the internal builder state.
 func (rb *ResourceBuilder) Emit() pcommon.Resource {
 	r := rb.res

cmd/mdatagen/internal/testdata/documentation.md

@@ -0,0 +1,11 @@
+[comment]: <> (Code generated by mdatagen. DO NOT EDIT.)
+
+# sample
+
+## Resource Attributes
+
+| Name | Description | Values | Enabled |
+| ---- | ----------- | ------ | ------- |
+| host.id | The unique host identifier | Any Str | true |
+| host.name | The hostname | Any Str | true |
+| process.pid | The process identifier | Any Int | true |

cmd/mdatagen/internal/testdata/entity_duplicate_attributes.yaml

@@ -0,0 +1,31 @@
+type: sample
+status:
+  class: receiver
+  stability:
+    stable: [metrics]
+
+resource_attributes:
+  host.id:
+    description: The host identifier
+    type: string
+    enabled: true
+  host.name:
+    description: The hostname
+    type: string
+    enabled: true
+  process.pid:
+    description: The process identifier
+    type: int
+    enabled: true
+
+entities:
+  host:
+    id_attributes:
+      - host.id
+    description_attributes:
+      - host.name
+  process:
+    id_attributes:
+      - process.pid
+    description_attributes:
+      - host.name

cmd/mdatagen/internal/testdata/entity_empty_id_attributes.yaml

@@ -0,0 +1,17 @@
+type: sample
+status:
+  class: receiver
+  stability:
+    stable: [metrics]
+
+resource_attributes:
+  host.name:
+    description: The hostname
+    type: string
+    enabled: true
+
+entities:
+  host:
+    id_attributes: []
+    description_attributes:
+      - host.name

cmd/mdatagen/internal/testdata/entity_undefined_description_attribute.yaml

@@ -0,0 +1,18 @@
+type: sample
+status:
+  class: receiver
+  stability:
+    stable: [metrics]
+
+resource_attributes:
+  host.id:
+    description: The host identifier
+    type: string
+    enabled: true
+
+entities:
+  host:
+    id_attributes:
+      - host.id
+    description_attributes:
+      - host.missing

cmd/mdatagen/internal/testdata/entity_undefined_id_attribute.yaml

@@ -0,0 +1,18 @@
+type: sample
+status:
+  class: receiver
+  stability:
+    stable: [metrics]
+
+resource_attributes:
+  host.name:
+    description: The hostname
+    type: string
+    enabled: true
+
+entities:
+  host:
+    id_attributes:
+      - host.missing
+    description_attributes:
+      - host.name