Skip to content

Add support for lazy protobuf deserialization of Metric #14371

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ldufr wants to merge 5 commits into
base: main
Choose a base branch
from
+19,857 −194
Open

Add support for lazy protobuf deserialization of Metric #14371

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9cdfa30
Added lazy annotation
ldufr Nov 17, 2025
7bd5605
Fixing manual tests
ldufr Nov 28, 2025
a48e3bf
regen
ldufr Jan 5, 2026
46065e2
Add verification of Lazy messages at marshalling time
ldufr Nov 27, 2025
6374bf4
regen
ldufr Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions internal/cmd/pdatagen/internal/pdata/base_slices.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type messageSlice struct {
structName string
packageName string
elementNullable bool
lazy bool
element *messageStruct
}

Expand Down Expand Up @@ -57,6 +58,7 @@ func (ss *messageSlice) templateFields(packageInfo *PackageInfo) map[string]any
"elementName": ss.element.getName(),
"elementOriginName": ss.getElementOriginName(),
"elementNullable": ss.elementNullable,
"lazy": ss.lazy,
"origAccessor": origAccessor(hasWrapper),
"stateAccessor": stateAccessor(hasWrapper),
"packageName": packageInfo.name,
Expand All @@ -82,6 +84,9 @@ func (ss *messageSlice) getHasOnlyInternal() bool {
}

func (ss *messageSlice) getElementOriginName() string {
if ss.lazy {
return "Lazy"+ss.element.getOriginName()
}
return ss.element.getOriginName()
}

Expand Down
10 changes: 10 additions & 0 deletions internal/cmd/pdatagen/internal/pdata/one_of_field.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,11 @@ const oneOfUnmarshalProtoTemplate = `
{{ .GenUnmarshalProto }}
{{ end }}`

const oneOfSkipProtoTemplate = `
{{- range .fields }}
{{ .GenSkipProto }}
{{- end }}`

type OneOfField struct {
originFieldName string
typeName string
Expand Down Expand Up @@ -229,6 +234,11 @@ func (of *oneOfProtoField) GenUnmarshalProto() string {
return template.Execute(t, of.templateFields())
}

func (of *oneOfProtoField) GenSkipProto() string {
t := template.Parse("oneOfSkipProtoTemplate", []byte(oneOfSkipProtoTemplate))
return template.Execute(t, of.templateFields())
}

func (of *oneOfProtoField) templateFields() map[string]any {
return map[string]any{
"originFieldName": of.originFieldName,
Expand Down
5 changes: 5 additions & 0 deletions internal/cmd/pdatagen/internal/pdata/packages.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
)

var nonInternalDeps = []string{
`"go.opentelemetry.io/collector/pdata"`,
`"go.opentelemetry.io/collector/pdata/internal"`,
`"go.opentelemetry.io/collector/pdata/pcommon"`,
`"go.opentelemetry.io/collector/pdata/plog"`,
Expand Down Expand Up @@ -136,6 +137,10 @@ func (p *Package) GenerateProtoMessageFiles() error {
if err := os.WriteFile(path, pm.GenerateMessage(p.info.imports, p.info.testImports), 0o600); err != nil {
return err
}
path = filepath.Join("pdata", "internal", "generated_proto_lazy"+strings.ToLower(s.getOriginName())+".go")
if err := os.WriteFile(path, pm.GenerateLazyMessage(p.info.imports, p.info.testImports), 0o600); err != nil {
return err
}
p.info.testImports = saveTestImports
}
return nil
Expand Down
1 change: 1 addition & 0 deletions internal/cmd/pdatagen/internal/pdata/pmetric_package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ var scopeMetrics = &messageStruct{
var metricSlice = &messageSlice{
structName: "MetricSlice",
elementNullable: true,
lazy: true,
element: metric,
}

Expand Down
3 changes: 3 additions & 0 deletions internal/cmd/pdatagen/internal/pdata/slice_field.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ type SliceField struct {
protoID uint32
returnSlice baseSlice
hideAccessors bool
lazy bool
}

func (sf *SliceField) GenerateAccessors(ms *messageStruct) string {
Expand Down Expand Up @@ -68,6 +69,7 @@ func (sf *SliceField) toProtoField(ms *messageStruct) proto.FieldInterface {
ParentMessageName: ms.protoName,
Repeated: sf.protoType != proto.TypeBytes,
Nullable: sf.returnSlice.getElementNullable(),
Lazy: sf.lazy,
}
}

Expand All @@ -88,6 +90,7 @@ func (sf *SliceField) templateFields(ms *messageStruct) map[string]any {
"stateAccessor": stateAccessor(ms.getHasWrapper()),
"elementHasWrapper": sf.returnSlice.getHasWrapper(),
"elementNullable": sf.returnSlice.getElementNullable(),
"lazy": sf.lazy,
}
}

Expand Down
26 changes: 25 additions & 1 deletion internal/cmd/pdatagen/internal/pdata/templates/slice.go.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,29 @@ func (es {{ .structName }}) Len() int {
// e := es.At(i)
// ... // Do something with the element
// }
{{ if not .lazy -}}
func (es {{ .structName }}) At(i int) {{ .elementName }} {
return new{{ .elementName }}({{ if not .elementNullable }}&{{ end }}(*es.{{ .origAccessor }})[i], es.{{ .stateAccessor }})
}
{{- else -}}
func (es {{ .structName }}) At(i int) {{ .elementName }} {
res, err := (*es.{{ .origAccessor }})[i].FinishUnmarshal(&(*es.{{ .origAccessor }})[i].{{ .elementName }})
if err != nil {
return New{{ .elementName }}()
}
return new{{ .elementName }}(res, es.{{ .stateAccessor }})
}

func (es {{ .structName }}) Unmarshal(i int, m *{{ .elementName }}) error {
var buf internal.{{ .elementOriginName }}
res, err := (*es.{{ .origAccessor }})[i].FinishUnmarshal(&buf.{{ .elementName }})
if err != nil {
return err
}
*m = new{{ .elementName }}(res, es.{{ .stateAccessor }})
return nil
}
{{- end }}
Comment on lines +73 to +82
Copy link
Member

@bogdandrutu bogdandrutu Jan 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should not publicly expose this.

Copy link
Author

@ldufr ldufr Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The name is definitely too much details, but I'm not sure we can easily get rid of the functionality it does.

I considered few other name, like ReadInto, Get, but didn't find anything great. I wonder if you have any better name in mind?

I don't think it would ever be possible for this function to error, because we eagerly check the wire format, but hard to be 100%. Also, regardless whether we error or not, I don't think we can replace the At implementation, because some code calls At(), then modify the data, expecting it will modify the original document. This is why, the At deserialize the data in the document, which won't break existing code, but makes the lazy deserializing useless.

Copy link
Member

@bogdandrutu bogdandrutu Jan 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's make sure we make progress. Can you split the first PR into a PR that adds the "VerifyUnmarshalProto"?


// All returns an iterator over index-value pairs in the slice.
//
Expand Down Expand Up @@ -104,7 +124,11 @@ func (es {{ .structName }}) EnsureCapacity(newCap int) {
func (es {{ .structName }}) AppendEmpty() {{ .elementName }} {
es.{{ .stateAccessor }}.AssertMutable()
*es.{{ .origAccessor }} = append(*es.{{ .origAccessor }}, {{- if .elementNullable }}internal.New{{ .elementOriginName }}(){{ else }}internal.{{ .elementOriginName }}{}{{ end }})
return es.At(es.Len() - 1)
{{ if not .lazy -}}
return new{{ .elementName }}({{ if not .elementNullable }}&{{ end }}(*es.{{ .origAccessor }})[es.Len() - 1], es.{{ .stateAccessor }})
{{- else -}}
return new{{ .elementName }}(&(*es.{{ .origAccessor }})[es.Len() - 1].{{ .elementName }}, es.{{ .stateAccessor }})
{{- end }}
}

// MoveAndAppendTo moves all elements from the current slice and appends them to the dest.
Expand Down
3 changes: 3 additions & 0 deletions internal/cmd/pdatagen/internal/proto/field.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ type FieldInterface interface {

GenUnmarshalProto() string

GenSkipProto() string

GenMessageField() string

GenOneOfMessages() string
Expand All @@ -56,6 +58,7 @@ type Field struct {
ID uint32
Repeated bool
Nullable bool
Lazy bool
}

func (pf *Field) GetName() string { return pf.Name }
Expand Down
8 changes: 8 additions & 0 deletions internal/cmd/pdatagen/internal/proto/message.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ var (
messageTemplateBytes []byte
messageTemplate = template.Parse("message_internal_test.go", messageTemplateBytes)

//go:embed templates/lazy_message.go.tmpl
lazyMessageTemplateBytes []byte
lazyMessageTemplate = template.Parse("message_internal_test.go", lazyMessageTemplateBytes)

//go:embed templates/message_test.go.tmpl
messageTestTemplateBytes []byte
messageTestTemplate = template.Parse("message_internal_test.go", messageTestTemplateBytes)
Expand All @@ -33,6 +37,10 @@ func (ms *Message) GenerateMessage(imports, testImports []string) []byte {
return []byte(template.Execute(messageTemplate, ms.templateFields(imports, testImports)))
}

func (ms *Message) GenerateLazyMessage(imports, testImports []string) []byte {
return []byte(template.Execute(lazyMessageTemplate, ms.templateFields(imports, testImports)))
}

func (ms *Message) GenerateMessageTests(imports, testImports []string) []byte {
return []byte(template.Execute(messageTestTemplate, ms.templateFields(imports, testImports)))
}
Expand Down
Loading