Merge branch 'master' into fcovalente-system-log-test

Author: franciscovalentecastro

confgenerator/agentmetrics.go

@@ -80,7 +80,7 @@ var grpcToStringStatus = map[string]string{
 
 func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]otel.ReceiverPipeline, pipelines map[string]otel.Pipeline, ctx context.Context) {
 	// Receiver pipelines names should have 1 underscore to avoid collision with user configurations.
-	receiverPipelines["agent_prometheus"] = r.PrometheusMetricsPipeline()
+	receiverPipelines["agent_prometheus"] = r.PrometheusMetricsPipeline(ctx)
 
 	// Pipeline names should have no underscores to avoid collision with user configurations.
 	pipelines["otel"] = otel.Pipeline{
@@ -101,15 +101,15 @@ func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]o
 		Processors:           r.LoggingMetricsPipelineProcessors(),
 	}
 
-	receiverPipelines["ops_agent"] = r.OpsAgentPipeline()
+	receiverPipelines["ops_agent"] = r.OpsAgentPipeline(ctx)
 	pipelines["opsagent"] = otel.Pipeline{
 		Type:                 "metrics",
 		ReceiverPipelineName: "ops_agent",
 	}
 }
 
-func (r AgentSelfMetrics) PrometheusMetricsPipeline() otel.ReceiverPipeline {
-	return otel.ReceiverPipeline{
+func (r AgentSelfMetrics) PrometheusMetricsPipeline(ctx context.Context) otel.ReceiverPipeline {
+	return ConvertGCMSystemExporterToOtlpExporter(otel.ReceiverPipeline{
 		Receiver: otel.Component{
 			Type: "prometheus",
 			Config: map[string]interface{}{
@@ -150,7 +150,7 @@ func (r AgentSelfMetrics) PrometheusMetricsPipeline() otel.ReceiverPipeline {
 				),
 			},
 		},
-	}
+	}, ctx)
 }
 
 func (r AgentSelfMetrics) OtelPipelineProcessors(ctx context.Context) []otel.Component {
@@ -373,18 +373,18 @@ func (r AgentSelfMetrics) LoggingMetricsPipelineProcessors() []otel.Component {
 	}
 }
 
-func (r AgentSelfMetrics) OpsAgentPipeline() otel.ReceiverPipeline {
-	receiver_config := map[string]any{
+func (r AgentSelfMetrics) OpsAgentPipeline(ctx context.Context) otel.ReceiverPipeline {
+	receiverConfig := map[string]any{
 		"include": []string{
 			filepath.Join(r.OtelRuntimeDir, "enabled_receivers_otlp.json"),
 			filepath.Join(r.OtelRuntimeDir, "feature_tracking_otlp.json")},
 		"replay_file":   true,
 		"poll_interval": time.Duration(60 * time.Second).String(),
 	}
-	return otel.ReceiverPipeline{
+	return ConvertGCMSystemExporterToOtlpExporter(otel.ReceiverPipeline{
 		Receiver: otel.Component{
 			Type:   "otlpjsonfile",
-			Config: receiver_config,
+			Config: receiverConfig,
 		},
 		ExporterTypes: map[string]otel.ExporterType{
 			"metrics": otel.System,
@@ -394,7 +394,7 @@ func (r AgentSelfMetrics) OpsAgentPipeline() otel.ReceiverPipeline {
 				otel.Transform("metric", "datapoint", []ottl.Statement{"set(time, Now())"}),
 			},
 		},
-	}
+	}, ctx)
 }
 
 // intentionally not registered as a component because this is not created by users

confgenerator/confgenerator.go

@@ -82,7 +82,6 @@ func ConvertGCMSystemExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx
 
 func ConvertToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context, isPrometheus bool, isSystem bool) otel.ReceiverPipeline {
 	expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
-	resource, _ := platform.FromContext(ctx).GetResource()
 	if !expOtlpExporter {
 		return pipeline
 	}
@@ -91,7 +90,6 @@ func ConvertToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context,
 		return pipeline
 	}
 	pipeline.ExporterTypes["metrics"] = otel.OTLP
-	pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.GCPProjectID(resource.ProjectName()))
 	if isSystem {
 		pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricsRemoveInstrumentationLibraryLabelsAttributes())
 		pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricsRemoveServiceAttributes())
@@ -100,9 +98,7 @@ func ConvertToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context,
 	// b/476109839: For prometheus metrics using the OTLP exporter. The dots "." in the metric name are NOT replaced with underscore "_".
 	// This is diffrent from the GMP endpoint.
 	if isPrometheus {
-
 		pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricUnknownCounter())
-		pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricStartTime())
 		// If a metric already has a domain, it will not be considered a prometheus metric by the UTR endpoint unless we add the prefix.
 		// This behavior is the same as the GCM/GMP exporters.
 		pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricsTransform(otel.AddPrefix("prometheus.googleapis.com")))
@@ -194,6 +190,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi
 		OtelLogging:         uc.Logging.Service.OTelLogging,
 	}
 	agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines, ctx)
+	resource, _ := p.GetResource()
 
 	otelConfig, err := otel.ModularConfig{
 		LogLevel:          uc.getOTelLogLevel(),
@@ -216,6 +213,8 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi
 					// The OTLP exporter doesn't batch by default like the googlecloud.* exporters.
 					// We need this to avoid the API point limits.
 					"metrics": {
+						otel.GCPProjectID(resource.ProjectName()),
+						otel.MetricStartTime(),
 						otel.BatchProcessor(200, 200, "200ms"),
 					},
 					// Batching logs improves log export performance.

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/otel.yaml

@@ -1,12 +1,4 @@
 exporters:
-  googlecloud:
-    metric:
-      instrumentation_library_labels: false
-      prefix: ""
-      resource_filters: []
-      service_resource_labels: false
-      skip_create_descriptor: true
-    user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
   googlecloud/otel:
     metric:
       instrumentation_library_labels: true
@@ -28,7 +20,7 @@ processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
     - system.cpu.utilization
-  batch/otlp/otlp_metrics_0:
+  batch/otlp/otlp_metrics_2:
     send_batch_max_size: 200
     send_batch_size: 200
     timeout: 200ms
@@ -105,7 +97,7 @@ processors:
     - location
   interval/loggingmetrics_7:
     interval: 1m
-  metricstarttime/otlp_5:
+  metricstarttime/otlp/otlp_metrics_1:
     strategy: subtract_initial_point
   metricstransform/fluentbit_1:
     transforms:
@@ -658,23 +650,13 @@ processors:
       include: ^(.*)$$
       match_type: regexp
       new_name: agent.googleapis.com/$${1}
-  metricstransform/otlp_6:
+  metricstransform/otlp_4:
     transforms:
     - action: update
       include: ^(.*)$$
       match_type: regexp
       new_name: prometheus.googleapis.com/$${1}
-  resource/hostmetrics_1_1:
-    attributes:
-    - action: insert
-      key: gcp.project_id
-      value: test-project
-  resource/hostmetrics_3:
-    attributes:
-    - action: insert
-      key: gcp.project_id
-      value: test-project
-  resource/otlp_3:
+  resource/otlp/otlp_metrics_0:
     attributes:
     - action: insert
       key: gcp.project_id
@@ -699,13 +681,13 @@ processors:
       - delete_key(resource.attributes, "service.instance.id")
       - delete_key(resource.attributes, "server.port")
       - delete_key(resource.attributes, "url.scheme")
-  transform/hostmetrics_1_2:
+  transform/agent_prometheus_1:
     metric_statements:
     - context: scope
       statements:
       - set(name, "")
       - set(version, "")
-  transform/hostmetrics_1_3:
+  transform/agent_prometheus_2:
     metric_statements:
     - context: resource
       error_mode: silent
@@ -714,13 +696,28 @@ processors:
       - delete_key(attributes, "service.instance.id")
       - delete_key(attributes, "service.namespace")
       - delete_key(attributes, "service.version")
-  transform/hostmetrics_4:
+  transform/hostmetrics_1_1:
     metric_statements:
     - context: scope
       statements:
       - set(name, "")
       - set(version, "")
-  transform/hostmetrics_5:
+  transform/hostmetrics_1_2:
+    metric_statements:
+    - context: resource
+      error_mode: silent
+      statements:
+      - delete_key(attributes, "service.name")
+      - delete_key(attributes, "service.instance.id")
+      - delete_key(attributes, "service.namespace")
+      - delete_key(attributes, "service.version")
+  transform/hostmetrics_3:
+    metric_statements:
+    - context: scope
+      statements:
+      - set(name, "")
+      - set(version, "")
+  transform/hostmetrics_4:
     metric_statements:
     - context: resource
       error_mode: silent
@@ -756,6 +753,21 @@ processors:
     - context: datapoint
       statements:
       - set(time, Now())
+  transform/ops_agent_1:
+    metric_statements:
+    - context: scope
+      statements:
+      - set(name, "")
+      - set(version, "")
+  transform/ops_agent_2:
+    metric_statements:
+    - context: resource
+      error_mode: silent
+      statements:
+      - delete_key(attributes, "service.name")
+      - delete_key(attributes, "service.instance.id")
+      - delete_key(attributes, "service.namespace")
+      - delete_key(attributes, "service.version")
   transform/otel_0:
     error_mode: ignore
     metric_statements:
@@ -772,7 +784,7 @@ processors:
       - set(attributes["cluster"], "__gce__") where "__gce__" != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine"
       - set(attributes["instance_name"], resource.attributes["host.name"]) where resource.attributes["host.name"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine"
       - set(attributes["machine_type"], resource.attributes["host.type"]) where resource.attributes["host.type"] != nil and resource.attributes["cloud.platform"] == "gcp_compute_engine"
-  transform/otlp_4:
+  transform/otlp_3:
     error_mode: ignore
     metric_statements:
     - context: metric
@@ -833,42 +845,51 @@ service:
       - agentmetrics/hostmetrics_0
       - filter/hostmetrics_1
       - metricstransform/hostmetrics_2
-      - resource/hostmetrics_3
+      - transform/hostmetrics_3
       - transform/hostmetrics_4
-      - transform/hostmetrics_5
       - filter/default__pipeline_hostmetrics_0
       - resourcedetection/_global_0
-      - batch/otlp/otlp_metrics_0
+      - resource/otlp/otlp_metrics_0
+      - metricstarttime/otlp/otlp_metrics_1
+      - batch/otlp/otlp_metrics_2
       receivers:
       - hostmetrics/hostmetrics
     metrics/default__pipeline_hostmetrics_1:
       exporters:
       - otlp/otlp
       processors:
       - metricstransform/hostmetrics_1_0
-      - resource/hostmetrics_1_1
+      - transform/hostmetrics_1_1
       - transform/hostmetrics_1_2
-      - transform/hostmetrics_1_3
       - filter/default__pipeline_hostmetrics_1_0
       - resourcedetection/_global_0
-      - batch/otlp/otlp_metrics_0
+      - resource/otlp/otlp_metrics_0
+      - metricstarttime/otlp/otlp_metrics_1
+      - batch/otlp/otlp_metrics_2
       receivers:
       - nvml/hostmetrics_1
     metrics/fluentbit:
       exporters:
-      - googlecloud
+      - otlp/otlp
       processors:
       - transform/agent_prometheus_0
+      - transform/agent_prometheus_1
+      - transform/agent_prometheus_2
       - filter/fluentbit_0
       - metricstransform/fluentbit_1
       - resourcedetection/_global_0
+      - resource/otlp/otlp_metrics_0
+      - metricstarttime/otlp/otlp_metrics_1
+      - batch/otlp/otlp_metrics_2
       receivers:
       - prometheus/agent_prometheus
     metrics/loggingmetrics:
       exporters:
-      - googlecloud
+      - otlp/otlp
       processors:
       - transform/agent_prometheus_0
+      - transform/agent_prometheus_1
+      - transform/agent_prometheus_2
       - transform/loggingmetrics_0
       - filter/loggingmetrics_1
       - filter/loggingmetrics_2
@@ -879,26 +900,39 @@ service:
       - interval/loggingmetrics_7
       - metricstransform/loggingmetrics_8
       - resourcedetection/_global_0
+      - resource/otlp/otlp_metrics_0
+      - metricstarttime/otlp/otlp_metrics_1
+      - batch/otlp/otlp_metrics_2
       receivers:
       - prometheus/agent_prometheus
     metrics/opsagent:
       exporters:
-      - googlecloud
+      - otlp/otlp
       processors:
       - transform/ops_agent_0
+      - transform/ops_agent_1
+      - transform/ops_agent_2
       - resourcedetection/_global_0
+      - resource/otlp/otlp_metrics_0
+      - metricstarttime/otlp/otlp_metrics_1
+      - batch/otlp/otlp_metrics_2
       receivers:
       - otlpjsonfile/ops_agent
     metrics/otel:
       exporters:
-      - googlecloud
+      - otlp/otlp
       processors:
       - transform/agent_prometheus_0
+      - transform/agent_prometheus_1
+      - transform/agent_prometheus_2
       - transform/otel_0
       - filter/otel_1
       - filter/otel_2
       - metricstransform/otel_3
       - resourcedetection/_global_0
+      - resource/otlp/otlp_metrics_0
+      - metricstarttime/otlp/otlp_metrics_1
+      - batch/otlp/otlp_metrics_2
       receivers:
       - prometheus/agent_prometheus
     metrics/otlp_otlp:
@@ -908,11 +942,11 @@ service:
       - resourcedetection/otlp_0
       - transform/otlp_1
       - groupbyattrs/otlp_2
-      - resource/otlp_3
-      - transform/otlp_4
-      - metricstarttime/otlp_5
-      - metricstransform/otlp_6
-      - batch/otlp/otlp_metrics_0
+      - transform/otlp_3
+      - metricstransform/otlp_4
+      - resource/otlp/otlp_metrics_0
+      - metricstarttime/otlp/otlp_metrics_1
+      - batch/otlp/otlp_metrics_2
       receivers:
       - otlp/otlp
     traces/traces_otlp_otlp: