Skip to content

Enable OpenTelemetry logging for unmodified default pipelines. #2283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
quentinmit wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Enable OpenTelemetry logging for unmodified default pipelines. #2283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c5a83ec
Enable OpenTelemetry logging for unmodified default pipelines.
quentinmit Apr 29, 2026
107b23d
Update goldens
quentinmit Apr 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
4 changes: 1 addition & 3 deletions confgenerator/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1096,8 +1096,6 @@ func (uc *UnifiedConfig) loggingPipelines(ctx context.Context) ([]PipelineInstan
}
platformDefaultConfig := BuiltInConfStructs[platform.FromContext(ctx).Name()].Logging
exp_otlp := experiments.FromContext(ctx)["otlp_logging"]
// N.B. Temporarily gate the "auto" otel logging behind an experiment flag.
exp_otel := experiments.FromContext(ctx)["otel_logging"]
force_otel := l.Service.OTelLogging
var out []PipelineInstance
for _, pID := range otel.SortedKeys(l.Service.Pipelines) {
Expand Down Expand Up @@ -1140,7 +1138,7 @@ func (uc *UnifiedConfig) loggingPipelines(ctx context.Context) ([]PipelineInstan
Processors: processors,
}
if (force_otel != nil && *force_otel) || // User asked for OTel logging
(exp_otel && force_otel == nil && isDefaultPipeline) || // Unmodified default pipeline
(force_otel == nil && isDefaultPipeline) || // Unmodified default pipeline
(receiver.Type() == "otlp" && exp_otlp) { // OTLP receiver
instance.Backend = BackendOTel
}
Expand Down
74 changes: 0 additions & 74 deletions ..._compatible_with_explicit_exporters/golden/linux-gpu/adea349dc2d92cd07daa1d7847f5e96a.lua
View file
Open in desktop

This file was deleted.

34 changes: 0 additions & 34 deletions ...ens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/fluent_bit_main.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,6 @@
Scrape_Interval 60
Scrape_On_Start True

[INPUT]
Buffer_Chunk_Size 512k
Buffer_Max_Size 2M
DB ${buffers_dir}/default_pipeline_syslog
DB.locking true
Key message
Mem_Buf_Limit 10M
Name tail
Path /var/log/messages,/var/log/syslog
Read_from_Head True
Rotate_Wait 30
Skip_Long_Lines On
Tag default_pipeline.syslog
storage.type filesystem

[INPUT]
Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"}
Interval_NSec 0
Expand Down Expand Up @@ -69,12 +54,6 @@
Tag ops-agent-health
storage.type memory

[FILTER]
Match default_pipeline.syslog
Name lua
call process
script adea349dc2d92cd07daa1d7847f5e96a.lua

[FILTER]
Match ops-agent-fluent-bit
Name lua
Expand Down Expand Up @@ -155,19 +134,6 @@
call process
script 4d6012ff003886818fb9b9285b4af962.lua

[OUTPUT]
Match_Regex ^(default_pipeline\.syslog)$
Name stackdriver
Retry_Limit 3
http_request_key logging.googleapis.com/httpRequest
net.connect_timeout_log_error False
resource gce_instance
stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
storage.total_limit_size 2G
tls On
tls.verify Off
workers 8

[OUTPUT]
Match_Regex ^(ops-agent-health|ops-agent-fluent-bit)$
Name stackdriver
Expand Down
73 changes: 73 additions & 0 deletions ...stdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,24 @@ exporters:
service_resource_labels: false
skip_create_descriptor: true
user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
googlecloud/logging:
sending_queue:
batch:
flush_timeout: 200ms
max_size: 1000
min_size: 1000
sizer: items
block_on_overflow: true
enabled: true
num_consumers: 10
queue_size: 12000
sizer: items
storage: file_storage
timeout: 3600s
extensions:
file_storage:
create_directory: true
directory: file_storage
processors:
agentmetrics/hostmetrics_0:
blank_label_metrics:
Expand Down Expand Up @@ -612,7 +630,52 @@ processors:
- context: metric
statements:
- extract_count_metric(true) where name == "grpc_client_attempt_duration"
transform/syslog_0:
error_mode: ignore
log_statements:
- context: log
statements:
- delete_key(cache, "__field_0") where (cache != nil and cache["__field_0"] != nil)
- set(cache["__field_0"], attributes["compute.googleapis.com/instance_group_manager/name"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/name"] != nil)
- delete_key(cache, "__field_1") where (cache != nil and cache["__field_1"] != nil)
- set(cache["__field_1"], attributes["compute.googleapis.com/instance_group_manager/zone"]) where (attributes != nil and attributes["compute.googleapis.com/instance_group_manager/zone"] != nil)
- delete_key(cache, "__field_2") where (cache != nil and cache["__field_2"] != nil)
- set(cache["__field_2"], attributes["compute.googleapis.com/resource_name"]) where (attributes != nil and attributes["compute.googleapis.com/resource_name"] != nil)
- delete_key(cache, "__field_3") where (cache != nil and cache["__field_3"] != nil)
- set(cache["__field_3"], attributes["gcp.log_name"]) where (attributes != nil and attributes["gcp.log_name"] != nil)
- delete_key(cache, "value") where (cache != nil and cache["value"] != nil)
- set(cache["value"], cache["__field_0"])
- set(cache["value"], "test-mig") where cache["value"] == nil
- set(attributes["compute.googleapis.com/instance_group_manager/name"], cache["value"]) where (cache != nil and cache["value"] != nil)
- delete_key(cache, "value") where (cache != nil and cache["value"] != nil)
- set(cache["value"], cache["__field_1"])
- set(cache["value"], "test-zone") where cache["value"] == nil
- set(attributes["compute.googleapis.com/instance_group_manager/zone"], cache["value"]) where (cache != nil and cache["value"] != nil)
- delete_key(cache, "value") where (cache != nil and cache["value"] != nil)
- set(cache["value"], cache["__field_2"])
- set(cache["value"], "") where cache["value"] == nil
- set(attributes["compute.googleapis.com/resource_name"], cache["value"]) where (cache != nil and cache["value"] != nil)
- delete_key(cache, "value") where (cache != nil and cache["value"] != nil)
- set(cache["value"], cache["__field_3"])
- set(cache["value"], "syslog") where cache["value"] == nil
- set(attributes["gcp.log_name"], cache["value"]) where (cache != nil and cache["value"] != nil)
receivers:
filelog/syslog:
exclude: []
fingerprint_size: 5kb
include:
- /var/log/messages
- /var/log/syslog
include_file_name: false
operators:
- from: body
id: body
to: body.message
type: move
preserve_leading_whitespaces: true
preserve_trailing_whitespaces: true
start_at: beginning
storage: file_storage
hostmetrics/hostmetrics:
collection_interval: 60s
scrapers:
Expand Down Expand Up @@ -651,7 +714,17 @@ receivers:
- targets:
- 0.0.0.0:20201
service:
extensions:
- file_storage
pipelines:
logs/logs_default__pipeline_syslog:
exporters:
- googlecloud/logging
processors:
- transform/syslog_0
- resourcedetection/_global_0
receivers:
- filelog/syslog
metrics/default__pipeline_hostmetrics:
exporters:
- googlecloud
Expand Down
74 changes: 0 additions & 74 deletions ...ward_compatible_with_explicit_exporters/golden/linux/adea349dc2d92cd07daa1d7847f5e96a.lua
View file
Open in desktop

This file was deleted.

34 changes: 0 additions & 34 deletions ...goldens/all-backward_compatible_with_explicit_exporters/golden/linux/fluent_bit_main.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,6 @@
Scrape_Interval 60
Scrape_On_Start True

[INPUT]
Buffer_Chunk_Size 512k
Buffer_Max_Size 2M
DB ${buffers_dir}/default_pipeline_syslog
DB.locking true
Key message
Mem_Buf_Limit 10M
Name tail
Path /var/log/messages,/var/log/syslog
Read_from_Head True
Rotate_Wait 30
Skip_Long_Lines On
Tag default_pipeline.syslog
storage.type filesystem

[INPUT]
Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"}
Interval_NSec 0
Expand Down Expand Up @@ -69,12 +54,6 @@
Tag ops-agent-health
storage.type memory

[FILTER]
Match default_pipeline.syslog
Name lua
call process
script adea349dc2d92cd07daa1d7847f5e96a.lua

[FILTER]
Match ops-agent-fluent-bit
Name lua
Expand Down Expand Up @@ -155,19 +134,6 @@
call process
script 4d6012ff003886818fb9b9285b4af962.lua

[OUTPUT]
Match_Regex ^(default_pipeline\.syslog)$
Name stackdriver
Retry_Limit 3
http_request_key logging.googleapis.com/httpRequest
net.connect_timeout_log_error False
resource gce_instance
stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
storage.total_limit_size 2G
tls On
tls.verify Off
workers 8

[OUTPUT]
Match_Regex ^(ops-agent-health|ops-agent-fluent-bit)$
Name stackdriver
Expand Down
Loading
Loading