feat(gitlab): correlate CI job logs with OTel traces via journald

When both GitLab runner and OTel are configured, Podman is set to use
the journald log driver so CI job container output is captured by
systemd journal. The otelcol config gains a journald/gitlab-jobs
receiver that parses CONTAINER_NAME to extract job_id, project_id, and
runner_token, and the existing filelog/gitlab-runner receiver gets
regex_parser operators to promote job= and runner= fields from the
runner daemon log body into attributes. Both streams share job_id,
enabling cross-stream correlation in any OTel backend.

The journald log driver change is gated on a new LogToJournald field in
GitLabRunnerArgs, set only when hasOtel is true in the IBM Power and
IBM Z providers. The otelcol receivers remain gated on the existing
MonitorGitLabRunner flag.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: adrianriobo

pkg/integrations/gitlab/glrunner.go

@@ -44,12 +44,13 @@ func (args *GitLabRunnerArgs) GetUserDataValues() *integrations.UserDataValues {
 		return nil
 	}
 	return &integrations.UserDataValues{
-		Name:       args.Name,
-		Token:      args.AuthToken, // Use auth token (set by Pulumi during deployment)
-		CliURL:     downloadURL(),
-		RepoURL:    args.URL,
-		Unsecure:   args.Unsecure,
-		Concurrent: args.Concurrent,
+		Name:          args.Name,
+		Token:         args.AuthToken, // Use auth token (set by Pulumi during deployment)
+		CliURL:        downloadURL(),
+		RepoURL:       args.URL,
+		Unsecure:      args.Unsecure,
+		Concurrent:    args.Concurrent,
+		LogToJournald: args.LogToJournald,
 	}
 }
 

pkg/integrations/gitlab/snippet-linux.sh

@@ -32,7 +32,7 @@ if [ -z "$_dns_servers" ] && command -v nmcli &>/dev/null; then
         | tr '\n' ' ' | xargs)
 fi
 if [ -z "$_dns_servers" ]; then
-    _dns_servers=$(awk '/^nameserver/ && $2 !~ /^127\./ {print $2}' /etc/resolv.conf \
+    _dns_servers=$(awk '/^nameserver/ && $2 !~ /^127\./ && $2 != "::1" {print $2}' /etc/resolv.conf \
         | tr '\n' ' ' | xargs)
 fi
 if [ -n "$_dns_servers" ]; then
@@ -66,6 +66,32 @@ if [ -n "$_dns_servers" ]; then
     fi
 fi
 
+{{- if .LogToJournald}}
+# Set journald as the container log driver so CI job output is captured by the
+# systemd journal and can be correlated with runner daemon logs via job_id.
+sudo mkdir -p /etc/containers
+if [ ! -f /etc/containers/containers.conf ]; then
+    printf '[containers]\nlog_driver = "journald"\n' \
+        | sudo tee /etc/containers/containers.conf > /dev/null
+elif grep -q '^\[containers\]' /etc/containers/containers.conf; then
+    if awk '/^\[containers\]/{f=1;next} /^\[/{f=0} f && /^log_driver/{found=1} END{exit !found}' \
+            /etc/containers/containers.conf; then
+        # Replace existing log_driver within [containers]
+        awk '/^\[containers\]/{s=1} /^\[/ && !/^\[containers\]/{s=0}
+             s && /^log_driver/{$0="log_driver = \"journald\""} 1' \
+            /etc/containers/containers.conf \
+            | sudo tee /etc/containers/containers.conf.tmp > /dev/null \
+            && sudo mv /etc/containers/containers.conf.tmp /etc/containers/containers.conf
+    else
+        sudo sed -i '/^\[containers\]/a log_driver = "journald"' \
+            /etc/containers/containers.conf
+    fi
+else
+    printf '\n[containers]\nlog_driver = "journald"\n' \
+        | sudo tee -a /etc/containers/containers.conf > /dev/null
+fi
+{{- end}}
+
 # Register runner using docker executor backed by Podman
 # --docker-privileged is required for Podman: containers need CAP_SYS_ADMIN to mount /proc
 sudo gitlab-runner register \

pkg/integrations/gitlab/types.go

@@ -26,6 +26,7 @@ type GitLabRunnerArgs struct {
 	Arch        *Arch     // Target architecture
 	User        string    // OS user to run as (only used when Unsecure is true)
 	AuthToken   string    // Runner authentication token (set by Pulumi during deployment)
-	Unsecure    bool      // When false (default) a dedicated gitlab-runner system user is created; when true the runner service runs as User
-	Concurrent  int       // Maximum number of concurrent jobs (written to config.toml; 0 means leave at default of 1)
+	Unsecure      bool // When false (default) a dedicated gitlab-runner system user is created; when true the runner service runs as User
+	Concurrent    int  // Maximum number of concurrent jobs (written to config.toml; 0 means leave at default of 1)
+	LogToJournald bool // When true, sets Podman log_driver=journald so CI job output is captured by systemd journal for OTel correlation
 }

pkg/integrations/integrations.go

@@ -6,16 +6,17 @@ import (
 )
 
 type UserDataValues struct {
-	CliURL     string
-	User       string
-	Name       string
-	Token      string
-	Labels     string
-	Port       string
-	RepoURL    string
-	Executor   string
-	Unsecure   bool
-	Concurrent int
+	CliURL         string
+	User           string
+	Name           string
+	Token          string
+	Labels         string
+	Port           string
+	RepoURL        string
+	Executor       string
+	Unsecure       bool
+	Concurrent     int
+	LogToJournald  bool
 }
 
 type IntegrationConfig interface {

pkg/integrations/otelcol/snippet-linux.sh

@@ -143,10 +143,31 @@ receivers:
     - type: remove
       id: remove_file_name
       field: attributes["log.file.name"]
+    - type: regex_parser
+      id: parse_job_id
+      parse_from: body
+      regex: '\bjob=(?P<job_id>\d+)'
+      on_error: send
+    - type: regex_parser
+      id: parse_runner_token
+      parse_from: body
+      regex: '\brunner=(?P<runner_token>\w+)'
+      on_error: send
     attributes:
       index: "{{.Index}}"
       _sourceCategory: gitlab-runner
       _sourceHost: ${env:HOSTNAME}
+  journald/gitlab-jobs:
+    operators:
+    - type: regex_parser
+      id: parse_container_name
+      parse_from: attributes["CONTAINER_NAME"]
+      regex: '^runner-(?P<runner_token>.+?)-project-(?P<project_id>\d+)-concurrent-(?P<concurrent_id>\d+)-(?P<job_id>\d+)$'
+      on_error: send
+    attributes:
+      index: "{{.Index}}"
+      _sourceCategory: gitlab-runner-jobs
+      _sourceHost: ${env:HOSTNAME}
 {{- end}}
 processors:
   filter/drop_null_bytes:
@@ -185,7 +206,7 @@ service:
       level: "basic"
   pipelines:
     logs:
-      receivers: [filelog/syslog, filelog/secure, filelog/audit{{if .MonitorGitLabRunner}}, filelog/gitlab-runner{{end}}]
+      receivers: [filelog/syslog, filelog/secure, filelog/audit{{if .MonitorGitLabRunner}}, filelog/gitlab-runner, journald/gitlab-jobs{{end}}]
       processors: [filter/drop_null_bytes, resource, batch]
       exporters: [otlphttp]
 OTELEOF

pkg/provider/ibmcloud/action/ibm-power/ibm-power.go

@@ -192,6 +192,7 @@ func (r *pwRequest) deploy(ctx *pulumi.Context) error {
 		}
 		gateway := subnetInfo.Gateway
 		localArgs := *glRunnerArgs
+		localArgs.LogToJournald = hasOtel
 		piUserDataInput = authToken.ApplyT(func(token string) (*string, error) {
 			localArgs.AuthToken = token
 			glSnippet, err := integrations.GetIntegrationSnippetAsCloudInitWritableFile(&localArgs, defaultUser)

pkg/provider/ibmcloud/action/ibm-z/ibm-z.go

@@ -362,6 +362,7 @@ func (r *zRequest) buildUserDataInput() (pulumi.StringPtrInput, error) {
 	hasOtel := otelSet == 3
 	if r.glAuthToken != nil {
 		localArgs := *r.glRunnerArgsCopy
+		localArgs.LogToJournald = hasOtel
 		return r.glAuthToken.ApplyT(func(token string) (*string, error) {
 			localArgs.AuthToken = token
 			glSnippet, err := integrations.GetIntegrationSnippetAsCloudInitWritableFile(&localArgs, defaultUser)