feat: add distributed tracing with timing spans and trace propagation

Propagate trace context from build PipelineRuns to Snapshots,
integration PipelineRuns, and Release CRs. Emit waitDuration and
executeDuration timing spans for build and integration PipelineRuns.
Create a new root span when valid trace context is missing. See
docs/tracing.md.

Assisted-by: Claude Code
Signed-off-by: Josiah England <jengland@redhat.com>

Author: ci-operator

cmd/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"os"
@@ -25,6 +26,7 @@ import (
 	controllers "github.com/konflux-ci/integration-service/internal/controller"
 	iswebhook "github.com/konflux-ci/integration-service/internal/webhook/v1beta2"
 	imetrics "github.com/konflux-ci/integration-service/pkg/metrics"
+	"github.com/konflux-ci/integration-service/pkg/tracing"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -110,6 +112,18 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Initialize tracing
+	ctx := context.Background()
+	tracerProvider, err := tracing.New(ctx)
+	if err != nil {
+		setupLog.Error(err, "failed to initialize tracing")
+	}
+	defer func() {
+		if err := tracerProvider.Shutdown(ctx); err != nil {
+			setupLog.Error(err, "failed to shutdown tracer provider")
+		}
+	}()
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -193,7 +207,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	ctx := ctrl.SetupSignalHandler()
+	ctx = ctrl.SetupSignalHandler()
 	integrationMetrics := imetrics.NewIntegrationMetrics([]imetrics.AvailabilityProbe{imetrics.NewGithubAppAvailabilityProbe(mgr.GetClient())})
 	if err := integrationMetrics.InitMetrics(metrics.Registry); err != nil {
 		setupLog.Error(err, "unable to initialize metrics")

docs/README.md

@@ -5,6 +5,9 @@
 - [build-pipeline-controller](https://github.com/konflux-ci/integration-service/blob/main/docs/build_pipeline_controller.md)
 - [integration-pipeline-controller](https://github.com/konflux-ci/integration-service/blob/main/docs/integration_pipeline_controller.md)
 
+## Operational docs
+- [Distributed tracing](https://github.com/konflux-ci/integration-service/blob/main/docs/tracing.md)
+
 ## Creating or editing Mermaid diagrams
 
 Mermaid is a JS based diagramming tool that renders markdown style syntax to create/modify diagrams. Mermaid has [native support in Github](https://github.com/github/roadmap/issues/372)

docs/tracing.md

@@ -0,0 +1,49 @@
+<div align="center"><h1>Distributed tracing</h1></div>
+
+The operator emits OpenTelemetry spans for build and integration-test PipelineRuns it reconciles, and propagates the trace context forward onto Snapshots and downstream Release CRs so a single trace can span the build → test → release lifecycle.
+
+## Configuration
+
+| Env var | Purpose | Default |
+|---|---|---|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP/gRPC collector URL. Unset disables tracing (noop provider). | *(unset)* |
+| `OTEL_TRACES_SAMPLER` | `always_on`, `always_off`, `traceidratio`, `parentbased_always_off`, `parentbased_traceidratio`. | `parentbased_always_on` |
+| `OTEL_TRACES_SAMPLER_ARG` | Ratio for ratio-based samplers (e.g. `0.1`). | *(unused unless a ratio sampler is selected)* |
+| `TRACING_LABEL_ACTION` | PipelineRun label read to populate `cicd.pipeline.action.name`. Empty string disables the attribute. | `delivery.tekton.dev/action` |
+| `TRACING_LABEL_APPLICATION` | PipelineRun label read to populate `delivery.tekton.dev.application`. Empty string disables the attribute. | `delivery.tekton.dev/application` |
+| `TRACING_LABEL_COMPONENT` | PipelineRun label read to populate `delivery.tekton.dev.component`. Empty string disables the attribute. | `delivery.tekton.dev/component` |
+
+## Emitted spans
+
+Two spans are emitted per PipelineRun when it completes:
+
+- `waitDuration` — `pr.CreationTimestamp` → `pr.Status.StartTime`
+- `executeDuration` — `pr.Status.StartTime` → `pr.Status.CompletionTime`
+
+The build-pipeline and integration-pipeline controllers each emit for their respective PipelineRun types. The `tekton.dev/timingEmitted` annotation guards against re-emission on subsequent reconciles.
+
+## Trace-context propagation
+
+Parenting follows the W3C Trace Context in the `tekton.dev/pipelinerunSpanContext` annotation. The annotation is propagated across resource boundaries so a single trace covers the full delivery flow:
+
+```
+build PipelineRun (annotation set by upstream)
+   └── Snapshot (annotation copied from the build PipelineRun)
+         ├── integration-test PipelineRun (annotation copied from the Snapshot)
+         └── Release CR (annotation copied from the Snapshot)
+```
+
+When the annotation is absent, spans are still emitted but without a parent.
+
+## Span attributes
+
+| Attribute | Span | Source |
+|---|---|---|
+| `namespace` | both | `pr.GetNamespace()` |
+| `pipelinerun` | both | `pr.GetName()` |
+| `delivery.tekton.dev.pipelinerun_uid` | both | `pr.GetUID()` |
+| `cicd.pipeline.action.name` | both | PipelineRun label (name configurable via `TRACING_LABEL_ACTION`) |
+| `delivery.tekton.dev.application` | both | PipelineRun label (name configurable via `TRACING_LABEL_APPLICATION`) |
+| `delivery.tekton.dev.component` | both | PipelineRun label (name configurable via `TRACING_LABEL_COMPONENT`) |
+| `cicd.pipeline.result` | execute | `Succeeded` condition mapped to the semconv `cicd.pipeline.result` enum (`success` / `failure` / `timeout` / `cancellation` / `error`) |
+| `delivery.tekton.dev.result_message` | execute | Earliest failing TaskRun's `Succeeded` condition message, falling back to the PipelineRun's own condition message. Omitted on success; truncated to 1024 bytes (UTF-8 safe). |

go.mod

@@ -47,6 +47,10 @@ require (
 	github.com/tektoncd/pipeline v1.7.0
 	github.com/tonglil/buflogr v1.1.1
 	gitlab.com/gitlab-org/api/client-go v0.134.0
+	go.opentelemetry.io/otel v1.40.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0
+	go.opentelemetry.io/otel/sdk v1.40.0
+	go.opentelemetry.io/otel/trace v1.40.0
 	go.uber.org/mock v0.6.0
 	go.uber.org/zap v1.27.1
 	golang.org/x/exp v0.0.0-20260112195511-716be5621a96
@@ -152,12 +156,8 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect
 	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect

internal/controller/buildpipeline/buildpipeline_adapter.go

@@ -34,6 +34,7 @@ import (
 	h "github.com/konflux-ci/integration-service/helpers"
 	"github.com/konflux-ci/integration-service/loader"
 	intgteststat "github.com/konflux-ci/integration-service/pkg/integrationteststatus"
+	"github.com/konflux-ci/integration-service/pkg/tracing"
 	"github.com/konflux-ci/integration-service/snapshot"
 	"github.com/konflux-ci/integration-service/status"
 	"github.com/konflux-ci/integration-service/tekton"
@@ -343,6 +344,9 @@ func (a *Adapter) EnsureSnapshotExistsApplication() (result controller.Operation
 		return controller.RequeueWithError(err)
 	}
 
+	// Emit timing spans for the build PipelineRun if not already emitted
+	a.emitBuildTimingSpans()
+
 	canRemoveFinalizer = true
 	return controller.ContinueProcessing()
 }
@@ -753,6 +757,11 @@ func (a *Adapter) prepareSnapshotForPipelineRun(pipelineRun *tektonv1.PipelineRu
 	prefixes := []string{gitops.BuildPipelineRunPrefix, gitops.TestLabelPrefix, gitops.CustomLabelPrefix, gitops.ReleaseLabelPrefix}
 	gitops.CopySnapshotLabelsAndAnnotations(&application.ObjectMeta, snapshot, a.component.Name, &pipelineRun.ObjectMeta, prefixes, true)
 
+	// Propagate span context from build PipelineRun to Snapshot for distributed tracing
+	if tp, found := pipelineRun.Annotations[tektonconsts.SpanContextAnnotation]; found && tp != "" {
+		snapshot.Annotations[tektonconsts.SpanContextAnnotation] = tp
+	}
+
 	snapshot.Labels[gitops.BuildPipelineRunNameLabel] = pipelineRun.Name
 	if pipelineRun.Status.CompletionTime != nil {
 		snapshot.Labels[gitops.BuildPipelineRunFinishTimeLabel] = strconv.FormatInt(pipelineRun.Status.CompletionTime.Unix(), 10)
@@ -1359,3 +1368,30 @@ func (a *Adapter) IsLatestBuildPipelineRunInComponentWithPRGroupHash(buildPlr *t
 	a.logger.Info(fmt.Sprintf("The build pipelineRun %s/%s with pr group %s is not the latest for its component, skipped", buildPlr.Namespace, buildPlr.Name, prGroupName))
 	return false, nil
 }
+
+// emitBuildTimingSpans emits timing spans for the build PipelineRun if not already emitted
+func (a *Adapter) emitBuildTimingSpans() {
+	// Check if timing spans have already been emitted
+	if _, found := a.pipelineRun.Annotations[tektonconsts.TimingEmittedAnnotation]; found {
+		return // Already emitted
+	}
+
+	// Get span context from the PipelineRun
+	tp := a.pipelineRun.Annotations[tektonconsts.SpanContextAnnotation]
+
+	// Emit timing spans
+	if tracing.EmitTimingSpans(a.context, a.client, a.pipelineRun, tracing.LoadLabelNames(), tp) {
+		// Mark as emitted to avoid duplicates on subsequent reconciles
+		err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			var err error
+			a.pipelineRun, err = a.loader.GetPipelineRun(a.context, a.client, a.pipelineRun.Name, a.pipelineRun.Namespace)
+			if err != nil {
+				return err
+			}
+			return tekton.AnnotateBuildPipelineRun(a.context, a.pipelineRun, tektonconsts.TimingEmittedAnnotation, "true", a.client)
+		})
+		if err != nil {
+			a.logger.Error(err, "Failed to mark build timing spans as emitted")
+		}
+	}
+}

internal/controller/integrationpipeline/integrationpipeline_adapter.go

@@ -26,6 +26,7 @@ import (
 	h "github.com/konflux-ci/integration-service/helpers"
 	"github.com/konflux-ci/integration-service/loader"
 	intgteststat "github.com/konflux-ci/integration-service/pkg/integrationteststatus"
+	"github.com/konflux-ci/integration-service/pkg/tracing"
 	"github.com/konflux-ci/integration-service/status"
 
 	tektonconsts "github.com/konflux-ci/integration-service/tekton/consts"
@@ -107,6 +108,11 @@ func (a *Adapter) EnsureStatusReportedInSnapshot() (controller.OperationResult,
 		return controller.RequeueWithError(fmt.Errorf("failed to update test status in snapshot: %w", err))
 	}
 
+	// Emit timing spans for the integration test PipelineRun if finished
+	if h.HasPipelineRunFinished(a.pipelineRun) {
+		a.emitTestTimingSpans()
+	}
+
 	// Remove the finalizer from Integration PLRs if the snapshot is not group or component type and its PLR has finished
 	if (!gitops.IsGroupSnapshot(a.snapshot) && !gitops.IsComponentSnapshot(a.snapshot)) && (h.HasPipelineRunFinished(a.pipelineRun) ||
 		pipelinerunStatus == intgteststat.IntegrationTestStatusDeleted) {
@@ -206,3 +212,43 @@ func (a *Adapter) annotateIntegrationPipelineRunLogURL(ctx context.Context, adap
 		return err
 	})
 }
+
+// emitTestTimingSpans emits timing spans for the integration test PipelineRun if not already emitted.
+// Works for any snapshot type:
+//   - Component/Group snapshots: Parented under delivery trace from build PLR
+//   - Override snapshots: Emitted without parent trace (still useful for timing metrics)
+func (a *Adapter) emitTestTimingSpans() {
+	// Check if timing spans have already been emitted
+	if _, found := a.pipelineRun.Annotations[tektonconsts.TimingEmittedAnnotation]; found {
+		return // Already emitted
+	}
+
+	// Get span context from the snapshot (which got it from the build PLR)
+	tp := ""
+	if a.snapshot != nil && a.snapshot.Annotations != nil {
+		tp = a.snapshot.Annotations[tektonconsts.SpanContextAnnotation]
+	}
+
+	// Emit timing spans
+	emitted := tracing.EmitTimingSpans(a.context, a.client, a.pipelineRun, tracing.LoadLabelNames(), tp)
+
+	if emitted {
+		// Mark as emitted to avoid duplicates on subsequent reconciles
+		err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			var err error
+			a.pipelineRun, err = a.loader.GetPipelineRun(a.context, a.client, a.pipelineRun.Name, a.pipelineRun.Namespace)
+			if err != nil {
+				return err
+			}
+			patch := client.MergeFrom(a.pipelineRun.DeepCopy())
+			if a.pipelineRun.Annotations == nil {
+				a.pipelineRun.Annotations = make(map[string]string)
+			}
+			a.pipelineRun.Annotations[tektonconsts.TimingEmittedAnnotation] = "true"
+			return a.client.Patch(a.context, a.pipelineRun, patch)
+		})
+		if err != nil {
+			a.logger.Error(err, "Failed to mark test timing spans as emitted")
+		}
+	}
+}

internal/controller/snapshot/snapshot_adapter.go

@@ -1074,6 +1074,11 @@ func (a *Adapter) prepareGroupSnapshot(application *applicationapiv1alpha1.Appli
 		return nil, nil, err
 	}
 
+	// Propagate span context from triggering snapshot
+	if tp := a.snapshot.Annotations[tektonconsts.SpanContextAnnotation]; tp != "" {
+		groupSnapshot.Annotations[tektonconsts.SpanContextAnnotation] = tp
+	}
+
 	return groupSnapshot, componentSnapshotInfos, nil
 }