feat: add distributed tracing with timing spans and trace propagation

Propagate W3C trace context from build PipelineRuns to Snapshots,
integration PipelineRuns, and Release CRs. Emit waitDuration and
executeDuration timing spans on completed PipelineRuns across both
ComponentGroup and Application model flows.

Tracing is opt-in via OTEL_EXPORTER_OTLP_ENDPOINT; without it set, the
service uses a noop tracer. The sampler family is selected via
OTEL_TRACES_SAMPLER.

Assisted-by: Claude Code
Signed-off-by: Josiah England <jengland@redhat.com>

Author: ci-operator

cmd/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"os"
@@ -25,6 +26,7 @@ import (
 	controllers "github.com/konflux-ci/integration-service/internal/controller"
 	iswebhook "github.com/konflux-ci/integration-service/internal/webhook/v1beta2"
 	imetrics "github.com/konflux-ci/integration-service/pkg/metrics"
+	"github.com/konflux-ci/integration-service/pkg/tracing"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -110,6 +112,16 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Initialize tracing
+	tracerProvider := tracing.New()
+	defer func() {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := tracerProvider.Shutdown(shutdownCtx); err != nil {
+			setupLog.Error(err, "failed to shutdown tracer provider")
+		}
+	}()
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and

docs/README.md

@@ -5,6 +5,9 @@
 - [build-pipeline-controller](https://github.com/konflux-ci/integration-service/blob/main/docs/build_pipeline_controller.md)
 - [integration-pipeline-controller](https://github.com/konflux-ci/integration-service/blob/main/docs/integration_pipeline_controller.md)
 
+## Operational docs
+- [Distributed tracing](https://github.com/konflux-ci/integration-service/blob/main/docs/tracing.md)
+
 ## Creating or editing Mermaid diagrams
 
 Mermaid is a JS based diagramming tool that renders markdown style syntax to create/modify diagrams. Mermaid has [native support in Github](https://github.com/github/roadmap/issues/372)

docs/tracing.md

@@ -0,0 +1,53 @@
+<div align="center"><h1>Distributed tracing</h1></div>
+
+The operator emits OpenTelemetry spans for build and integration-test PipelineRuns it reconciles, and propagates the trace context forward onto Snapshots and downstream Release CRs so a single trace can span the build, test, and release lifecycle.
+
+## Configuration
+
+| Env var | Purpose | Default |
+|---|---|---|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP/gRPC collector URL. Unset disables tracing (noop provider). | *(unset)* |
+| `OTEL_TRACES_SAMPLER` | `always_on`, `always_off`, `traceidratio`, `parentbased_always_off`, `parentbased_traceidratio`. | `parentbased_always_on` |
+| `OTEL_TRACES_SAMPLER_ARG` | Ratio for ratio-based samplers (e.g. `0.1`). | *(unused unless a ratio sampler is selected)* |
+| `TRACING_LABEL_ACTION` | PipelineRun label read to populate `cicd.pipeline.action.name`. Empty string disables the attribute. | `delivery.tekton.dev/action` |
+| `TRACING_LABEL_APPLICATION` | PipelineRun label read to populate `delivery.tekton.dev.application`. Empty string disables the attribute. | `delivery.tekton.dev/application` |
+| `TRACING_LABEL_COMPONENT` | PipelineRun label read to populate `delivery.tekton.dev.component`. Empty string disables the attribute. | `delivery.tekton.dev/component` |
+
+## Emitted spans
+
+Two spans are emitted per PipelineRun when it completes:
+
+- `waitDuration`: `pr.CreationTimestamp` to `pr.Status.StartTime`
+- `executeDuration`: `pr.Status.StartTime` to `pr.Status.CompletionTime`
+
+The build-pipeline and integration-pipeline controllers each emit for their respective PipelineRun types. The `delivery.tekton.dev/timingEmitted` annotation guards against re-emission on subsequent reconciles.
+
+## Trace-context propagation
+
+Parenting follows the W3C Trace Context in the `tekton.dev/pipelinerunSpanContext` annotation. The annotation is propagated across resource boundaries so a single trace covers the full delivery flow:
+
+```
+build PipelineRun (annotation set by upstream)
+   └── Snapshot (annotation copied from the build PipelineRun)
+         ├── integration-test PipelineRun (annotation copied from the Snapshot)
+         └── Release CR (annotation copied from the Snapshot)
+```
+
+When the annotation is absent, spans are still emitted but without a parent.
+
+## Span attributes
+
+| Attribute | Span | Source |
+|---|---|---|
+| `namespace` | both | `pr.GetNamespace()` |
+| `pipelinerun` | both | `pr.GetName()` |
+| `delivery.tekton.dev.pipelinerun_uid` | both | `pr.GetUID()` |
+| `cicd.pipeline.action.name` | both | PipelineRun label (name configurable via `TRACING_LABEL_ACTION`) |
+| `delivery.tekton.dev.application` | both | PipelineRun label (name configurable via `TRACING_LABEL_APPLICATION`) |
+| `delivery.tekton.dev.component` | both | PipelineRun label (name configurable via `TRACING_LABEL_COMPONENT`) |
+| `cicd.pipeline.result` | execute | `Succeeded` condition mapped to the semconv `cicd.pipeline.result` enum (`success` / `failure` / `timeout` / `cancellation` / `error`) |
+| `delivery.tekton.dev.result_message` | execute | PipelineRun's `Succeeded` condition message on failure. Omitted on success; truncated to 1024 bytes (UTF-8 safe). |
+
+## Superseded Snapshot dedup
+
+When the snapshot controller skips auto-release because a newer Snapshot for the same Application has already been released, a single `waitDuration` span is emitted on the superseded Snapshot's trace context, anchored at the Snapshot's `CreationTimestamp` and ending at the `AutoReleased` condition's `LastTransitionTime`. It carries `cicd.pipeline.result=skip` and `delivery.tekton.dev.result_message="Released in newer Snapshot"` so the trace distinguishes the deliberate dedup from a broken chain.

e2e-tests/pkg/clients/gitlab/git.go

@@ -10,6 +10,40 @@ import (
 	"gitlab.com/gitlab-org/api/client-go/v2"
 )
 
+// GitLab pre-receive hooks surface 5xx wrapped in a 4xx; RetryTransport's status-code check misses it.
+func isGitlabTransientError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "Internal API error") ||
+		strings.Contains(msg, "504 Gateway Timeout")
+}
+
+func (gc *GitlabClient) createBranchWithRetry(projectID string, opt *gitlab.CreateBranchOptions) (*gitlab.Response, error) {
+	const maxAttempts = 5
+	const baseDelay = 2 * time.Second
+	var resp *gitlab.Response
+	var err error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		_, resp, err = gc.client.Branches.CreateBranch(projectID, opt)
+		if err == nil {
+			return resp, nil
+		}
+		if resp != nil && resp.StatusCode == http.StatusConflict {
+			return resp, err
+		}
+		if !isGitlabTransientError(err) || attempt == maxAttempts {
+			return resp, err
+		}
+		delay := baseDelay * time.Duration(1<<(attempt-1))
+		fmt.Printf("[gitlab-retry] CreateBranch attempt %d/%d: %v; retrying in %s\n",
+			attempt, maxAttempts, err, delay)
+		time.Sleep(delay)
+	}
+	return resp, err
+}
+
 // CreateBranch creates a new branch in a GitLab project with the given projectID and newBranchName
 func (gc *GitlabClient) CreateBranch(projectID, newBranchName, defaultBranch string) error {
 	// Prepare the branch creation request
@@ -19,7 +53,7 @@ func (gc *GitlabClient) CreateBranch(projectID, newBranchName, defaultBranch str
 	}
 
 	// Perform the branch creation
-	_, _, err := gc.client.Branches.CreateBranch(projectID, branchOpts)
+	_, err := gc.createBranchWithRetry(projectID, branchOpts)
 	if err != nil {
 		return fmt.Errorf("failed to create branch %s in project %s: %w", newBranchName, projectID, err)
 	}
@@ -77,7 +111,7 @@ func (gc *GitlabClient) CreateGitlabNewBranch(projectID, branchName, sha, baseBr
 		Branch: &branchName,
 		Ref:    &sha,
 	}
-	_, resp, err := gc.client.Branches.CreateBranch(projectID, opt)
+	resp, err := gc.createBranchWithRetry(projectID, opt)
 	if err != nil {
 		// Check if the error is due to the branch already existing
 		if resp != nil && resp.StatusCode == http.StatusConflict {

gitops/snapshot.go

@@ -251,6 +251,10 @@ const (
 	// SnapshotAutoReleasedCondition is the condition for marking if Snapshot was auto-released released with AppStudio.
 	SnapshotAutoReleasedCondition = "AutoReleased"
 
+	// SnapshotSupersededMessage is the AutoReleased condition message when
+	// an older Snapshot is skipped by a newer Snapshot for the same Application.
+	SnapshotSupersededMessage = "Released in newer Snapshot"
+
 	// SnapshotAddedToGlobalCandidateListCondition is the condition for marking if Snapshot's component was added to
 	// the global candidate list.
 	SnapshotAddedToGlobalCandidateListCondition = "AddedToGlobalCandidateList"

go.mod

@@ -47,6 +47,10 @@ require (
 	github.com/tektoncd/pipeline v1.7.0
 	github.com/tonglil/buflogr v1.1.1
 	gitlab.com/gitlab-org/api/client-go/v2 v2.36.0
+	go.opentelemetry.io/otel v1.40.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0
+	go.opentelemetry.io/otel/sdk v1.40.0
+	go.opentelemetry.io/otel/trace v1.40.0
 	go.uber.org/mock v0.6.0
 	go.uber.org/zap v1.27.1
 	golang.org/x/oauth2 v0.36.0
@@ -151,12 +155,8 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect
 	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect

internal/controller/buildpipeline/buildpipeline_adapter.go

@@ -34,6 +34,7 @@ import (
 	h "github.com/konflux-ci/integration-service/helpers"
 	"github.com/konflux-ci/integration-service/loader"
 	intgteststat "github.com/konflux-ci/integration-service/pkg/integrationteststatus"
+	"github.com/konflux-ci/integration-service/pkg/tracing"
 	"github.com/konflux-ci/integration-service/snapshot"
 	"github.com/konflux-ci/integration-service/status"
 	"github.com/konflux-ci/integration-service/tekton"
@@ -158,6 +159,7 @@ func (a *Adapter) EnsureSnapshotExists() (result controller.OperationResult, err
 	var canRemoveFinalizer bool
 
 	defer func() {
+		a.emitBuildTimingSpans()
 		// Don't write a failure annotation for transient Chains-not-signed errors
 		annotationErr := err
 		if h.IsChainsNotSignedError(err) {
@@ -261,6 +263,7 @@ func (a *Adapter) EnsureSnapshotExistsApplication() (result controller.Operation
 	var canRemoveFinalizer bool
 
 	defer func() {
+		a.emitBuildTimingSpans()
 		// Don't write a failure annotation for transient Chains-not-signed errors
 		annotationErr := err
 		if h.IsChainsNotSignedError(err) {
@@ -847,6 +850,11 @@ func (a *Adapter) prepareSnapshotForPipelineRun(pipelineRun *tektonv1.PipelineRu
 	prefixes := []string{gitops.BuildPipelineRunPrefix, gitops.TestLabelPrefix, gitops.CustomLabelPrefix, gitops.ReleaseLabelPrefix}
 	gitops.CopySnapshotLabelsAndAnnotations(&application.ObjectMeta, snapshot, a.component.Name, &pipelineRun.ObjectMeta, prefixes, true)
 
+	// Propagate span context from build PipelineRun to Snapshot for distributed tracing
+	if tp, found := pipelineRun.Annotations[tracing.SpanContextAnnotation]; found && tp != "" {
+		snapshot.Annotations[tracing.SpanContextAnnotation] = tp
+	}
+
 	snapshot.Labels[gitops.BuildPipelineRunNameLabel] = pipelineRun.Name
 	if pipelineRun.Status.CompletionTime != nil {
 		snapshot.Labels[gitops.BuildPipelineRunFinishTimeLabel] = strconv.FormatInt(pipelineRun.Status.CompletionTime.Unix(), 10)
@@ -1466,3 +1474,18 @@ func (a *Adapter) IsLatestBuildPipelineRunInComponentWithPRGroupHash(buildPlr *t
 	a.logger.Info(fmt.Sprintf("The build pipelineRun %s/%s with pr group %s is not the latest for its component, skipped", buildPlr.Namespace, buildPlr.Name, prGroupName))
 	return false, nil
 }
+
+// emitBuildTimingSpans emits timing spans for the build PipelineRun if not already emitted
+func (a *Adapter) emitBuildTimingSpans() {
+	spanContext := a.pipelineRun.Annotations[tracing.SpanContextAnnotation]
+	patched, err := tracing.EmitAndMarkTimingSpans(a.context, a.pipelineRun, spanContext, "", a.client, func() (*tektonv1.PipelineRun, error) {
+		return a.loader.GetPipelineRun(a.context, a.client, a.pipelineRun.Name, a.pipelineRun.Namespace)
+	})
+	if err != nil {
+		a.logger.Error(err, "Failed to emit and mark build timing spans")
+		return
+	}
+	if patched != nil {
+		a.pipelineRun = patched
+	}
+}

internal/controller/buildpipeline/buildpipeline_adapter_test.go

@@ -32,13 +32,16 @@ import (
 	"github.com/konflux-ci/integration-service/helpers"
 	"github.com/konflux-ci/integration-service/loader"
 	intgteststat "github.com/konflux-ci/integration-service/pkg/integrationteststatus"
+	"github.com/konflux-ci/integration-service/pkg/tracing"
 	"github.com/konflux-ci/integration-service/snapshot"
 	"github.com/konflux-ci/integration-service/status"
 	"github.com/konflux-ci/integration-service/tekton"
 	"github.com/konflux-ci/operator-toolkit/metadata"
 	"knative.dev/pkg/apis"
 	v1 "knative.dev/pkg/apis/duck/v1"
 
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"go.uber.org/mock/gomock"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -3841,6 +3844,95 @@ var _ = Describe("Pipeline Adapter", Ordered, func() {
 
 	})
 
+	When("emitBuildTimingSpans is called", func() {
+		BeforeEach(func() {
+			adapter = createAdapter()
+		})
+
+		It("does nothing when the PipelineRun already has the timingEmitted annotation", func() {
+			adapter.pipelineRun.Annotations[tracing.TimingEmittedAnnotation] = "true"
+			adapter.emitBuildTimingSpans()
+			Expect(adapter.pipelineRun.Annotations[tracing.TimingEmittedAnnotation]).To(Equal("true"))
+		})
+
+		It("does not annotate the PipelineRun when the global tracer provider is the noop", func() {
+			adapter.pipelineRun.Status.StartTime = &metav1.Time{Time: time.Now().Add(-time.Minute)}
+			adapter.pipelineRun.Status.CompletionTime = &metav1.Time{Time: time.Now()}
+			Expect(k8sClient.Status().Update(ctx, adapter.pipelineRun)).To(Succeed())
+
+			adapter.emitBuildTimingSpans()
+			Expect(adapter.pipelineRun.GetAnnotations()).NotTo(HaveKey(tracing.TimingEmittedAnnotation))
+		})
+
+		It("does not annotate the PipelineRun when start or completion time is missing", func() {
+			prev := otel.GetTracerProvider()
+			tp := sdktrace.NewTracerProvider()
+			otel.SetTracerProvider(tp)
+			DeferCleanup(func() {
+				otel.SetTracerProvider(prev)
+				_ = tp.Shutdown(ctx)
+			})
+
+			adapter.emitBuildTimingSpans()
+			Expect(adapter.pipelineRun.GetAnnotations()).NotTo(HaveKey(tracing.TimingEmittedAnnotation))
+		})
+
+		It("emits spans and patches the timingEmitted annotation when the run has completed", func() {
+			prev := otel.GetTracerProvider()
+			tp := sdktrace.NewTracerProvider()
+			otel.SetTracerProvider(tp)
+			DeferCleanup(func() {
+				otel.SetTracerProvider(prev)
+				_ = tp.Shutdown(ctx)
+			})
+
+			adapter.pipelineRun.Status.StartTime = &metav1.Time{Time: time.Now().Add(-time.Minute)}
+			adapter.pipelineRun.Status.CompletionTime = &metav1.Time{Time: time.Now()}
+			Expect(k8sClient.Status().Update(ctx, adapter.pipelineRun)).To(Succeed())
+			adapter.context = toolkit.GetMockedContext(ctx, []toolkit.MockData{
+				{
+					ContextKey: loader.GetPipelineRunContextKey,
+					Resource:   adapter.pipelineRun,
+				},
+			})
+
+			adapter.emitBuildTimingSpans()
+
+			Eventually(func(g Gomega) {
+				updated := &tektonv1.PipelineRun{}
+				g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: adapter.pipelineRun.Name, Namespace: adapter.pipelineRun.Namespace}, updated)).To(Succeed())
+				g.Expect(updated.GetAnnotations()).To(HaveKeyWithValue(tracing.TimingEmittedAnnotation, "true"))
+			}).Should(Succeed())
+		})
+
+		It("logs and returns without marking the run when the refetch errors, so reconciliation continues and the next pass retries", func() {
+			prev := otel.GetTracerProvider()
+			tp := sdktrace.NewTracerProvider()
+			otel.SetTracerProvider(tp)
+			DeferCleanup(func() {
+				otel.SetTracerProvider(prev)
+				_ = tp.Shutdown(ctx)
+			})
+
+			adapter.pipelineRun.Status.StartTime = &metav1.Time{Time: time.Now().Add(-time.Minute)}
+			adapter.pipelineRun.Status.CompletionTime = &metav1.Time{Time: time.Now()}
+			Expect(k8sClient.Status().Update(ctx, adapter.pipelineRun)).To(Succeed())
+			adapter.context = toolkit.GetMockedContext(ctx, []toolkit.MockData{
+				{
+					ContextKey: loader.GetPipelineRunContextKey,
+					Err:        errors.New("refetch boom"),
+				},
+			})
+
+			Expect(func() { adapter.emitBuildTimingSpans() }).NotTo(Panic())
+			Expect(adapter.pipelineRun.GetAnnotations()).NotTo(HaveKey(tracing.TimingEmittedAnnotation))
+
+			stored := &tektonv1.PipelineRun{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: adapter.pipelineRun.Name, Namespace: adapter.pipelineRun.Namespace}, stored)).To(Succeed())
+			Expect(stored.GetAnnotations()).NotTo(HaveKey(tracing.TimingEmittedAnnotation))
+		})
+	})
+
 	createAdapter = func() *Adapter {
 		adapter = NewAdapter(ctx, buildPipelineRun, hasComp, &[]v1beta2.ComponentGroup{*hasCompGroup}, logger, loader.NewMockLoader(), k8sClient)
 		return adapter