Add root trace span for build command and fix S3 cache timeout

- Add leeway.command span that wraps entire build execution including
  workspace loading and cache operations
- Refactor telemetry package with singleton pattern and span utilities
  (Initialize, Shutdown, Enabled, Tracer, StartSpan, FinishSpan)
- Move OTel tracing initialization to root command for reuse across commands
- Replace ListObjects with parallel HeadObject calls in ExistingPackages
  to fix timeout on buckets with millions of objects
- Add tracing spans for cache download operations with size attributes
- Add String() method to DownloadStatus for cleaner span attributes

Co-authored-by: Ona <no-reply@ona.com>

Author: WVerlaek

cmd/build.go

@@ -20,12 +20,11 @@ import (
 	"github.com/gookit/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"go.opentelemetry.io/otel"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )
 
-// CleanupFunc is a function that performs cleanup operations and must be deferred
-type CleanupFunc func()
+
 
 // buildCmd represents the build command
 var buildCmd = &cobra.Command{
@@ -58,13 +57,35 @@ Examples:
 	},
 }
 
-func build(cmd *cobra.Command, args []string) error {
+func build(cmd *cobra.Command, args []string) (buildErr error) {
+	// Create command span if tracing is enabled
+	commandCtx := cmd.Context()
+	if telemetry.Enabled() {
+		parentCtx := rootSpanCtx
+		if parentCtx == nil {
+			parentCtx = cmd.Context()
+		}
+		var span trace.Span
+		commandCtx, span = telemetry.StartSpan(parentCtx, "leeway.command",
+			attribute.String("leeway.version", leeway.Version),
+			attribute.String("leeway.command", "build"),
+		)
+		defer telemetry.FinishSpan(span, &buildErr)
+	}
+
 	_, pkg, _, _ := getTarget(args, false)
 	if pkg == nil {
 		return errors.New("build needs a package")
 	}
-	opts, localCache, shutdown := getBuildOpts(cmd)
-	defer shutdown()
+
+	// Add target package info to command span
+	if telemetry.Enabled() {
+		if span := trace.SpanFromContext(commandCtx); span.IsRecording() {
+			span.SetAttributes(attribute.String("leeway.target.package", pkg.FullName()))
+		}
+	}
+
+	opts, localCache := getBuildOpts(cmd, commandCtx)
 
 	var (
 		watch, _ = cmd.Flags().GetBool("watch")
@@ -240,13 +261,9 @@ func addBuildFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("report-github", os.Getenv("GITHUB_OUTPUT") != "", "Report package build success/failure to GitHub Actions using the GITHUB_OUTPUT environment variable")
 	cmd.Flags().Bool("fixed-build-dir", true, "Use a fixed build directory for each package, instead of based on the package version, to better utilize caches based on absolute paths (defaults to true)")
 	cmd.Flags().Bool("docker-export-to-cache", false, "Export Docker images to cache instead of pushing directly (enables SLSA L3 compliance)")
-	cmd.Flags().String("otel-endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), "OpenTelemetry OTLP endpoint URL for tracing (defaults to $OTEL_EXPORTER_OTLP_ENDPOINT)")
-	cmd.Flags().Bool("otel-insecure", os.Getenv("OTEL_EXPORTER_OTLP_INSECURE") == "true", "Disable TLS for OTLP endpoint (for local development only, defaults to $OTEL_EXPORTER_OTLP_INSECURE)")
-	cmd.Flags().String("trace-parent", os.Getenv("TRACEPARENT"), "W3C Trace Context traceparent header for distributed tracing (defaults to $TRACEPARENT)")
-	cmd.Flags().String("trace-state", os.Getenv("TRACESTATE"), "W3C Trace Context tracestate header for distributed tracing (defaults to $TRACESTATE)")
 }
 
-func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, CleanupFunc) {
+func getBuildOpts(cmd *cobra.Command, commandCtx context.Context) ([]leeway.BuildOption, cache.LocalCache) {
 	// Track if user explicitly set LEEWAY_DOCKER_EXPORT_TO_CACHE before workspace loading.
 	// This allows us to distinguish:
 	// - User set explicitly: High priority (overrides package config)
@@ -347,59 +364,9 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
 		reporter = append(reporter, leeway.NewGitHubReporter())
 	}
 
-	// Initialize OpenTelemetry reporter if endpoint is configured
-	var tracerProvider *sdktrace.TracerProvider
-	var otelShutdown func()
-	if otelEndpoint, err := cmd.Flags().GetString("otel-endpoint"); err != nil {
-		log.Fatal(err)
-	} else if otelEndpoint != "" {
-		// Set leeway version for telemetry
-		telemetry.SetLeewayVersion(leeway.Version)
-
-		// Get insecure flag
-		otelInsecure, err := cmd.Flags().GetBool("otel-insecure")
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		// Initialize tracer with the provided endpoint and TLS configuration
-		tp, err := telemetry.InitTracer(context.Background(), otelEndpoint, otelInsecure)
-		if err != nil {
-			log.WithError(err).Warn("failed to initialize OpenTelemetry tracer")
-		} else {
-			tracerProvider = tp
-
-			// Parse trace context if provided
-			traceParent, _ := cmd.Flags().GetString("trace-parent")
-			traceState, _ := cmd.Flags().GetString("trace-state")
-
-			parentCtx := context.Background()
-			if traceParent != "" {
-				if err := telemetry.ValidateTraceParent(traceParent); err != nil {
-					log.WithError(err).Warn("invalid trace-parent format")
-				} else {
-					ctx, err := telemetry.ParseTraceContext(traceParent, traceState)
-					if err != nil {
-						log.WithError(err).Warn("failed to parse trace context")
-					} else {
-						parentCtx = ctx
-					}
-				}
-			}
-
-			// Create OTel reporter
-			tracer := otel.Tracer("leeway")
-			reporter = append(reporter, leeway.NewOTelReporter(tracer, parentCtx))
-
-			// Create shutdown function
-			otelShutdown = func() {
-				shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-				defer cancel()
-				if err := telemetry.Shutdown(shutdownCtx, tracerProvider); err != nil {
-					log.WithError(err).Warn("failed to shutdown tracer provider")
-				}
-			}
-		}
+	// Add OpenTelemetry reporter if tracing is enabled
+	if telemetry.Enabled() {
+		reporter = append(reporter, leeway.NewOTelReporter(telemetry.Tracer(), commandCtx))
 	}
 
 	dontTest, err := cmd.Flags().GetBool("dont-test")
@@ -465,11 +432,6 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
 		dockerExportSet = true
 	}
 
-	// Create a no-op shutdown function if otelShutdown is nil
-	if otelShutdown == nil {
-		otelShutdown = func() {}
-	}
-
 	return []leeway.BuildOption{
 		leeway.WithLocalCache(localCache),
 		leeway.WithRemoteCache(remoteCache),
@@ -488,7 +450,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
 		leeway.WithInFlightChecksums(inFlightChecksums),
 		leeway.WithDockerExportToCache(dockerExportToCache, dockerExportSet),
 		leeway.WithDockerExportEnv(dockerExportEnvValue, dockerExportEnvSet),
-	}, localCache, otelShutdown
+	}, localCache
 }
 
 type pushOnlyRemoteCache struct {
@@ -621,6 +583,10 @@ func getRemoteCache(cmd *cobra.Command) cache.RemoteCache {
 			if err != nil {
 				log.Fatalf("cannot access remote S3 cache: %v", err)
 			}
+			// Set tracer if tracing is enabled
+			if telemetry.Enabled() {
+				rc.SetTracer(telemetry.Tracer())
+			}
 			return rc
 		default:
 			if slsaConfig != nil && slsaConfig.Verification {

cmd/build_test.go

@@ -242,7 +242,7 @@ func TestGetBuildOptsWithInFlightChecksums(t *testing.T) {
 			}
 
 			// Test getBuildOpts function
-			opts, localCache, _ := getBuildOpts(cmd)
+			opts, localCache := getBuildOpts(cmd, cmd.Context())
 
 			// We can't directly test the WithInFlightChecksums option since it's internal,
 			// but we can verify the function doesn't error and returns options

cmd/provenance-assert.go

@@ -125,7 +125,7 @@ func getProvenanceTarget(cmd *cobra.Command, args []string) (bundleFN, pkgFN str
 			log.Fatal("provenance export requires a package")
 		}
 
-		_, cache, _ := getBuildOpts(cmd)
+		_, cache := getBuildOpts(cmd, cmd.Context())
 
 		var ok bool
 		pkgFN, ok = cache.Location(pkg)

cmd/root.go

@@ -13,6 +13,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/gitpod-io/leeway/pkg/leeway"
+	"github.com/gitpod-io/leeway/pkg/leeway/telemetry"
 )
 
 const (
@@ -95,6 +96,9 @@ var (
 	buildArgs []string
 	verbose   bool
 	variant   string
+
+	// rootSpanCtx holds the context with parent trace information (if provided)
+	rootSpanCtx context.Context
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -134,6 +138,37 @@ variables have an effect on leeway:
 		if verbose {
 			log.SetLevel(log.DebugLevel)
 		}
+
+		// Initialize OpenTelemetry tracing if endpoint is configured
+		otelEndpoint, _ := cmd.Flags().GetString("otel-endpoint")
+		if otelEndpoint != "" {
+			telemetry.SetLeewayVersion(leeway.Version)
+
+			otelInsecure, _ := cmd.Flags().GetBool("otel-insecure")
+			if err := telemetry.Initialize(cmd.Context(), otelEndpoint, otelInsecure); err != nil {
+				log.WithError(err).Warn("failed to initialize OpenTelemetry tracer")
+			} else {
+				// Parse trace context if provided
+				traceParent, _ := cmd.Flags().GetString("trace-parent")
+				traceState, _ := cmd.Flags().GetString("trace-state")
+
+				rootSpanCtx = cmd.Context()
+				if traceParent != "" {
+					if err := telemetry.ValidateTraceParent(traceParent); err != nil {
+						log.WithError(err).Warn("invalid trace-parent format")
+					} else if ctx, err := telemetry.ParseTraceContext(traceParent, traceState); err != nil {
+						log.WithError(err).Warn("failed to parse trace context")
+					} else {
+						rootSpanCtx = ctx
+					}
+				}
+			}
+		}
+	},
+	PersistentPostRun: func(cmd *cobra.Command, args []string) {
+		if err := telemetry.Shutdown(context.Background()); err != nil {
+			log.WithError(err).Warn("failed to shutdown tracer provider")
+		}
 	},
 	BashCompletionFunction: bashCompletionFunc,
 }
@@ -183,6 +218,12 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&variant, "variant", "", "selects a package variant")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enables verbose logging")
 	rootCmd.PersistentFlags().Bool("dut", false, "used for testing only - doesn't actually do anything")
+
+	// OpenTelemetry tracing flags
+	rootCmd.PersistentFlags().String("otel-endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), "OpenTelemetry OTLP endpoint URL for tracing (defaults to $OTEL_EXPORTER_OTLP_ENDPOINT)")
+	rootCmd.PersistentFlags().Bool("otel-insecure", os.Getenv("OTEL_EXPORTER_OTLP_INSECURE") == "true", "Disable TLS for OTLP endpoint (for local development only, defaults to $OTEL_EXPORTER_OTLP_INSECURE)")
+	rootCmd.PersistentFlags().String("trace-parent", os.Getenv("TRACEPARENT"), "W3C Trace Context traceparent header for distributed tracing (defaults to $TRACEPARENT)")
+	rootCmd.PersistentFlags().String("trace-state", os.Getenv("TRACESTATE"), "W3C Trace Context tracestate header for distributed tracing (defaults to $TRACESTATE)")
 }
 
 func getWorkspace() (leeway.Workspace, error) {

cmd/run.go

@@ -27,7 +27,7 @@ Should any of the scripts fail Leeway will exit with an exit code of 1 once all
 				if script == nil {
 					return errors.New("run needs a script")
 				}
-				opts, _, _ := getBuildOpts(cmd)
+				opts, _ := getBuildOpts(cmd, cmd.Context())
 				return script.Run(opts...)
 			})
 		}

cmd/sbom-export.go

@@ -32,7 +32,7 @@ If no package is specified, the workspace's default target is used.`,
 		}
 
 		// Get build options and cache
-		_, localCache, _ := getBuildOpts(cmd)
+		_, localCache := getBuildOpts(cmd, cmd.Context())
 
 		// Get output format and file
 		format, _ := cmd.Flags().GetString("format")

cmd/sbom-scan.go

@@ -30,7 +30,7 @@ If no package is specified, the workspace's default target is used.`,
 		}
 
 		// Get cache
-		_, localCache, _ := getBuildOpts(cmd)
+		_, localCache := getBuildOpts(cmd, cmd.Context())
 
 		// Get output directory
 		outputDir, _ := cmd.Flags().GetString("output-dir")