Add root trace span for build command and fix S3 cache timeout

- Add leeway.command span that wraps entire build execution including
  workspace loading and cache operations
- Replace ListObjects with parallel HeadObject calls in ExistingPackages
  to fix timeout on buckets with millions of objects
- Add Bytes field to DownloadResult to track artifact sizes
- Add debug logging for cache operations

Co-authored-by: Ona <no-reply@ona.com>

Author: WVerlaek

cmd/build.go

@@ -21,7 +21,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/otel"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // CleanupFunc is a function that performs cleanup operations and must be deferred
@@ -58,12 +60,35 @@ Examples:
 	},
 }
 
-func build(cmd *cobra.Command, args []string) error {
+func build(cmd *cobra.Command, args []string) (buildErr error) {
+	// Initialize tracer early to capture full command execution time
+	commandCtx, commandSpan, tracerShutdown := initCommandTracer(cmd)
+	defer func() {
+		// Set span status based on build result before shutdown
+		if commandSpan != nil {
+			if buildErr != nil {
+				commandSpan.RecordError(buildErr)
+				commandSpan.SetStatus(codes.Error, buildErr.Error())
+			} else {
+				commandSpan.SetStatus(codes.Ok, "command completed successfully")
+			}
+		}
+		tracerShutdown()
+	}()
+
 	_, pkg, _, _ := getTarget(args, false)
 	if pkg == nil {
 		return errors.New("build needs a package")
 	}
-	opts, localCache, shutdown := getBuildOpts(cmd)
+
+	// Add target package info to command span
+	if commandSpan != nil {
+		commandSpan.SetAttributes(
+			attribute.String("leeway.target.package", pkg.FullName()),
+		)
+	}
+
+	opts, localCache, shutdown := getBuildOpts(cmd, commandCtx)
 	defer shutdown()
 
 	var (
@@ -197,6 +222,71 @@ func saveBuildResult(ctx context.Context, loc string, localCache cache.LocalCach
 	return nil
 }
 
+// initCommandTracer initializes the OpenTelemetry tracer and creates a root span
+// for the entire command execution. Returns the context with the span, the span itself,
+// and a shutdown function that must be deferred.
+func initCommandTracer(cmd *cobra.Command) (context.Context, trace.Span, CleanupFunc) {
+	otelEndpoint, err := cmd.Flags().GetString("otel-endpoint")
+	if err != nil || otelEndpoint == "" {
+		return context.Background(), nil, func() {}
+	}
+
+	// Set leeway version for telemetry
+	telemetry.SetLeewayVersion(leeway.Version)
+
+	// Get insecure flag
+	otelInsecure, _ := cmd.Flags().GetBool("otel-insecure")
+
+	// Initialize tracer with the provided endpoint and TLS configuration
+	tp, err := telemetry.InitTracer(context.Background(), otelEndpoint, otelInsecure)
+	if err != nil {
+		log.WithError(err).Warn("failed to initialize OpenTelemetry tracer")
+		return context.Background(), nil, func() {}
+	}
+
+	// Parse trace context if provided
+	traceParent, _ := cmd.Flags().GetString("trace-parent")
+	traceState, _ := cmd.Flags().GetString("trace-state")
+
+	parentCtx := context.Background()
+	if traceParent != "" {
+		if err := telemetry.ValidateTraceParent(traceParent); err != nil {
+			log.WithError(err).Warn("invalid trace-parent format")
+		} else {
+			ctx, err := telemetry.ParseTraceContext(traceParent, traceState)
+			if err != nil {
+				log.WithError(err).Warn("failed to parse trace context")
+			} else {
+				parentCtx = ctx
+			}
+		}
+	}
+
+	// Create root span for the entire command execution
+	tracer := otel.Tracer("leeway")
+	ctx, span := tracer.Start(parentCtx, "leeway.command",
+		trace.WithSpanKind(trace.SpanKindInternal),
+	)
+
+	// Add command attributes
+	span.SetAttributes(
+		attribute.String("leeway.version", leeway.Version),
+		attribute.String("leeway.command", "build"),
+	)
+
+	// Create shutdown function
+	shutdown := func() {
+		span.End()
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := telemetry.Shutdown(shutdownCtx, tp); err != nil {
+			log.WithError(err).Warn("failed to shutdown tracer provider")
+		}
+	}
+
+	return ctx, span, shutdown
+}
+
 func init() {
 	rootCmd.AddCommand(buildCmd)
 
@@ -246,7 +336,7 @@ func addBuildFlags(cmd *cobra.Command) {
 	cmd.Flags().String("trace-state", os.Getenv("TRACESTATE"), "W3C Trace Context tracestate header for distributed tracing (defaults to $TRACESTATE)")
 }
 
-func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, CleanupFunc) {
+func getBuildOpts(cmd *cobra.Command, commandCtx context.Context) ([]leeway.BuildOption, cache.LocalCache, CleanupFunc) {
 	// Track if user explicitly set LEEWAY_DOCKER_EXPORT_TO_CACHE before workspace loading.
 	// This allows us to distinguish:
 	// - User set explicitly: High priority (overrides package config)
@@ -347,59 +437,12 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
 		reporter = append(reporter, leeway.NewGitHubReporter())
 	}
 
-	// Initialize OpenTelemetry reporter if endpoint is configured
-	var tracerProvider *sdktrace.TracerProvider
-	var otelShutdown func()
-	if otelEndpoint, err := cmd.Flags().GetString("otel-endpoint"); err != nil {
-		log.Fatal(err)
-	} else if otelEndpoint != "" {
-		// Set leeway version for telemetry
-		telemetry.SetLeewayVersion(leeway.Version)
-
-		// Get insecure flag
-		otelInsecure, err := cmd.Flags().GetBool("otel-insecure")
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		// Initialize tracer with the provided endpoint and TLS configuration
-		tp, err := telemetry.InitTracer(context.Background(), otelEndpoint, otelInsecure)
-		if err != nil {
-			log.WithError(err).Warn("failed to initialize OpenTelemetry tracer")
-		} else {
-			tracerProvider = tp
-
-			// Parse trace context if provided
-			traceParent, _ := cmd.Flags().GetString("trace-parent")
-			traceState, _ := cmd.Flags().GetString("trace-state")
-
-			parentCtx := context.Background()
-			if traceParent != "" {
-				if err := telemetry.ValidateTraceParent(traceParent); err != nil {
-					log.WithError(err).Warn("invalid trace-parent format")
-				} else {
-					ctx, err := telemetry.ParseTraceContext(traceParent, traceState)
-					if err != nil {
-						log.WithError(err).Warn("failed to parse trace context")
-					} else {
-						parentCtx = ctx
-					}
-				}
-			}
-
-			// Create OTel reporter
-			tracer := otel.Tracer("leeway")
-			reporter = append(reporter, leeway.NewOTelReporter(tracer, parentCtx))
-
-			// Create shutdown function
-			otelShutdown = func() {
-				shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-				defer cancel()
-				if err := telemetry.Shutdown(shutdownCtx, tracerProvider); err != nil {
-					log.WithError(err).Warn("failed to shutdown tracer provider")
-				}
-			}
-		}
+	// Add OpenTelemetry reporter if tracer was initialized (commandCtx has a span)
+	// The tracer is initialized in initCommandTracer() which creates the root leeway.command span.
+	// The OTelReporter will create leeway.build as a child of leeway.command.
+	if otelEndpoint, _ := cmd.Flags().GetString("otel-endpoint"); otelEndpoint != "" {
+		tracer := otel.Tracer("leeway")
+		reporter = append(reporter, leeway.NewOTelReporter(tracer, commandCtx))
 	}
 
 	dontTest, err := cmd.Flags().GetBool("dont-test")
@@ -465,10 +508,8 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
 		dockerExportSet = true
 	}
 
-	// Create a no-op shutdown function if otelShutdown is nil
-	if otelShutdown == nil {
-		otelShutdown = func() {}
-	}
+	// Return a no-op cleanup function since tracer shutdown is handled by initCommandTracer
+	noopCleanup := func() {}
 
 	return []leeway.BuildOption{
 		leeway.WithLocalCache(localCache),
@@ -488,7 +529,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
 		leeway.WithInFlightChecksums(inFlightChecksums),
 		leeway.WithDockerExportToCache(dockerExportToCache, dockerExportSet),
 		leeway.WithDockerExportEnv(dockerExportEnvValue, dockerExportEnvSet),
-	}, localCache, otelShutdown
+	}, localCache, noopCleanup
 }
 
 type pushOnlyRemoteCache struct {

cmd/build_test.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"os"
 	"testing"
 
@@ -242,7 +243,7 @@ func TestGetBuildOptsWithInFlightChecksums(t *testing.T) {
 			}
 
 			// Test getBuildOpts function
-			opts, localCache, _ := getBuildOpts(cmd)
+			opts, localCache, _ := getBuildOpts(cmd, context.Background())
 
 			// We can't directly test the WithInFlightChecksums option since it's internal,
 			// but we can verify the function doesn't error and returns options

cmd/provenance-assert.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"io"
@@ -125,7 +126,7 @@ func getProvenanceTarget(cmd *cobra.Command, args []string) (bundleFN, pkgFN str
 			log.Fatal("provenance export requires a package")
 		}
 
-		_, cache, _ := getBuildOpts(cmd)
+		_, cache, _ := getBuildOpts(cmd, context.Background())
 
 		var ok bool
 		pkgFN, ok = cache.Location(pkg)

cmd/run.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 
 	log "github.com/sirupsen/logrus"
@@ -27,7 +28,7 @@ Should any of the scripts fail Leeway will exit with an exit code of 1 once all
 				if script == nil {
 					return errors.New("run needs a script")
 				}
-				opts, _, _ := getBuildOpts(cmd)
+				opts, _, _ := getBuildOpts(cmd, context.Background())
 				return script.Run(opts...)
 			})
 		}

cmd/sbom-export.go

@@ -32,7 +32,7 @@ If no package is specified, the workspace's default target is used.`,
 		}
 
 		// Get build options and cache
-		_, localCache, _ := getBuildOpts(cmd)
+		_, localCache, _ := getBuildOpts(cmd, context.Background())
 
 		// Get output format and file
 		format, _ := cmd.Flags().GetString("format")

cmd/sbom-scan.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"os"
 
 	"github.com/gitpod-io/leeway/pkg/leeway"
@@ -30,7 +31,7 @@ If no package is specified, the workspace's default target is used.`,
 		}
 
 		// Get cache
-		_, localCache, _ := getBuildOpts(cmd)
+		_, localCache, _ := getBuildOpts(cmd, context.Background())
 
 		// Get output directory
 		outputDir, _ := cmd.Flags().GetString("output-dir")