Skip to content

Add root trace span for build command and fix S3 cache timeout #326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
WVerlaek merged 2 commits into from
Jan 15, 2026
Merged

Add root trace span for build command and fix S3 cache timeout #326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e26c752
Add root trace span for build command and fix S3 cache timeout
WVerlaek Jan 15, 2026
864e621
go fmt
WVerlaek Jan 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 14 additions & 72 deletions cmd/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,9 @@ import (
"github.com/gookit/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"go.opentelemetry.io/otel"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/attribute"
)

// CleanupFunc is a function that performs cleanup operations and must be deferred
type CleanupFunc func()

// buildCmd represents the build command
var buildCmd = &cobra.Command{
Use: "build [targetPackage]",
Expand Down Expand Up @@ -58,13 +54,18 @@ Examples:
},
}

func build(cmd *cobra.Command, args []string) error {
func build(cmd *cobra.Command, args []string) (buildErr error) {
_, span := telemetry.StartSpan(cmd.Context(), "leeway.build")
defer telemetry.FinishSpan(span, &buildErr)

_, pkg, _, _ := getTarget(args, false)
if pkg == nil {
return errors.New("build needs a package")
}
opts, localCache, shutdown := getBuildOpts(cmd)
defer shutdown()

span.SetAttributes(attribute.String("leeway.target.package", pkg.FullName()))

opts, localCache := getBuildOpts(cmd)

var (
watch, _ = cmd.Flags().GetBool("watch")
Expand Down Expand Up @@ -240,13 +241,9 @@ func addBuildFlags(cmd *cobra.Command) {
cmd.Flags().Bool("report-github", os.Getenv("GITHUB_OUTPUT") != "", "Report package build success/failure to GitHub Actions using the GITHUB_OUTPUT environment variable")
cmd.Flags().Bool("fixed-build-dir", true, "Use a fixed build directory for each package, instead of based on the package version, to better utilize caches based on absolute paths (defaults to true)")
cmd.Flags().Bool("docker-export-to-cache", false, "Export Docker images to cache instead of pushing directly (enables SLSA L3 compliance)")
cmd.Flags().String("otel-endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), "OpenTelemetry OTLP endpoint URL for tracing (defaults to $OTEL_EXPORTER_OTLP_ENDPOINT)")
cmd.Flags().Bool("otel-insecure", os.Getenv("OTEL_EXPORTER_OTLP_INSECURE") == "true", "Disable TLS for OTLP endpoint (for local development only, defaults to $OTEL_EXPORTER_OTLP_INSECURE)")
cmd.Flags().String("trace-parent", os.Getenv("TRACEPARENT"), "W3C Trace Context traceparent header for distributed tracing (defaults to $TRACEPARENT)")
cmd.Flags().String("trace-state", os.Getenv("TRACESTATE"), "W3C Trace Context tracestate header for distributed tracing (defaults to $TRACESTATE)")
}

func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, CleanupFunc) {
func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) {
// Track if user explicitly set LEEWAY_DOCKER_EXPORT_TO_CACHE before workspace loading.
// This allows us to distinguish:
// - User set explicitly: High priority (overrides package config)
Expand Down Expand Up @@ -347,59 +344,9 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
reporter = append(reporter, leeway.NewGitHubReporter())
}

// Initialize OpenTelemetry reporter if endpoint is configured
var tracerProvider *sdktrace.TracerProvider
var otelShutdown func()
if otelEndpoint, err := cmd.Flags().GetString("otel-endpoint"); err != nil {
log.Fatal(err)
} else if otelEndpoint != "" {
// Set leeway version for telemetry
telemetry.SetLeewayVersion(leeway.Version)

// Get insecure flag
otelInsecure, err := cmd.Flags().GetBool("otel-insecure")
if err != nil {
log.Fatal(err)
}

// Initialize tracer with the provided endpoint and TLS configuration
tp, err := telemetry.InitTracer(context.Background(), otelEndpoint, otelInsecure)
if err != nil {
log.WithError(err).Warn("failed to initialize OpenTelemetry tracer")
} else {
tracerProvider = tp

// Parse trace context if provided
traceParent, _ := cmd.Flags().GetString("trace-parent")
traceState, _ := cmd.Flags().GetString("trace-state")

parentCtx := context.Background()
if traceParent != "" {
if err := telemetry.ValidateTraceParent(traceParent); err != nil {
log.WithError(err).Warn("invalid trace-parent format")
} else {
ctx, err := telemetry.ParseTraceContext(traceParent, traceState)
if err != nil {
log.WithError(err).Warn("failed to parse trace context")
} else {
parentCtx = ctx
}
}
}

// Create OTel reporter
tracer := otel.Tracer("leeway")
reporter = append(reporter, leeway.NewOTelReporter(tracer, parentCtx))

// Create shutdown function
otelShutdown = func() {
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := telemetry.Shutdown(shutdownCtx, tracerProvider); err != nil {
log.WithError(err).Warn("failed to shutdown tracer provider")
}
}
}
// Add OpenTelemetry reporter if tracing is enabled
if telemetry.Enabled() {
reporter = append(reporter, leeway.NewOTelReporter(telemetry.Tracer(), cmd.Context()))
}

dontTest, err := cmd.Flags().GetBool("dont-test")
Expand Down Expand Up @@ -465,11 +412,6 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
dockerExportSet = true
}

// Create a no-op shutdown function if otelShutdown is nil
if otelShutdown == nil {
otelShutdown = func() {}
}

return []leeway.BuildOption{
leeway.WithLocalCache(localCache),
leeway.WithRemoteCache(remoteCache),
Expand All @@ -488,7 +430,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache, C
leeway.WithInFlightChecksums(inFlightChecksums),
leeway.WithDockerExportToCache(dockerExportToCache, dockerExportSet),
leeway.WithDockerExportEnv(dockerExportEnvValue, dockerExportEnvSet),
}, localCache, otelShutdown
}, localCache
}

type pushOnlyRemoteCache struct {
Expand Down
2 changes: 1 addition & 1 deletion cmd/build_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ func TestGetBuildOptsWithInFlightChecksums(t *testing.T) {
}

// Test getBuildOpts function
opts, localCache, _ := getBuildOpts(cmd)
opts, localCache := getBuildOpts(cmd)

// We can't directly test the WithInFlightChecksums option since it's internal,
// but we can verify the function doesn't error and returns options
Expand Down
2 changes: 1 addition & 1 deletion cmd/provenance-assert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func getProvenanceTarget(cmd *cobra.Command, args []string) (bundleFN, pkgFN str
log.Fatal("provenance export requires a package")
}

_, cache, _ := getBuildOpts(cmd)
_, cache := getBuildOpts(cmd)

var ok bool
pkgFN, ok = cache.Location(pkg)
Expand Down
52 changes: 52 additions & 0 deletions cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,12 @@ import (
"github.com/gookit/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"go.opentelemetry.io/otel/attribute"
otelTrace "go.opentelemetry.io/otel/trace"
"golang.org/x/xerrors"

"github.com/gitpod-io/leeway/pkg/leeway"
"github.com/gitpod-io/leeway/pkg/leeway/telemetry"
)

const (
Expand Down Expand Up @@ -95,6 +98,9 @@ var (
buildArgs []string
verbose bool
variant string

// commandSpan is the root span for the current command execution
commandSpan otelTrace.Span
)

// rootCmd represents the base command when called without any subcommands
Expand Down Expand Up @@ -134,6 +140,46 @@ variables have an effect on leeway:
if verbose {
log.SetLevel(log.DebugLevel)
}

// Initialize OpenTelemetry tracing if endpoint is configured
otelEndpoint, _ := cmd.Flags().GetString("otel-endpoint")
if otelEndpoint != "" {
telemetry.SetLeewayVersion(leeway.Version)

otelInsecure, _ := cmd.Flags().GetBool("otel-insecure")
if err := telemetry.Initialize(cmd.Context(), otelEndpoint, otelInsecure); err != nil {
log.WithError(err).Warn("failed to initialize OpenTelemetry tracer")
} else {
// Parse trace context if provided
traceParent, _ := cmd.Flags().GetString("trace-parent")
traceState, _ := cmd.Flags().GetString("trace-state")

parentCtx := cmd.Context()
if traceParent != "" {
if err := telemetry.ValidateTraceParent(traceParent); err != nil {
log.WithError(err).Warn("invalid trace-parent format")
} else if ctx, err := telemetry.ParseTraceContext(traceParent, traceState); err != nil {
log.WithError(err).Warn("failed to parse trace context")
} else {
parentCtx = ctx
}
}

// Create command span and update command context
var ctx context.Context
ctx, commandSpan = telemetry.StartSpan(parentCtx, "leeway.command",
attribute.String("leeway.version", leeway.Version),
attribute.String("leeway.command", cmd.Name()),
)
cmd.SetContext(ctx)
}
Comment on lines +145 to +175
Copy link
Copy Markdown
Member Author

@WVerlaek WVerlaek Jan 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

extracted from build.go to root.go, so we trace all leeway commands by default and creates a span that wraps the full command duration. We were missing e.g. remote cache lookup time before

}
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
telemetry.FinishSpan(commandSpan, nil)
if err := telemetry.Shutdown(context.Background()); err != nil {
log.WithError(err).Warn("failed to shutdown tracer provider")
}
},
BashCompletionFunction: bashCompletionFunc,
}
Expand Down Expand Up @@ -183,6 +229,12 @@ func init() {
rootCmd.PersistentFlags().StringVar(&variant, "variant", "", "selects a package variant")
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enables verbose logging")
rootCmd.PersistentFlags().Bool("dut", false, "used for testing only - doesn't actually do anything")

// OpenTelemetry tracing flags
rootCmd.PersistentFlags().String("otel-endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), "OpenTelemetry OTLP endpoint URL for tracing (defaults to $OTEL_EXPORTER_OTLP_ENDPOINT)")
rootCmd.PersistentFlags().Bool("otel-insecure", os.Getenv("OTEL_EXPORTER_OTLP_INSECURE") == "true", "Disable TLS for OTLP endpoint (for local development only, defaults to $OTEL_EXPORTER_OTLP_INSECURE)")
rootCmd.PersistentFlags().String("trace-parent", os.Getenv("TRACEPARENT"), "W3C Trace Context traceparent header for distributed tracing (defaults to $TRACEPARENT)")
rootCmd.PersistentFlags().String("trace-state", os.Getenv("TRACESTATE"), "W3C Trace Context tracestate header for distributed tracing (defaults to $TRACESTATE)")
}

func getWorkspace() (leeway.Workspace, error) {
Expand Down
2 changes: 1 addition & 1 deletion cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ Should any of the scripts fail Leeway will exit with an exit code of 1 once all
if script == nil {
return errors.New("run needs a script")
}
opts, _, _ := getBuildOpts(cmd)
opts, _ := getBuildOpts(cmd)
return script.Run(opts...)
})
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/sbom-export.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ If no package is specified, the workspace's default target is used.`,
}

// Get build options and cache
_, localCache, _ := getBuildOpts(cmd)
_, localCache := getBuildOpts(cmd)

// Get output format and file
format, _ := cmd.Flags().GetString("format")
Expand Down
2 changes: 1 addition & 1 deletion cmd/sbom-scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ If no package is specified, the workspace's default target is used.`,
}

// Get cache
_, localCache, _ := getBuildOpts(cmd)
_, localCache := getBuildOpts(cmd)

// Get output directory
outputDir, _ := cmd.Flags().GetString("output-dir")
Expand Down
Loading
Loading