[9.2] (backport #11237) Add OTEL_COMPONENT=true to packaging to allow building a seperate OTEL EDOT binary (#11417)

* Add `OTEL_COMPONENT=true` to packaging to allow building a seperate OTEL EDOT binary (#11237)

* Add ability to build seperate edot binary. Add build flag to change build.

* Get this working.

* Fix crossBuild log.

* Fix imports.

* fixes from code review.

* Revert the change for packageAgent.

* Update binary name.

* Fix docker container. Add CI steps for build.

* Change to serial deps.

* Try not using go list.

* Revert "Change to serial deps."

This reverts commit 329de79.

* Fix BK to build windows OTEL_COMPONENT.

(cherry picked from commit 9f1c464)

* Change to SerialDeps.

---------

Co-authored-by: Blake Rouse <[email protected]>

Author: mergify[bot]

.buildkite/integration.pipeline.yml

@@ -39,6 +39,26 @@ steps:
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2204_X86_64}"
 
+      # Temporary build step to verify building the elastic-agent works.
+      # This will be removed once OTEL_COMPONENT is removed, because it
+      # became the default.
+      - label: ":package: amd64: OTEL_COMPONENT zip,tar.gz,rpm,deb "
+        key: packaging-amd64-otel-component
+        env:
+          PLATFORMS: "windows/amd64,linux/amd64"
+          PACKAGES: "zip,tar.gz,rpm,deb"
+          OTEL_COMPONENT: "true"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          image: "${IMAGE_UBUNTU_2204_X86_64}"
+
       - label: ":package: amd64: FIPS tar.gz"
         key: "packaging-amd64-fips"
         env:
@@ -73,6 +93,26 @@ steps:
           instanceType: "c6g.2xlarge"
           image: "${IMAGE_UBUNTU_2204_ARM_64}"
 
+      # Temporary build step to verify building the elastic-agent works.
+      # This will be removed once OTEL_COMPONENT is removed, because it
+      # became the default.
+      - label: ":package: arm64: OTEL_COMPONENT zip,tar.gz"
+        key: packaging-arm64-otel-component
+        env:
+          PLATFORMS: "windows/arm64,linux/arm64"
+          PACKAGES: "tar.gz,zip"
+          OTEL_COMPONENT: "true"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "aws"
+          instanceType: "c6g.2xlarge"
+          image: "${IMAGE_UBUNTU_2204_ARM_64}"
+
       - label: ":package: arm64: FIPS tar.gz"
         key: "packaging-arm64-fips"
         env:
@@ -158,6 +198,40 @@ steps:
           diskSizeGb: 200
           image: "${IMAGE_UBUNTU_2204_ARM_64}"
 
+      - label: ":package: amd64: OTEL_COMPONENT Containers"
+        key: packaging-containers-amd64-otel-component
+        env:
+          PACKAGES: "docker"
+          PLATFORMS: "linux/amd64"
+          OTEL_COMPONENT: "true"
+        command: |
+          .buildkite/scripts/steps/integration-package.sh
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          diskSizeGb: 200
+          image: "${IMAGE_UBUNTU_2204_X86_64}"
+        plugins:
+          - *vault_docker_login
+
+      - label: ":package: arm64: OTEL_COMPONENT Containers"
+        key: packaging-containers-arm64-otel-component
+        env:
+          PACKAGES: "docker"
+          PLATFORMS: "linux/arm64"
+          OTEL_COMPONENT: "true"
+        command: |
+          .buildkite/scripts/steps/integration-package.sh
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "aws"
+          instanceType: "c6g.2xlarge"
+          diskSizeGb: 200
+          image: "${IMAGE_UBUNTU_2204_ARM_64}"
+
   - label: "Triggering Integration tests"
     command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
 

dev-tools/mage/build.go

@@ -108,6 +108,10 @@ func DefaultBuildArgs() BuildArgs {
 		}
 	}
 
+	if OTELComponentBuild {
+		args.ExtraFlags = append(args.ExtraFlags, "-tags=otelexternal")
+	}
+
 	if DevBuild {
 		// Disable optimizations (-N) and inlining (-l) for debugging.
 		args.ExtraFlags = append(args.ExtraFlags, `-gcflags=all=-N -l`)

dev-tools/mage/crossbuild.go

@@ -74,6 +74,13 @@ type CrossBuildOption func(params *crossBuildParams)
 // ImageSelectorFunc returns the name of the builder image.
 type ImageSelectorFunc func(platform string) (string, error)
 
+// WithName adjust the name of the cross build action.
+func WithName(name string) CrossBuildOption {
+	return func(params *crossBuildParams) {
+		params.Name = name
+	}
+}
+
 // ForPlatforms filters the platforms based on the given expression.
 func ForPlatforms(expr string) func(params *crossBuildParams) {
 	return func(params *crossBuildParams) {
@@ -123,6 +130,7 @@ func AddPlatforms(expressions ...string) func(params *crossBuildParams) {
 }
 
 type crossBuildParams struct {
+	Name          string
 	Platforms     BuildPlatformList
 	Target        string
 	Serial        bool
@@ -132,11 +140,12 @@ type crossBuildParams struct {
 
 // CrossBuild executes a given build target once for each target platform.
 func CrossBuild(options ...CrossBuildOption) error {
-	fmt.Println("--- CrossBuild Elastic-Agent")
 	params := crossBuildParams{Platforms: Platforms, Target: defaultCrossBuildTarget, ImageSelector: CrossBuildImage}
+	params.Name = "Elastic-Agent"
 	for _, opt := range options {
 		opt(&params)
 	}
+	fmt.Printf("--- CrossBuild %s\n", params.Name)
 
 	if len(params.Platforms) == 0 {
 		log.Printf("Skipping cross-build of target=%v because platforms list is empty.", params.Target)
@@ -178,7 +187,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 	// Build the magefile for Linux, so we can run it inside the container.
 	mg.Deps(buildMage)
 
-	log.Println("crossBuild: Platform list =", params.Platforms)
+	log.Printf("crossBuild(%s): Platform list = %s\n", params.Name, params.Platforms)
 	var deps []interface{}
 	for _, buildPlatform := range params.Platforms {
 		if !buildPlatform.Flags.CanCrossBuild() {
@@ -364,6 +373,7 @@ func (b GolangCrossBuilder) Build() error {
 		"--env", fmt.Sprintf("DEV=%v", DevBuild),
 		"--env", fmt.Sprintf("EXTERNAL=%v", ExternalBuild),
 		"--env", fmt.Sprintf("FIPS=%v", FIPSBuild),
+		"--env", fmt.Sprintf("OTEL_COMPONENT=%v", OTELComponentBuild),
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,

dev-tools/mage/gotest.go

@@ -88,6 +88,9 @@ func testTagsFromEnv() []string {
 	if FIPSBuild {
 		tags = append(tags, "requirefips", "ms_tls13kdf")
 	}
+	if OTELComponentBuild {
+		tags = append(tags, "otelexternal")
+	}
 	return tags
 }
 

dev-tools/mage/settings.go

@@ -89,10 +89,11 @@ var (
 
 	BeatProjectType ProjectType
 
-	Snapshot      bool
-	DevBuild      bool
-	ExternalBuild bool
-	FIPSBuild     bool
+	Snapshot           bool
+	DevBuild           bool
+	ExternalBuild      bool
+	FIPSBuild          bool
+	OTELComponentBuild bool
 
 	versionQualified bool
 	versionQualifier string
@@ -164,6 +165,11 @@ func initGlobals() {
 		panic(fmt.Errorf("failed to parse FIPS env value: %w", err))
 	}
 
+	OTELComponentBuild, err = strconv.ParseBool(EnvOr("OTEL_COMPONENT", "false"))
+	if err != nil {
+		panic(fmt.Errorf("failed to parse OTEL_COMPONENT env value: %w", err))
+	}
+
 	versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
 
 	// order matters: this will override some of the values. Those values can be used
@@ -230,6 +236,7 @@ func varMap(args ...map[string]interface{}) map[string]interface{} {
 		"DEV":             DevBuild,
 		"EXTERNAL":        ExternalBuild,
 		"FIPS":            FIPSBuild,
+		"OTEL_COMPONENT":  OTELComponentBuild,
 		"Qualifier":       versionQualifier,
 		"CI":              CI,
 	}

dev-tools/packaging/packages.yml

@@ -369,6 +369,10 @@ shared:
     'data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}':
       source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
       mode: 0755
+    'data/{{.BeatName}}-{{ commit_short }}/elastic-otel-collector{{.BinaryExt}}':
+      source: build/golang-crossbuild/elastic-otel-collector-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+      mode: 0755
+      skip_on_missing: true
     'data/{{.BeatName}}-{{ commit_short }}/package.version':
       content: >
         {{ agent_package_version }}
@@ -390,6 +394,10 @@ shared:
     'data/{{.BeatName}}-{{ commit_short }}/elastic-agent.app/Contents/MacOS/{{.BeatName}}{{.BinaryExt}}':
       source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
       mode: 0755
+    'data/{{.BeatName}}-{{ commit_short }}/elastic-otel-collector{{.BinaryExt}}':
+      source: build/golang-crossbuild/elastic-otel-collector-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+      mode: 0755
+      skip_on_missing: true
     'data/{{.BeatName}}-{{ commit_short }}/package.version':
       content: >
         {{ agent_package_version }}

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -44,6 +44,7 @@ RUN true && \
     (chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/pf-elastic-collector || true) && \
     (chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/pf-elastic-symbolizer || true) && \
     (chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/pf-host-agent || true) && \
+    (chmod 0755 {{ $beatHome }}/data/elastic-agent-*/elastic-otel-collector || true) && \
     (chmod 0755 {{ $beatHome }}/data/elastic-agent-*/otelcol || true) && \
     (chmod 0755 {{ $beatHome }}/otelcol || true) && \
     find {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/components -name "*.yml*" -type f -exec chmod 0644 {} \; && \

internal/edot/cmd/otel.go

@@ -70,9 +70,10 @@ func NewOtelCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Comman
 		SilenceErrors: true,
 	}
 
+	origHelp := cmd.HelpFunc()
 	cmd.SetHelpFunc(func(c *cobra.Command, s []string) {
 		hideInheritedFlags(c)
-		c.Root().HelpFunc()(c, s)
+		origHelp(c, s)
 	})
 
 	SetupOtelFlags(cmd.Flags())

internal/edot/cmd/validate.go

@@ -30,9 +30,10 @@ func newValidateCommandWithArgs(_ []string, _ *cli.IOStreams) *cobra.Command {
 	}
 
 	SetupOtelFlags(cmd.Flags())
+	origHelpFunc := cmd.HelpFunc()
 	cmd.SetHelpFunc(func(c *cobra.Command, s []string) {
 		hideInheritedFlags(c)
-		c.Root().HelpFunc()(c, s)
+		origHelpFunc(c, s)
 	})
 
 	return cmd

internal/edot/main.go

@@ -0,0 +1,22 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	edotCmd "github.com/elastic/elastic-agent/internal/edot/cmd"
+	"github.com/elastic/elastic-agent/internal/pkg/cli"
+)
+
+func main() {
+	cmd := edotCmd.NewOtelCommandWithArgs(os.Args, cli.NewIOStreams())
+	err := cmd.Execute()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}