Correctly handle unix signals in EDOT (#10908)

* Handle unix signals in EDOT

* Add changelog entry

* Fix changelog entry

* Move removing agent config file to fixture.ConfigureOtel

Author: swiatekm

changelog/fragments/1761917853-fix-otel-signal-handling.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix signal handling for the EDOT Collector
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/agent/cmd/otel.go

@@ -119,7 +119,9 @@ func RunCollector(cmdCtx context.Context, configFiles []string, supervised bool,
 	}
 
 	defer cancel()
-	go service.ProcessWindowsControlEvents(stopCollector)
+	if settings.otelSettings.DisableGracefulShutdown { // TODO: Harmonize these settings
+		service.HandleSignals(stopCollector, cancel)
+	}
 
 	return otel.Run(ctx, stop, settings.otelSettings)
 }

pkg/testing/fixture.go

@@ -277,6 +277,23 @@ func (f *Fixture) Configure(ctx context.Context, yamlConfig []byte) error {
 	return os.WriteFile(cfgFilePath, yamlConfig, 0600)
 }
 
+// ConfigureOtel replaces the default Agent otel mode configuration file with the provided
+// configuration. This must be called after `Prepare` is called.
+func (f *Fixture) ConfigureOtel(ctx context.Context, yamlConfig []byte) error {
+	err := f.EnsurePrepared(ctx)
+	if err != nil {
+		return err
+	}
+
+	// remove elastic-agent.yml, otel should be independent
+	if removeErr := os.Remove(filepath.Join(f.WorkDir(), "elastic-agent.yml")); removeErr != nil && !os.IsNotExist(removeErr) {
+		return removeErr
+	}
+
+	cfgFilePath := filepath.Join(f.workDir, "otel.yml")
+	return os.WriteFile(cfgFilePath, yamlConfig, 0600)
+}
+
 // SetUninstallToken sets uninstall token
 func (f *Fixture) SetUninstallToken(uninstallToken string) {
 	f.uninstallToken = uninstallToken

testing/integration/ess/otel_test.go

@@ -28,8 +28,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/elastic/go-elasticsearch/v8"
-
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-libs/testing/estools"
 	"github.com/elastic/elastic-agent-libs/transport/tlscommontest"
@@ -38,6 +36,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/tools/testcontext"
 	"github.com/elastic/elastic-agent/testing/integration"
+	"github.com/elastic/go-elasticsearch/v8"
 )
 
 const apmProcessingContent = `2023-06-19 05:20:50 ERROR This is a test error message
@@ -87,6 +86,64 @@ service:
         - debug
         - otlp/elastic`
 
+func TestOtelStartShutdown(t *testing.T) {
+	define.Require(t, define.Requirements{
+		Group: integration.Default,
+		Local: true,
+		OS: []define.OS{
+			{Type: define.Linux},
+			{Type: define.Darwin},
+		},
+	})
+
+	otelConfig := `receivers:
+  nop:
+exporters:
+  nop:
+service:
+  pipelines:
+    logs:
+      receivers:
+        - nop
+      exporters:
+        - nop
+`
+	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+	require.NoError(t, err)
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
+	err = fixture.Prepare(ctx)
+	require.NoError(t, err)
+
+	err = fixture.ConfigureOtel(t.Context(), []byte(otelConfig))
+	require.NoError(t, err)
+
+	cmd, err := fixture.PrepareAgentCommand(ctx, []string{"otel"})
+	require.NoError(t, err)
+
+	output := strings.Builder{}
+	cmd.Stderr = &output
+	cmd.Stdout = &output
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Log("Elastic-Agent output:")
+			t.Log(output.String())
+		}
+	})
+
+	require.NoError(t, cmd.Start(), "could not start otel collector")
+	require.EventuallyWithT(t, func(collect *assert.CollectT) {
+		assert.Contains(collect, output.String(), "Everything is ready")
+	}, time.Second*30, time.Second)
+
+	// stop the collector and check that it emitted logs indicating a graceful shutdown
+	require.NoError(t, cmd.Process.Signal(os.Interrupt))
+	require.NoError(t, cmd.Wait())
+	assert.Contains(t, output.String(), "Shutdown complete")
+}
+
 func TestOtelFileProcessing(t *testing.T) {
 	define.Require(t, define.Requirements{
 		Group: integration.Default,