Merge branch 'main' into fix/s3-poller-backup-delete

Author: MichaelKatsoulis

.github/workflows/fragment-in-pr.yml

@@ -3,6 +3,9 @@ name: fragment-in-pr
 on:
   pull_request:
     types: [ labeled, unlabeled, opened, reopened, synchronize ]
+permissions:
+  contents: read
+
 jobs:
   fragments:
     if: "!contains(github.event.pull_request.labels.*.name, 'skip-changelog') && !contains(github.event.pull_request.labels.*.name, 'backport')"

.github/workflows/release-notes.yml

@@ -12,6 +12,9 @@ on:
         type: string
         required: true
 
+permissions:
+  contents: read
+
 jobs:
   build-release-notes:
     uses: elastic/elastic-agent-changelog-tool/.github/workflows/generate-release-notes.yml@main

.mergify.yml

@@ -357,3 +357,11 @@ pull_request_rules:
       backport:
         branches:
           - "9.4"
+  - name: backport patches to 9.5 branch
+    conditions:
+      - merged
+      - label=backport-9.5
+    actions:
+      backport:
+        branches:
+          - "9.5"

NOTICE.txt

@@ -17142,11 +17142,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gopacket
-Version: v1.1.20-0.20241002174017-e8c5fda595e6
+Version: v1.1.20-0.20260708143104-cfe2cc0f0cc7
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/gopacket@v1.1.20-0.20241002174017-e8c5fda595e6/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/gopacket@v1.1.20-0.20260708143104-cfe2cc0f0cc7/LICENSE:
 
 Copyright (c) 2012 Google, Inc. All rights reserved.
 Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved.

changelog/fragments/1783113269-fix-packetbeat-lazy-load-wpcap-dll.yaml

@@ -0,0 +1,50 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Load the Npcap wpcap.dll lazily to avoid blocking Npcap upgrades on Windows
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+description: >
+  On Windows, importing gopacket/pcap loaded wpcap.dll in the package init, so
+  every Beat that links Packetbeat's capture code held the DLL open even when it
+  never captured traffic. This stopped Packetbeat from replacing wpcap.dll while
+  upgrading Npcap and the install failed. The DLL is now loaded lazily, only when
+  Packetbeat needs it.
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: packetbeat
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/14517

changelog/fragments/1783461968-filebeat-sync-run-and-shutdown-functions.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Synchronize Filebeat Run and Shutdown functions. 
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: filebeat
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/beats/pull/51800
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

filebeat/beater/filebeat.go

@@ -83,6 +83,7 @@ type Filebeat struct {
 	pipeline                 beat.PipelineConnector
 	logger                   *logp.Logger
 	otelStatusFactoryWrapper func(cfgfile.RunnerFactory) cfgfile.RunnerFactory
+	runReady                 *closeOnce
 }
 
 type PluginFactory func(beat.Info, statestore.States) []v2.Plugin
@@ -174,6 +175,7 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
 
 	fb := &Filebeat{
 		done:           make(chan struct{}),
+		runReady:       &closeOnce{ch: make(chan struct{})},
 		config:         &config,
 		moduleRegistry: moduleRegistry,
 		pluginFactory:  plugins,
@@ -270,6 +272,9 @@ func (fb *Filebeat) loadModulesPipelines(b *beat.Beat) error {
 func (fb *Filebeat) Run(b *beat.Beat) error {
 	var err error
 	config := fb.config
+	// Close runReady so that Shutdown doesn't have to wait no
+	// matter how we exit from Run.
+	defer fb.runReady.Close()
 
 	if b.Manager != nil {
 		b.Manager.RegisterDiagnosticHook("input_metrics", "Metrics from active inputs.",
@@ -530,6 +535,8 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 
 	// Add done channel to wait for shutdown signal
 	waitFinished.AddChan(fb.done)
+	// Safe for Shutdown to be called
+	fb.runReady.Close()
 	waitFinished.Wait()
 
 	// Stop reloadable lists, autodiscover -> Stop crawler -> stop inputs -> stop harvesters.
@@ -588,9 +595,24 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 
 // Stop is called on exit to stop the crawling, spooling and registration processes.
 func (fb *Filebeat) Stop() {
+	fb.StopWithContext(context.Background())
+}
+
+// StopWithContext is like Stop but respects ctx when waiting for Run to reach
+// its ready state, so the caller's deadline is not consumed by the wait.
+func (fb *Filebeat) StopWithContext(ctx context.Context) {
 	fb.logger.Info("Stopping filebeat")
 
-	// Stop Filebeat
+	// Wait for Run to reach waitFinished.Wait() before closing done, so that
+	// Stop is never delivered before the beater is ready to handle it.
+	select {
+	case <-fb.runReady.ch:
+	case <-ctx.Done():
+		fb.logger.Warn("Context cancelled waiting for Run to reach ready state; stopping anyway")
+	case <-time.After(5 * time.Second):
+		fb.logger.Warn("Timed out waiting for Run to reach ready state; stopping anyway")
+	}
+
 	fb.stopOnce.Do(func() { close(fb.done) })
 }
 
@@ -605,3 +627,14 @@ func newPipelineLoaderFactory(ctx context.Context, esConfig *conf.C, info beat.I
 	}
 	return pipelineLoaderFactory
 }
+
+type closeOnce struct {
+	ch   chan struct{}
+	once sync.Once
+}
+
+func (coc *closeOnce) Close() {
+	coc.once.Do(func() {
+		close(coc.ch)
+	})
+}

filebeat/beater/stop_test.go

@@ -0,0 +1,61 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package beater
+
+import (
+	"testing"
+	"time"
+
+	"github.com/elastic/elastic-agent-libs/logp"
+)
+
+// TestStopWaitsForRunReady proves that Stop does not close the done channel
+// until Run has closed runReady (i.e., reached waitFinished.Wait).
+// This guards against a race where the OTel collector calls Shutdown before
+// the beat's Run goroutine has initialised its shutdown-signal machinery.
+func TestStopWaitsForRunReady(t *testing.T) {
+	fb := &Filebeat{
+		done:     make(chan struct{}),
+		runReady: &closeOnce{ch: make(chan struct{})},
+		logger:   logp.NewNopLogger(),
+	}
+
+	stopDone := make(chan struct{})
+	go func() {
+		defer close(stopDone)
+		fb.Stop()
+	}()
+
+	// done must still be open: Stop is waiting for runReady to be closed.
+	select {
+	case <-fb.done:
+		t.Fatal("Stop closed done before Run closed runReady")
+	default:
+	}
+
+	// Simulate Run() reaching the waitFinished.Wait() call.
+	fb.runReady.Close()
+
+	select {
+	case <-fb.done:
+	case <-time.After(500 * time.Millisecond):
+		t.Fatal("Stop did not close done after runReady was closed")
+	}
+
+	<-stopDone
+}