Merge branch 'main' into log-go-be-container

.buildkite/integration.pipeline.yml

@@ -106,7 +106,7 @@ steps:
     key: "k8s-integration-tests"
     env:
       K8S_VERSION: "v1.31.0"
-      KIND_VERSION: "v0.20.0"
+      KIND_VERSION: "v0.24.0"
     command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
     artifact_paths:
       - "build/k8s-logs*/*"

.buildkite/pipeline.yml

@@ -184,7 +184,7 @@ steps:
       - label: "K8s tests: {{matrix.k8s_version}}"
         env:
           K8S_VERSION: "v{{matrix.k8s_version}}"
-          KIND_VERSION: "v0.20.0"
+          KIND_VERSION: "v0.24.0"
         command: ".buildkite/scripts/steps/k8s-tests.sh"
         agents:
           provider: "gcp"

.github/workflows/bump-golang.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@6b8881a17fc8038e884ec94ff72a49e8e8a4069f # v0.76.1
+        uses: updatecli/updatecli-action@92a13b95c2cd9f1c6742c965509203c6a5635ed7 # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli-bump-golang.yml --values .ci/updatecli/values.d/scm.yml

.golangci.yml

@@ -112,17 +112,15 @@ linters-settings:
     # Forbid any `replace` directives that are intended temporarily only during
     # development. The modules listed below are intended to be replaced permanently.
     replace-allow-list:
-      - github.com/Microsoft/go-winio
       - github.com/Shopify/sarama
       - github.com/dop251/goja
       - github.com/dop251/goja_nodejs
       - github.com/fsnotify/fsnotify
       - github.com/openshift/api
-      - github.com/tonistiigi/fifo
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
 
   nolintlint:
     # Enable to ensure that nolint directives are all used. Default is true.
@@ -138,17 +136,17 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
     checks: ["all"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
 
   gosec:
     excludes:

Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.22.7
+ARG GO_VERSION=1.22.8
 FROM circleci/golang:${GO_VERSION}
 
 

Dockerfile.skaffold

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.22.7
+ARG GO_VERSION=1.22.8
 ARG crossbuild_image="docker.elastic.co/beats-dev/golang-crossbuild"
 ARG AGENT_VERSION=8.9.0-SNAPSHOT
 ARG AGENT_IMAGE="docker.elastic.co/beats/elastic-agent"

changelog/fragments/1723675050-Call-fleet-server-audit-unenroll-endpoint-on-uninstall.yaml

@@ -0,0 +1,35 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Call fleet-server audit/unenroll endpoint on uninstall
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+   Uninstalling a fleet-managed elastic-agent instance will now do a
+   best-effort attempt to notify fleet-server of the agent removal so the
+   agent should not appear as offline.
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/5302
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/484

changelog/fragments/1727453779-inspect-command-will-redact-secret_paths-in-policy.yaml

@@ -0,0 +1,34 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: inspect command will redact secret_paths in policy
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  The elastic-agent inspect command will now redact any secret values when displaying output.
+  The keys that are redacted are expected to be defined in the "secret_paths" attribute.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1727778754-update-base-ubuntu-from-20.04-to-24.04.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Update Ubuntu base image from 20.04 to 24.04
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1727994385-Stop-elastic-agent-inspect-from-printing-output-twice.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Stop elastic-agent inspect from printing output twice
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/5692
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/4471

deploy/helm/elastic-agent/README.md

@@ -67,7 +67,7 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations
 | kubernetes.namespace | string | `"default"` | kubernetes namespace |
 | kubernetes.hints.enabled | bool | `false` | enable [elastic-agent autodiscovery](https://www.elastic.co/guide/en/fleet/current/elastic-agent-kubernetes-autodiscovery.html) feature |
 | kubernetes.state.enabled | bool | `true` | integration global switch to enable state streams based on kube-state-metrics. Note that setting this to `false` results in overriding and *disabling all* the respective state streams |
-| kubernetes.state.deployKSM | bool | `true` | deploy kube-state-metrics service as a sidecar container to the elastic agent of `ksmShared` preset. If set to `false`, kube-state-metrics will *not* get deployed and `clusterWide` agent preset will be used for collecting kube-state-metrics. |
+| kubernetes.state.deployKSM | bool | `true` | deploy kube-state-metrics service as a sidecar container to the elastic agent of `ksmSharded` preset. If set to `false`, kube-state-metrics will *not* get deployed and `clusterWide` agent preset will be used for collecting kube-state-metrics. |
 | kubernetes.state.host | string | `"kube-state-metrics:8080"` | host of the kube-state-metrics service. Note that this used only when `deployKSM` is set to `false`. |
 | kubernetes.state.vars | object | `{}` | state streams variables such as `add_metadata`, `hosts`, `period`, `bearer_token_file`. Please note that colliding vars also defined in respective state streams will *not* be overridden. |
 | kubernetes.metrics.enabled | bool | `true` | integration global switch to enable metric streams based on kubelet. Note that setting this to false results in overriding and *disabling all* the respective metric streams |
@@ -135,7 +135,7 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations
 | agent.image | object | `{"pullPolicy":"IfNotPresent","repository":"docker.elastic.co/beats/elastic-agent","tag":"9.0.0-SNAPSHOT"}` | image configuration |
 | agent.engine | string | `"k8s"` | generate kubernetes manifests or [ECK](https://github.com/elastic/cloud-on-k8s) CRDs |
 | agent.unprivileged | bool | `false` | enable unprivileged mode |
-| agent.presets | map[string]{} | `{ "perNode" : {...}, "clusterWider": {...}, "ksmShared": {...} }` | Map of deployment presets for the Elastic Agent. The key of the map is the name of the preset. See more for the presets required by the built-in Kubernetes integration [here](./values.yaml) |
+| agent.presets | map[string]{} | `{ "perNode" : {...}, "clusterWide": {...}, "ksmSharded": {...} }` | Map of deployment presets for the Elastic Agent. The key of the map is the name of the preset. See more for the presets required by the built-in Kubernetes integration [here](./values.yaml) |
 
 ### 3.1 - Elastic-Agent Managed Configuration
 | Key | Type | Default | Description |

deploy/helm/elastic-agent/values.yaml

@@ -336,7 +336,7 @@ agent:
   # required by the built-in Kubernetes integration [here](./values.yaml)
   # @notationType -- map[string]{}
   # @section -- 3 - Elastic-Agent Configuration
-  # @default -- `{ "perNode" : {...}, "clusterWider": {...}, "ksmShared": {...} }`
+  # @default -- `{ "perNode" : {...}, "clusterWide": {...}, "ksmSharded": {...} }`
   presets:
     #  examplePreset:
     #    mode: [deployment, statefulset, daemonset]

dev-tools/mage/pkgtypes.go

@@ -10,6 +10,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"fmt"
+	"hash/fnv"
 	"io"
 	"io/fs"
 	"log"
@@ -24,7 +25,7 @@ import (
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
-	"github.com/mitchellh/hashstructure"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -500,12 +501,15 @@ func copyInstallScript(spec PackageSpec, script string, local *string) error {
 }
 
 func (s PackageSpec) hash() string {
-	h, err := hashstructure.Hash(s, nil)
+	out, err := yaml.Marshal(s)
 	if err != nil {
-		panic(fmt.Errorf("failed to compute hash of spec: %w", err))
+		panic(fmt.Errorf("failed to marshal spec: %w", err))
 	}
 
-	hash := strconv.FormatUint(h, 10)
+	h := fnv.New64()
+	h.Write(out)
+
+	hash := strconv.FormatUint(h.Sum64(), 10)
 	if len(hash) > 10 {
 		hash = hash[0:10]
 	}

dev-tools/packaging/packages.yml

@@ -225,11 +225,11 @@ shared:
   - &docker_ubuntu_spec
     docker_variant: 'basic'
     extra_vars:
-      from: '--platform=linux/amd64 ubuntu:20.04'
+      from: '--platform=linux/amd64 ubuntu:24.04'
   - &docker_ubuntu_arm_spec
     docker_variant: 'basic'
     extra_vars:
-      from: '--platform=linux/arm64 ubuntu:20.04'
+      from: '--platform=linux/arm64 ubuntu:24.04'
 
   - &docker_ubi_spec
     docker_variant: 'ubi'

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -74,6 +74,9 @@ RUN for iter in {1..10}; do microdnf update -y && microdnf install -y tar gzip f
 {{- end }}
 
 {{- if contains .from "ubuntu" }}
+# Delete the existing "ubuntu" user to prevent conflict with the user that is added later.
+# See https://bugs.launchpad.net/cloud-images/+bug/2005129/comments/2.
+RUN touch /var/mail/ubuntu && chown ubuntu /var/mail/ubuntu && userdel -r ubuntu
 RUN for iter in {1..10}; do \
         apt-get update -y && \
         DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl gawk xz-utils && \

docs/elastic-agent-logging.md

@@ -77,5 +77,5 @@ receives.
 
 ## Total footprint
 Given the two log outputs and their default log rotation policies, the
-Elastic-Agent needs about 210Mb (20Mb x 7 + 10Mb + 7 = 210Mb) of disk
+Elastic-Agent needs about 210Mb (20Mb x 7 + 10Mb x 7 = 210Mb) of disk
 for logging.