Merge branch 'main' into log-go-be-container

changelog/fragments/1727271745-resource-limits-kubernetes.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Increase Elastic Agent memory requests and limits when deployed in Kubernetes.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/5614
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

deploy/kubernetes/Makefile

@@ -91,8 +91,7 @@ else
 		--label automation \
 		--label release_note:skip \
 		--base main \
-		--head $(ELASTIC_AGENT_BRANCH) \
-		--reviewer elastic/obs-cloudnative-monitoring
+		--head $(ELASTIC_AGENT_BRANCH)
 endif
 
 else

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml

@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml

@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/base/elastic-agent-managed-daemonset.yaml

@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-managed/extra/elastic-agent-managed-statefulset.yaml

@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-daemonset.yaml

@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/extra/elastic-agent-standalone-statefulset.yaml

@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

deploy/kubernetes/elastic-agent-managed-kubernetes.yaml

@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

deploy/kubernetes/elastic-agent-managed/elastic-agent-managed-daemonset.yaml

@@ -83,10 +83,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -759,10 +759,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml

@@ -90,10 +90,10 @@ spec:
             #    - SYS_ADMIN
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

dev-tools/kubernetes/base/elastic-agent-managed/daemonset.yaml

@@ -38,10 +38,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: proc
               mountPath: /hostfs/proc

dev-tools/kubernetes/base/elastic-agent-standalone/daemonset.yaml

@@ -54,10 +54,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

dev-tools/mage/dockerbuilder.go

@@ -46,7 +46,7 @@ func (b *dockerBuilder) Build() error {
 	}
 
 	if err := b.copyFiles(); err != nil {
-		return err
+		return fmt.Errorf("error copying files for docker variant %q: %w", b.DockerVariant, err)
 	}
 
 	if err := b.prepareBuild(); err != nil {

docs/manifests/elastic-agent-managed-gke-autopilot.yaml

@@ -73,7 +73,7 @@ spec:
           resources:
             limits:
               #cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios
-              memory: 700Mi
+              memory: 1Gi
               ephemeral-storage: "500Mi"
           volumeMounts:
             - name: varlog

docs/manifests/elastic-agent-standalone-gke-autopilot.yaml

@@ -384,7 +384,7 @@ spec:
           resources:
             limits:
               # cpu: 200m # Keep this commented. We dont set the cpu limit to avoid scheduling problems of agent in autopilot scenarios
-              memory: 700Mi
+              memory: 1Gi
               ephemeral-storage: "500Mi"
           volumeMounts:
             - name: datastreams

docs/manifests/kustomize-autosharding/elastic-agent-standalone-kubernetes-side-leader.yaml

@@ -602,10 +602,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

docs/manifests/kustomize-autosharding/elastic-agent-standalone-statefulset-side-ksm.yaml

@@ -409,10 +409,10 @@ spec:
             runAsUser: 0
           resources:
             limits:
-              memory: 700Mi
+              memory: 1Gi
             requests:
               cpu: 100m
-              memory: 400Mi
+              memory: 500Mi
           volumeMounts:
             - name: datastreams
               mountPath: /etc/elastic-agent/agent.yml

docs/test-framework-dev-guide.md

@@ -12,9 +12,16 @@ provides a high level overview of the testing framework.
 
 ### Dependencies
 
-Go version should be at least the same than the one in [.go-version](https://github.com/elastic/elastic-agent/blob/main/.go-version) file at the root of this repository
+#### Go version
+Go version should be at least the same than the one in [.go-version](https://github.com/elastic/elastic-agent/blob/main/.go-version) file at the root of this repository.
 
-[GCloud CLI](https://cloud.google.com/sdk/gcloud)
+
+### GCloud CLI
+The integration testing framework spins up resources in GCP.  To achieve this, it needs the
+[GCloud CLI](https://cloud.google.com/sdk/gcloud) to be installed on the system where the tests are initiated from.
+
+### Beats
+The Elastic Agent package that is used for integration tests packages Beats built from the Unified Release (as opposed to DRA).  There is no explicit action needed for this prerequisite but just keep in mind that if any Agent integration tests rely on certain Beats features or bugfixes, they may not be available in the integration tests yet because a unified release containing those features or bugfixes may not have happened yet.
 
 ### Configuration
 

magefile.go

@@ -1537,7 +1537,7 @@ func downloadBinary(ctx context.Context, project string, packageName string, bin
 		}
 
 		compl.Add(1)
-		fmt.Printf("Done downloading %s\n", packageName)
+		fmt.Printf("Done downloading %s into %s\n", packageName, targetPath)
 		return nil
 	}
 }
@@ -1590,7 +1590,8 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa
 			if mg.Verbose() {
 				log.Printf("--- Evaluating moving dependency %s to archive path %s\n", f, archivePath)
 			}
-			if !strings.Contains(f, packageSuffix) && !isPythonWheelPackage(f, packageVersion) {
+			// if the matched file name does not contain the platform suffix and it's not a platform-independent package, skip it
+			if !strings.Contains(f, packageSuffix) && !isPlatformIndependentPackage(f, packageVersion) {
 				if mg.Verbose() {
 					log.Printf("--- Skipped moving dependency %s to archive path\n", f)
 				}
@@ -1613,9 +1614,18 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa
 			if err := os.MkdirAll(targetDir, 0750); err != nil {
 				fmt.Printf("warning: failed to create directory %s: %s", targetDir, err)
 			}
-			if err := os.Rename(f, targetPath); err != nil {
-				panic(fmt.Errorf("failed renaming file: %w", err))
+
+			// Platform-independent packages need to be placed in the archive sub-folders for all platforms, copy instead of moving
+			if isPlatformIndependentPackage(f, packageVersion) {
+				if err := copyFile(f, targetPath); err != nil {
+					panic(fmt.Errorf("failed copying file: %w", err))
+				}
+			} else {
+				if err := os.Rename(f, targetPath); err != nil {
+					panic(fmt.Errorf("failed renaming file: %w", err))
+				}
 			}
+
 			if mg.Verbose() {
 				log.Printf("--- Moved dependency in archive path %s => %s\n", f, targetPath)
 			}
@@ -1625,10 +1635,37 @@ func movePackagesToArchive(dropPath string, platformPackageSuffixes []string, pa
 	return archivePath
 }
 
-func isPythonWheelPackage(f string, packageVersion string) bool {
+func copyFile(src, dst string) error {
+	srcStat, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("stat source file %q: %w", src, err)
+	}
+
+	srcF, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("opening source file %q: %w", src, err)
+	}
+	defer srcF.Close()
+
+	dstF, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, srcStat.Mode()|os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("opening/creating destination file %q: %w", dst, err)
+	}
+	defer dstF.Close()
+
+	_, err = io.Copy(dstF, srcF)
+	if err != nil {
+		return fmt.Errorf("copying file %q to %q: %w", src, dst, err)
+	}
+
+	return nil
+}
+
+func isPlatformIndependentPackage(f string, packageVersion string) bool {
 	fileBaseName := filepath.Base(f)
 	for _, spec := range manifest.ExpectedBinaries {
 		packageName := spec.GetPackageName(packageVersion, "")
+		// as of now only python wheels packages are platform-independent
 		if spec.PythonWheel && (fileBaseName == packageName || fileBaseName == packageName+sha512FileExt) {
 			return true
 		}

testing/integration/apm_propagation_test.go

@@ -55,9 +55,6 @@ func TestAPMConfig(t *testing.T) {
 		Group: Default,
 		Stack: &define.Stack{},
 	})
-
-	t.Skip("https://github.com/elastic/elastic-agent/issues/5624; apm-server not working correctly")
-
 	f, err := define.NewFixtureFromLocalBuild(t, define.Version())
 	require.NoError(t, err)