Merge branch 'main' into package-cloud-defend

Author: mjwolf

.buildkite/pipeline.yml

@@ -348,6 +348,19 @@ steps:
         - .buildkite/scripts/steps/beats_tests.sh
         - .buildkite/hooks/pre-command
 
+  # NOTE: This should help detecting issues earlier in the development cycle
+  # See https://github.com/elastic/elastic-agent/issues/11604
+  - label: "Trigger Elastic Agent Package"
+    if: build.pull_request.id != null
+    commands:
+      - .buildkite/scripts/steps/trigger-elastic-agent-package.sh
+      - .buildkite/scripts/steps/trigger-elastic-agent-package.sh | buildkite-agent pipeline upload
+    if_changed:
+      include:
+        - .buildkite/pipeline.elastic-agent-package.yml
+        - .buildkite/scripts/steps/package.sh
+        - .buildkite/scripts/steps/trigger-elastic-agent-package.sh
+
   # wait for CI to be done
   - wait: ~
 

.buildkite/scripts/steps/trigger-elastic-agent-package.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# Create a dynamic buildkite step for running the elastic-agent-package pipeline.
+#
+# Required environment variables:
+#  - BUILDKITE_PULL_REQUEST
+#  - BUILDKITE_COMMIT
+#  - BUILDKITE_BRANCH
+#  - BUILDKITE_PULL_REQUEST_BASE_BRANCH
+#
+
+if [ ! -f .package-version ]; then
+  echo ".package-version file not found!"
+  exit 1
+fi
+
+# No need for the snapshot but the three digits version is required
+BEAT_VERSION=$(jq -r .version .core_version)
+MANIFEST_URL=$(jq -r .manifest_url .package-version)
+
+cat << EOF
+  - label: ":pipeline: Run elastic-agent-package"
+    trigger: "elastic-agent-package"
+    build:
+      message: "#${BUILDKITE_PULL_REQUEST} - Verify elastic-agent-package works"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        DRA_VERSION: "${BEAT_VERSION}"
+        DRA_WORKFLOW: "snapshot"
+        DRA_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        DRA_DRY_RUN: "--dry-run"
+        MANIFEST_URL: "${MANIFEST_URL}"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
+EOF

.package-version

@@ -1,8 +1,8 @@
 {
   "version": "9.3.0-SNAPSHOT",
-  "build_id": "9.3.0-a70e9854",
-  "manifest_url": "https://snapshots.elastic.co/9.3.0-a70e9854/manifest-9.3.0-SNAPSHOT.json",
-  "summary_url": "https://snapshots.elastic.co/9.3.0-a70e9854/summary-9.3.0-SNAPSHOT.html",
+  "build_id": "9.3.0-008e98cd",
+  "manifest_url": "https://snapshots.elastic.co/9.3.0-008e98cd/manifest-9.3.0-SNAPSHOT.json",
+  "summary_url": "https://snapshots.elastic.co/9.3.0-008e98cd/summary-9.3.0-SNAPSHOT.html",
   "core_version": "9.3.0",
-  "stack_build_id": "9.3.0-a70e9854-SNAPSHOT"
+  "stack_build_id": "9.3.0-008e98cd-SNAPSHOT"
 }

dev-tools/packaging/testing/package_test.go

@@ -421,7 +421,7 @@ func checkDocker(t *testing.T, file string, fipsPackage bool) (string, int64) {
 		return checkEdotCollectorDocker(t, file)
 	}
 
-	p, info, err := readDocker(file, true)
+	p, info, err := readDocker(t, file, true)
 	if err != nil {
 		t.Errorf("error reading file %v: %v", file, err)
 		return "", -1
@@ -474,7 +474,7 @@ func dockerName(file string, labels map[string]string) (string, error) {
 }
 
 func checkEdotCollectorDocker(t *testing.T, file string) (string, int64) {
-	p, info, err := readDocker(file, true)
+	p, info, err := readDocker(t, file, true)
 	if err != nil {
 		t.Errorf("error reading file %v: %v", file, err)
 		return "", -1
@@ -500,7 +500,7 @@ func checkEdotCollectorDocker(t *testing.T, file string) (string, int64) {
 }
 
 func checkCompleteDocker(t *testing.T, file string) {
-	p, _, err := readDocker(file, false)
+	p, _, err := readDocker(t, file, false)
 	if err != nil {
 		t.Errorf("error reading file %v: %v", file, err)
 	}
@@ -1079,33 +1079,27 @@ func openZip(zipFile string) (*zip.ReadCloser, error) {
 	return r, nil
 }
 
-func readDocker(dockerFile string, filterWorkingDir bool) (*packageFile, *dockerInfo, error) {
+func readDocker(t *testing.T, dockerFile string, filterWorkingDir bool) (*packageFile, *dockerInfo, error) {
+	t.Helper()
+
 	// Read the manifest file first so that the config file and layer
 	// names are known in advance.
 	manifest, err := getDockerManifest(dockerFile)
-	if err != nil {
-		return nil, nil, err
-	}
+	require.NoError(t, err)
 
 	file, err := os.Open(dockerFile)
-	if err != nil {
-		return nil, nil, err
-	}
+	require.NoError(t, err)
 	defer file.Close()
 
 	var info *dockerInfo
 
 	stat, err := file.Stat()
-	if err != nil {
-		return nil, nil, err
-	}
+	require.NoError(t, err)
 
 	layers := make(map[string]*packageFile)
 
 	gzipReader, err := gzip.NewReader(file)
-	if err != nil {
-		return nil, nil, err
-	}
+	require.NoError(t, err)
 	defer gzipReader.Close()
 
 	tarReader := tar.NewReader(gzipReader)
@@ -1115,38 +1109,29 @@ func readDocker(dockerFile string, filterWorkingDir bool) (*packageFile, *docker
 			if errors.Is(err, io.EOF) {
 				break
 			}
-			return nil, nil, err
+			require.NoError(t, err)
 		}
 
 		switch {
 		case header.Name == manifest.Config:
 			info, err = readDockerInfo(tarReader)
-			if err != nil {
-				return nil, nil, err
-			}
+			require.NoError(t, err)
 		case slices.Contains(manifest.Layers, header.Name):
 			layer, err := readTarContents(header.Name, tarReader)
-			if err != nil {
-				return nil, nil, err
-			}
+			require.NoError(t, err)
 			layers[header.Name] = layer
 		}
 	}
 
-	if len(info.Config.Entrypoint) == 0 {
-		return nil, nil, fmt.Errorf("no entrypoint")
-	}
-
+	require.NotZero(t, len(info.Config.Entrypoint), "no entrypoint")
 	workingDir := info.Config.WorkingDir
 	entrypoint := info.Config.Entrypoint[0]
 
 	// Read layers in order and for each file keep only the entry seen in the later layer
 	p := &packageFile{Name: filepath.Base(dockerFile), Contents: map[string]packageEntry{}}
 	for _, layer := range manifest.Layers {
 		layerFile, found := layers[layer]
-		if !found {
-			return nil, nil, fmt.Errorf("layer not found: %s", layer)
-		}
+		require.True(t, found, fmt.Sprintf("layer not found: %s", layer))
 		for name, entry := range layerFile.Contents {
 			if excludedPathsPattern.MatchString(name) {
 				continue
@@ -1164,10 +1149,7 @@ func readDocker(dockerFile string, filterWorkingDir bool) (*packageFile, *docker
 		}
 	}
 
-	if len(p.Contents) == 0 {
-		return nil, nil, fmt.Errorf("no files found in docker working directory (%s)", info.Config.WorkingDir)
-	}
-
+	require.NotZero(t, len(p.Contents), fmt.Sprintf("no files found in docker working directory (%s)", info.Config.WorkingDir))
 	info.Size = stat.Size()
 	return p, info, nil
 }

internal/pkg/agent/application/coordinator/diagnostics_test.go

@@ -84,6 +84,7 @@ func TestDiagnosticLocalConfig(t *testing.T) {
 						ServerCA:          "/path/to/server/ca",
 					},
 				},
+				MetricsPeriod: monitoringCfg.DefaultMetricsCollectionInterval,
 			},
 		},
 	}
@@ -106,11 +107,12 @@ agent:
     http: null
     logs: false
     metrics: false
-    metrics_period: ""
+    metrics_period: "1m0s"
     namespace: ""
     pprof: null
     failure_threshold: null
     traces: true
+    use_output: ""
     apm:
       hosts:
         - host1

internal/pkg/agent/application/monitoring/component/testdata/monitoring_config_full_otel.yaml

@@ -1,8 +1,3 @@
-agent:
-  monitoring:
-    http:
-      enabled: false
-    metrics: true
 inputs:
 - _runtime_experimental: otel
   id: filestream-monitoring-agent

internal/pkg/agent/application/monitoring/component/testdata/monitoring_config_full_process.yaml

@@ -1,8 +1,3 @@
-agent:
-  monitoring:
-    http:
-      enabled: false
-    metrics: true
 inputs:
 - _runtime_experimental: process
   id: filestream-monitoring-agent