osquery_manager 1.26.0: osquery 5.22.1, Kibana ^9.4.0, osquery-gen beats tag/branch (#18305)

- Regenerate schemas and fields for osquery 5.22.1; bump package and Kibana constraint.
- Add optional beats.tag and beats.branch in osquery-gen (tag > branch > version).

Author: marc-gr

packages/osquery_manager/_dev/scripts/osquery-gen/README.md

@@ -2,10 +2,10 @@
 
 Config-driven generator for osquery_manager field/schema generation.
 
-This tool reads `osquery` and `beats` versions from `config.yml`, reads the ECS
-git ref from `packages/osquery_manager/_dev/build/build.yml`
+This tool reads `osquery` version and `beats` git selection from `config.yml`,
+reads the ECS git ref from `packages/osquery_manager/_dev/build/build.yml`
 (`dependencies.ecs.reference`, e.g. `git@v9.3.0`), resolves the latest matching
-patch where applicable, and generates:
+patch for osquery/beats when using semver config, and generates:
 
 - `packages/osquery_manager/data_stream/result/fields/osquery.yml`
 - `packages/osquery_manager/data_stream/result/fields/ecs.yml`
@@ -29,7 +29,15 @@ beats:
   version: "9.3"
 ```
 
-Version values can be:
+**Beats** (extension specs under `elastic/beats`): set **either** an explicit git ref **or** a semver-style `version`. Precedence is **tag > branch > version**:
+
+- `tag`: exact ref (for example `v9.4.2`); must contain the osquery extension specs path.
+- `branch`: branch name (for example `main`); same validation as `tag`.
+- `version`: exact patch or major/minor prefix (for example `9.3`) to auto-select a release tag, then the same ref probing as before.
+
+If both `tag` and `branch` are set, `tag` wins.
+
+**Osquery** `version` values can be:
 
 - exact patch (for example `5.18.1`)
 - major/minor prefix (for example `5.18`) to auto-select latest patch

packages/osquery_manager/_dev/scripts/osquery-gen/config.yml

@@ -1,6 +1,10 @@
 osquery:
   # Exact patch (5.21.0) or prefix (5.21) to auto-pick latest patch.
-  version: "5.21"
+  version: "5.22.1"
 beats:
-  # Exact patch (9.3.1) or prefix (9.3) to auto-pick latest patch.
-  version: "9.3"
+  # Git ref for elastic/beats osquery extension specs (raw.githubusercontent.com).
+  # Precedence: tag > branch > version (semver / prefix resolution).
+  # tag: "v9.4.2"
+  # branch: "main"
+  # Exact patch (9.3.1) or prefix (9.3) to auto-pick latest patch when tag/branch unset:
+  version: "9.4.0"

packages/osquery_manager/_dev/scripts/osquery-gen/main.go

@@ -28,9 +28,18 @@ type versionConfig struct {
 	Version string `yaml:"version"`
 }
 
+type beatsConfig struct {
+	// Tag is an exact git ref (e.g. v9.4.2). Takes precedence over Branch and Version.
+	Tag string `yaml:"tag"`
+	// Branch is a git branch name (e.g. main). Used when Tag is empty.
+	Branch string `yaml:"branch"`
+	// Version is a semver or major.minor prefix; used to resolve a tag when Tag and Branch are empty.
+	Version string `yaml:"version"`
+}
+
 type config struct {
 	Osquery versionConfig `yaml:"osquery"`
-	Beats   versionConfig `yaml:"beats"`
+	Beats   beatsConfig   `yaml:"beats"`
 }
 
 type packageBuildYAML struct {
@@ -70,13 +79,9 @@ func main() {
 	if err != nil {
 		log.Fatalf("resolve osquery version: %v", err)
 	}
-	beatsTag, err := resolveLatestPatch("elastic/beats", cfg.Beats.Version)
+	beatsRef, err := resolveBeatsGitRef(cfg.Beats)
 	if err != nil {
-		log.Fatalf("resolve beats version: %v", err)
-	}
-	beatsRef, err := resolveBeatsSpecsRef(cfg.Beats.Version, beatsTag)
-	if err != nil {
-		log.Fatalf("resolve beats specs ref: %v", err)
+		log.Fatalf("resolve beats git ref: %v", err)
 	}
 	ecsVersionSpec, err := loadECSVersionSpecFromBuildYAML(outputRoot)
 	if err != nil {
@@ -106,8 +111,8 @@ func loadConfig(path string) (config, error) {
 	if strings.TrimSpace(cfg.Osquery.Version) == "" {
 		return cfg, fmt.Errorf("osquery.version is required")
 	}
-	if strings.TrimSpace(cfg.Beats.Version) == "" {
-		return cfg, fmt.Errorf("beats.version is required")
+	if strings.TrimSpace(cfg.Beats.Tag) == "" && strings.TrimSpace(cfg.Beats.Branch) == "" && strings.TrimSpace(cfg.Beats.Version) == "" {
+		return cfg, fmt.Errorf("beats: at least one of tag, branch, or version is required")
 	}
 	return cfg, nil
 }
@@ -264,6 +269,39 @@ func (s semver) String() string {
 	return fmt.Sprintf("%d.%d.%d", s.Major, s.Minor, s.Patch)
 }
 
+// resolveBeatsGitRef picks the elastic/beats ref for extension specs: tag > branch > semver version resolution.
+func resolveBeatsGitRef(cfg beatsConfig) (string, error) {
+	if ref := strings.TrimSpace(cfg.Tag); ref != "" {
+		ok, err := refHasBeatsSpecs(ref)
+		if err != nil {
+			return "", fmt.Errorf("beats tag %q: %w", ref, err)
+		}
+		if !ok {
+			return "", fmt.Errorf("beats tag %q: no osquery extension specs at that ref", ref)
+		}
+		return ref, nil
+	}
+	if ref := strings.TrimSpace(cfg.Branch); ref != "" {
+		ok, err := refHasBeatsSpecs(ref)
+		if err != nil {
+			return "", fmt.Errorf("beats branch %q: %w", ref, err)
+		}
+		if !ok {
+			return "", fmt.Errorf("beats branch %q: no osquery extension specs at that ref", ref)
+		}
+		return ref, nil
+	}
+	versionSpec := strings.TrimSpace(cfg.Version)
+	if versionSpec == "" {
+		return "", fmt.Errorf("beats version is required when tag and branch are empty")
+	}
+	beatsTag, err := resolveLatestPatch("elastic/beats", versionSpec)
+	if err != nil {
+		return "", err
+	}
+	return resolveBeatsSpecsRef(versionSpec, beatsTag)
+}
+
 func resolveBeatsSpecsRef(versionSpec, resolvedPatch string) (string, error) {
 	candidates := make([]string, 0, 4)
 	if resolvedPatch != "" {

packages/osquery_manager/changelog.yml

@@ -1,4 +1,13 @@
 # newer versions go on top
+- version: "1.26.0"
+  changes:
+    - description: "Add optional beats `tag` and `branch` in osquery-gen config to pin elastic/beats extension specs"
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18305
+    - description: "Upgrade osquery version to 5.22.1"
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18305
+
 - version: "1.25.0"
   changes:
     - description: Emit schemas/metadata.json from osquery-gen with ecs_version and osquery_version

packages/osquery_manager/manifest.yml

@@ -1,14 +1,14 @@
 format_version: 3.5.8
 name: osquery_manager
 title: Osquery Manager
-version: 1.25.0
+version: 1.26.0
 description: Deploy Osquery with Elastic Agent, then run and schedule queries in Kibana
 type: integration
 categories:
   - security
 conditions:
   kibana:
-    version: "^9.3.0"
+    version: "^9.4.0"
   elastic:
     capabilities:
       - security

packages/osquery_manager/schemas/metadata.json

@@ -1 +1 @@
-{"ecs_version":"9.3.0","osquery_version":"5.21.0"}
+{"ecs_version":"9.3.0","osquery_version":"5.22.1"}