feat(local.file_match, loki.source.file): Match multiple files using doublestar {...} expressions (#5470)

Author: ptodev

.gitignore

@@ -34,6 +34,7 @@ node_modules
 .image-tag
 /alloy
 collector/otel_engine
+collector/otel_engine.exe
 
 # local testing files
 /targets.yaml

collector/go.mod

@@ -332,7 +332,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/bmatcuk/doublestar v1.3.4 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect
 	github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e // indirect
 	github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 // indirect

collector/go.sum

@@ -602,8 +602,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
-github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
-github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE=
 github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY=

docs/sources/reference/components/local/local.file_match.md

@@ -57,6 +57,8 @@ The `__path__` field uses [doublestar][] style glob patterns:
 * `/tmp/**/*.log` matches all subdirectories of `tmp` and include any files that end in `*.log`.
 * `/tmp/apache/*.log` matches only files in `/tmp/apache/` that end in `*.log`.
 * `/tmp/**` matches all subdirectories of `tmp`, `tmp` itself, and all files.
+* `/tmp/*.{log,txt,json}` matches files with `.log`, `.txt`, or `.json` extensions in `/tmp/`.
+* `/var/log/{nginx,apache}/*.log` matches `.log` files in either the `nginx` or `apache` subdirectories.
 
 `local.file_match` doesn't ignore files when you set `ignore_older_than` to the default, `0s`.
 
@@ -181,6 +183,35 @@ Replace the following:
 * _`<USERNAME>`_: The username to use for authentication to the Loki API.
 * _`<PASSWORD>`_: The password to use for authentication to the Loki API.
 
+### Match multiple patterns
+
+This example shows how to use the `{a,b,c}` pattern syntax to match multiple file extensions, multiple directories, and exclude multiple file types in a single configuration.
+
+```alloy
+local.file_match "logs" {
+  path_targets = [
+    // Match .log, .txt, and .json files from nginx, apache, or caddy directories
+    // Exclude compressed and backup files
+    {
+      "__path__"         = "/var/log/{nginx,apache,caddy}/*.{log,txt,json}",
+      "__path_exclude__" = "/var/log/{nginx,apache,caddy}/*.{gz,zip,bak,old}",
+      "job"              = "webserver",
+    },
+  ]
+}
+
+loki.source.file "logs" {
+  targets    = local.file_match.logs.targets
+  forward_to = [loki.write.endpoint.receiver]
+}
+
+loki.write "endpoint" {
+  endpoint {
+    url = "<LOKI_URL>"
+  }
+}
+```
+
 ### Send Kubernetes Pod logs to Loki
 
 This example finds all the logs on Pods and monitors them.

docs/sources/reference/components/loki/loki.source.file.md

@@ -142,6 +142,11 @@ Benefits of using `file_match` over `local.file_match`:
 When `enabled` is set to `true`, you can use glob patterns, for example, `/tmp/*.log` or `/var/log/**/*.log`, directly in the `targets` argument's `__path__` label.
 The component periodically scans the filesystem based on `sync_period` and automatically discovers new files, removes deleted files, and ignores files older than `ignore_older_than` if specified.
 
+The glob patterns support the `{a,b,c}` syntax for matching multiple alternatives:
+
+* `/tmp/*.{log,txt,json}` matches files with `.log`, `.txt`, or `.json` extensions.
+* `/var/log/{nginx,apache}/*.log` matches `.log` files in either the `nginx` or `apache` subdirectories.
+
 ## Exported fields
 
 `loki.source.file` doesn't export any fields.
@@ -262,6 +267,34 @@ loki.write "local" {
 }
 ```
 
+### Match multiple patterns
+
+This example shows how to use the `{a,b,c}` pattern syntax to match multiple file extensions, multiple directories, and exclude multiple file types in a single configuration.
+
+```alloy
+loki.source.file "logs" {
+  targets    = [
+    // Match .log, .txt, and .json files from nginx, apache, or caddy directories
+    // Exclude compressed and backup files
+    {
+      __path__         = "/var/log/{nginx,apache,caddy}/*.{log,txt,json}",
+      __path_exclude__ = "/var/log/{nginx,apache,caddy}/*.{gz,zip,bak,old}",
+      "job"            = "webserver",
+    },
+  ]
+  forward_to = [loki.write.local.receiver]
+  file_match {
+    enabled = true
+  }
+}
+
+loki.write "local" {
+  endpoint {
+    url = "loki:3100/api/v1/push"
+  }
+}
+```
+
 ### Decompression
 
 This example collects log entries from compressed files matching the `*.gz` pattern using the built-in `file_match` block with decompression enabled.

extension/alloyengine/go.mod

@@ -251,7 +251,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/bmatcuk/doublestar v1.3.4 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect
 	github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e // indirect
 	github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 // indirect

extension/alloyengine/go.sum

@@ -612,8 +612,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
-github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
-github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE=
 github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY=

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
 	github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.39.17
 	github.com/blang/semver/v4 v4.0.0
-	github.com/bmatcuk/doublestar v1.3.4
+	github.com/bmatcuk/doublestar/v4 v4.9.1
 	github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e
 	github.com/buger/jsonparser v1.1.1
 	github.com/burningalchemist/sql_exporter v0.0.0-20240103092044-466b38b6abc4
@@ -524,7 +524,6 @@ require (
 	github.com/beevik/ntp v1.3.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect
 	github.com/caarlos0/env/v9 v9.0.0
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect

go.sum

@@ -616,8 +616,6 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
-github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
-github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE=
 github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY=

internal/component/local/file_match/file.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/alloy/internal/component/discovery"
 	"github.com/grafana/alloy/internal/featuregate"
 	"github.com/grafana/alloy/internal/runtime/logging/level"
+	"github.com/grafana/alloy/internal/util/glob"
 )
 
 func init() {
@@ -42,10 +43,15 @@ type Component struct {
 	watches        []watch
 	watchDog       *time.Ticker
 	targetsChanged chan struct{}
+	globber        glob.Globber
 }
 
 // New creates a new local.file_match component.
 func New(o component.Options, args Arguments) (*Component, error) {
+	return newComponent(o, args, glob.NewGlobber())
+}
+
+func newComponent(o component.Options, args Arguments, globber glob.Globber) (*Component, error) {
 	c := &Component{
 		opts:     o,
 		mut:      sync.Mutex{},
@@ -54,6 +60,7 @@ func New(o component.Options, args Arguments) (*Component, error) {
 		watchDog: time.NewTicker(args.SyncPeriod),
 		// Buffered channel to avoid blocking
 		targetsChanged: make(chan struct{}, 1),
+		globber:        globber,
 	}
 
 	if err := c.Update(args); err != nil {
@@ -92,6 +99,7 @@ func (c *Component) Update(args component.Arguments) error {
 			target:          v,
 			log:             c.opts.Logger,
 			ignoreOlderThan: c.args.IgnoreOlderThan,
+			globber:         c.globber,
 		})
 	}