Skip to content

Commit 80892fd

Browse files
QuentinBissonQuentinBisson
authored
Merge branch 'main' into quentin/mimir-loki-namespace-separator
2 parents 04f9003 + 54664b2 commit 80892fd

38 files changed

Lines changed: 2708 additions & 1249 deletions

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ data-alloy/
1818
/packaging/windows/LICENSE
1919
/packaging/windows/agent-windows-amd64.exe
2020
internal/web/ui/dist
21+
internal/web/ui/src/test/generated_fixtures/
2122

2223
.DS_Store
2324
buildx-v*

‎docs/sources/reference/components/database_observability/database_observability.mysql.md‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ It forwards this data as log entries to Loki receivers and exports targets for P
2020
database_observability.mysql "<LABEL>" {
2121
data_source_name = <DATA_SOURCE_NAME>
2222
forward_to = [<LOKI_RECEIVERS>]
23-
targets = "<TARGET_LIST>"
2423
}
2524
```
2625

@@ -32,7 +31,7 @@ You can use the following arguments with `database_observability.mysql`:
3231
|--------------------------------------------|----------------------|-----------------------------------------------------------------------------|---------|----------|
3332
| `data_source_name` | `secret` | [Data Source Name][] for the MySQL server to connect to. | | yes |
3433
| `forward_to` | `list(LogsReceiver)` | Where to forward log entries after processing. | | yes |
35-
| `targets` | `list(map(string))` | List of targets to scrape. | | yes |
34+
| `targets` | `list(map(string))` | List of external targets to scrape. | | no |
3635
| `disable_collectors` | `list(string)` | A list of collectors to disable from the default set. | | no |
3736
| `enable_collectors` | `list(string)` | A list of collectors to enable on top of the default set. | | no |
3837
| `exclude_schemas` | `list(string)` | A list of schemas to exclude from monitoring. | | no |
@@ -56,19 +55,20 @@ You can use the following blocks with `database_observability.mysql`:
5655

5756
{{< docs/alloy-config >}}
5857

59-
| Block | Description | Required |
60-
|--------------------------------------|---------------------------------------------------|----------|
61-
| [`cloud_provider`][cloud_provider] | Provide Cloud Provider information. | no |
62-
| `cloud_provider` > [`aws`][aws] | Provide AWS database host information. | no |
63-
| `cloud_provider` > [`azure`][azure] | Provide Azure database host information. | no |
64-
| [`setup_consumers`][setup_consumers] | Configure the `setup_consumers` collector. | no |
65-
| [`setup_actors`][setup_actors] | Configure the `setup_actors` collector. | no |
66-
| [`query_details`][query_details] | Configure the queries collector. | no |
67-
| [`schema_details`][schema_details] | Configure the schema and table details collector. | no |
68-
| [`explain_plans`][explain_plans] | Configure the explain plans collector. | no |
69-
| [`locks`][locks] | Configure the locks collector. | no |
70-
| [`query_samples`][query_samples] | Configure the query samples collector. | no |
71-
| [`health_check`][health_check] | Configure the health check collector. | no |
58+
| Block | Description | Required |
59+
|--------------------------------------------------|---------------------------------------------------|----------|
60+
| [`cloud_provider`][cloud_provider] | Provide Cloud Provider information. | no |
61+
| `cloud_provider` > [`aws`][aws] | Provide AWS database host information. | no |
62+
| `cloud_provider` > [`azure`][azure] | Provide Azure database host information. | no |
63+
| [`setup_consumers`][setup_consumers] | Configure the `setup_consumers` collector. | no |
64+
| [`setup_actors`][setup_actors] | Configure the `setup_actors` collector. | no |
65+
| [`query_details`][query_details] | Configure the queries collector. | no |
66+
| [`schema_details`][schema_details] | Configure the schema and table details collector. | no |
67+
| [`explain_plans`][explain_plans] | Configure the explain plans collector. | no |
68+
| [`locks`][locks] | Configure the locks collector. | no |
69+
| [`query_samples`][query_samples] | Configure the query samples collector. | no |
70+
| [`health_check`][health_check] | Configure the health check collector. | no |
71+
| [`prometheus_exporter`][prometheus_exporter] | Configure the embedded mysqld_exporter. | no |
7272

7373
[cloud_provider]: #cloud_provider
7474
[aws]: #aws
@@ -81,6 +81,7 @@ You can use the following blocks with `database_observability.mysql`:
8181
[query_samples]: #query_samples
8282
[setup_actors]: #setup_actors
8383
[health_check]: #health_check
84+
[prometheus_exporter]: #prometheus_exporter
8485

8586
{{< /docs/alloy-config >}}
8687

@@ -166,12 +167,18 @@ The `azure` block supplies the identifying information for the database being mo
166167
| `collect_interval` | `duration` | How frequently to check if `setup_actors` are configured correctly. | `"1h"` | no |
167168

168169

169-
### `health_checks`
170+
### `health_check`
170171

171172
| Name | Type | Description | Default | Required |
172173
| -------------------------- | ---------- | ---------------------------------------------------------------------- | ------- | -------- |
173174
| `collect_interval` | `duration` | How frequently to run health checks. | `"1h"` | no |
174175

176+
### `prometheus_exporter`
177+
178+
The `prometheus_exporter` block configures the embedded mysqld_exporter scrapers.
179+
The `data_source_name` is inherited from the parent block.
180+
181+
Refer to [`prometheus.exporter.mysql`](../../prometheus/prometheus.exporter.mysql/) docs for the full list of supported arguments and sub-blocks.
175182

176183
## Example
177184

‎docs/sources/reference/components/database_observability/database_observability.postgres.md‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ It forwards this data as log entries to Loki receivers and exports targets for P
2020
database_observability.postgres "<LABEL>" {
2121
data_source_name = <DATA_SOURCE_NAME>
2222
forward_to = [<LOKI_RECEIVERS>]
23-
targets = "<TARGET_LIST>"
2423
}
2524
```
2625

@@ -32,7 +31,7 @@ You can use the following arguments with `database_observability.postgres`:
3231
|----------------------|----------------------|-------------------------------------------------------------|---------|----------|
3332
| `data_source_name` | `secret` | [Data Source Name][] for the Postgres server to connect to. | | yes |
3433
| `forward_to` | `list(LogsReceiver)` | Where to forward log entries after processing. | | yes |
35-
| `targets` | `list(map(string))` | List of targets to scrape. | | yes |
34+
| `targets` | `list(map(string))` | List of external targets to scrape for Prometheus metrics. | | no |
3635
| `disable_collectors` | `list(string)` | A list of collectors to disable from the default set. | | no |
3736
| `enable_collectors` | `list(string)` | A list of collectors to enable on top of the default set. | | no |
3837
| `exclude_databases` | `list(string)` | A list of databases to exclude from monitoring. | | no |
@@ -76,7 +75,8 @@ You can use the following blocks with `database_observability.postgres`:
7675
| [`query_samples`][query_samples] | Configure the query samples collector. | no |
7776
| [`schema_details`][schema_details] | Configure the schema and table details collector. | no |
7877
| [`explain_plans`][explain_plans] | Configure the explain plans collector. | no |
79-
| [`health_check`][health_check] | Configure the health check collector. | no |
78+
| [`health_check`][health_check] | Configure the health check collector. | no |
79+
| [`prometheus_exporter`][prometheus_exporter] | Configure the embedded postgres_exporter. | no |
8080

8181
[cloud_provider]: #cloud_provider
8282
[aws]: #aws
@@ -86,6 +86,7 @@ You can use the following blocks with `database_observability.postgres`:
8686
[schema_details]: #schema_details
8787
[explain_plans]: #explain_plans
8888
[health_check]: #health_check
89+
[prometheus_exporter]: #prometheus_exporter
8990

9091
{{< /docs/alloy-config >}}
9192

@@ -152,6 +153,13 @@ The `azure` block supplies the identifying information for the database being mo
152153
|--------------------|------------|------------------------------------------------------|---------|----------|
153154
| `collect_interval` | `duration` | How frequently to collect information from database. | `"1h"` | no |
154155

156+
### `prometheus_exporter`
157+
158+
The `prometheus_exporter` block configures the embedded postgres_exporter scrapers.
159+
The `data_source_name` is inherited from the parent block.
160+
161+
Refer to [`prometheus.exporter.postgres`](../../prometheus/prometheus.exporter.postgres/) docs for the full list of supported arguments and sub-blocks.
162+
155163
## `logs` collector
156164

157165
The `logs` collector processes PostgreSQL logs received through the `logs_receiver` entry point and exports Prometheus metrics for query and server errors.

‎internal/component/database_observability/mysql/component.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ var (
6262
type Arguments struct {
6363
DataSourceName alloytypes.Secret `alloy:"data_source_name,attr"`
6464
ForwardTo []loki.LogsReceiver `alloy:"forward_to,attr"`
65-
Targets []discovery.Target `alloy:"targets,attr"`
65+
Targets []discovery.Target `alloy:"targets,attr,optional"`
6666
EnableCollectors []string `alloy:"enable_collectors,attr,optional"`
6767
DisableCollectors []string `alloy:"disable_collectors,attr,optional"`
6868
ExcludeSchemas []string `alloy:"exclude_schemas,attr,optional"`
@@ -460,7 +460,11 @@ func (c *Component) connectAndStartCollectors(ctx context.Context) error {
460460
c.exporterCollector = nil
461461
}
462462

463-
if c.args.PrometheusExporter != nil {
463+
if len(c.args.Targets) == 0 {
464+
if c.args.PrometheusExporter == nil {
465+
d := PrometheusExporterArguments(exporter_mysql.DefaultArguments)
466+
c.args.PrometheusExporter = &d
467+
}
464468
exporterArgs := exporter_mysql.Arguments(*c.args.PrometheusExporter)
465469
exporterCfg := exporterArgs.Convert()
466470
scrapers := mysqld_exporter.GetScrapers(exporterCfg)

‎internal/component/database_observability/mysql/component_test.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,6 @@ func Test_PrometheusExporterBlock(t *testing.T) {
564564
cfg := `
565565
data_source_name = ""
566566
forward_to = []
567-
targets = []
568567
`
569568
var args Arguments
570569
err := syntax.Unmarshal([]byte(cfg), &args)
@@ -576,7 +575,6 @@ func Test_PrometheusExporterBlock(t *testing.T) {
576575
cfg := `
577576
data_source_name = ""
578577
forward_to = []
579-
targets = []
580578
prometheus_exporter {}
581579
`
582580
var args Arguments
@@ -587,11 +585,10 @@ func Test_PrometheusExporterBlock(t *testing.T) {
587585
assert.Equal(t, 2, exporterArgs.LockWaitTimeout) // default value
588586
})
589587

590-
t.Run("present with defaults when empty block", func(t *testing.T) {
588+
t.Run("present with custom collectors", func(t *testing.T) {
591589
cfg := `
592590
data_source_name = ""
593591
forward_to = []
594-
targets = []
595592
prometheus_exporter {
596593
enable_collectors = ["perf_schema.eventsstatements", "perf_schema.eventswaits"]
597594
}

‎internal/component/database_observability/postgres/component.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ var (
6767
type Arguments struct {
6868
DataSourceName alloytypes.Secret `alloy:"data_source_name,attr"`
6969
ForwardTo []loki.LogsReceiver `alloy:"forward_to,attr"`
70-
Targets []discovery.Target `alloy:"targets,attr"`
70+
Targets []discovery.Target `alloy:"targets,attr,optional"`
7171
EnableCollectors []string `alloy:"enable_collectors,attr,optional"`
7272
DisableCollectors []string `alloy:"disable_collectors,attr,optional"`
7373
ExcludeDatabases []string `alloy:"exclude_databases,attr,optional"`
@@ -400,7 +400,11 @@ func (c *Component) connectAndStartCollectors(ctx context.Context) error {
400400
}
401401
c.exporterCollectors = nil
402402

403-
if c.args.PrometheusExporter != nil {
403+
if len(c.args.Targets) == 0 {
404+
if c.args.PrometheusExporter == nil {
405+
d := PrometheusExporterArguments(exporter_postgres.DefaultArguments)
406+
c.args.PrometheusExporter = &d
407+
}
404408
exporterArgs := exporter_postgres.Arguments(*c.args.PrometheusExporter)
405409
slogLogger := slog.New(logging.NewSlogGoKitHandler(c.opts.Logger))
406410
dsn := string(c.args.DataSourceName)

‎internal/component/database_observability/postgres/component_test.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -741,7 +741,6 @@ func Test_PrometheusExporterBlock(t *testing.T) {
741741
cfg := `
742742
data_source_name = "postgresql://user:pass@localhost:5432/db"
743743
forward_to = []
744-
targets = []
745744
`
746745
var args Arguments
747746
err := syntax.Unmarshal([]byte(cfg), &args)
@@ -753,7 +752,6 @@ func Test_PrometheusExporterBlock(t *testing.T) {
753752
cfg := `
754753
data_source_name = "postgresql://user:pass@localhost:5432/db"
755754
forward_to = []
756-
targets = []
757755
prometheus_exporter {}
758756
`
759757
var args Arguments
@@ -769,7 +767,6 @@ func Test_PrometheusExporterBlock(t *testing.T) {
769767
cfg := `
770768
data_source_name = "postgresql://user:pass@localhost:5432/db"
771769
forward_to = []
772-
targets = []
773770
prometheus_exporter {
774771
disable_settings_metrics = true
775772
}

‎internal/component/loki/source/api/api.go‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package api
33
import (
44
"context"
55
"fmt"
6-
"reflect"
76
"sync"
87

98
"github.com/alecthomas/units"
@@ -14,7 +13,7 @@ import (
1413
"github.com/grafana/alloy/internal/component/common/loki"
1514
fnet "github.com/grafana/alloy/internal/component/common/net"
1615
"github.com/grafana/alloy/internal/component/common/relabel"
17-
"github.com/grafana/alloy/internal/component/loki/source/api/internal/lokipush"
16+
"github.com/grafana/alloy/internal/component/loki/source"
1817
"github.com/grafana/alloy/internal/featuregate"
1918
"github.com/grafana/alloy/internal/util"
2019
)
@@ -61,7 +60,7 @@ type Component struct {
6160
uncheckedCollector *util.UncheckedCollector
6261

6362
serverMut sync.Mutex
64-
server *lokipush.PushAPIServer
63+
server *source.Server
6564

6665
fanout *loki.Fanout
6766
}
@@ -120,8 +119,8 @@ func (c *Component) Update(args component.Arguments) error {
120119

121120
c.serverMut.Lock()
122121
defer c.serverMut.Unlock()
123-
serverNeedsRestarting := c.server == nil || !reflect.DeepEqual(c.server.ServerConfig(), *newArgs.Server)
124-
if serverNeedsRestarting {
122+
123+
if c.server.NeedsRestart(newArgs.Server) {
125124
if c.server != nil {
126125
c.server.Shutdown()
127126
}
@@ -130,23 +129,39 @@ func (c *Component) Update(args component.Arguments) error {
130129
// avoid issues with re-registering metrics with the same name, we create a
131130
// new registry for the server every time we create one, and pass it to an
132131
// unchecked collector to bypass uniqueness checking.
133-
serverRegistry := prometheus.NewRegistry()
134-
c.uncheckedCollector.SetCollector(serverRegistry)
132+
reg := prometheus.NewRegistry()
133+
c.uncheckedCollector.SetCollector(reg)
135134

136135
var err error
137-
c.server, err = lokipush.NewPushAPIServer(c.opts.Logger, newArgs.Server, c.handler, serverRegistry, int64(newArgs.MaxSendMessageSize))
136+
c.server, err = source.NewServer(c.opts.Logger, reg, c.handler, source.ServerConfig{
137+
Namespace: "loki_source_api",
138+
NetConfig: newArgs.Server,
139+
LogsConfig: &source.LogsConfig{
140+
FixedLabels: newArgs.labelSet(),
141+
RelabelRules: relabel.ComponentToPromRelabelConfigs(newArgs.RelabelRules),
142+
UseIncomingTimestamp: newArgs.UseIncomingTimestamp,
143+
},
144+
})
145+
138146
if err != nil {
139147
return fmt.Errorf("failed to create embedded server: %v", err)
140148
}
141149

142-
if err = c.server.Run(); err != nil {
150+
logsRoutes, handlerRoutes := newRoutes(int(newArgs.MaxSendMessageSize))
151+
if err = c.server.Run(logsRoutes, handlerRoutes); err != nil {
143152
return fmt.Errorf("failed to run embedded server: %v", err)
144153
}
154+
155+
return nil
145156
}
146157

147-
c.server.SetLabels(newArgs.labelSet())
148-
c.server.SetRelabelRules(newArgs.RelabelRules)
149-
c.server.SetKeepTimestamp(newArgs.UseIncomingTimestamp)
158+
if c.server != nil {
159+
c.server.Update(&source.LogsConfig{
160+
FixedLabels: newArgs.labelSet(),
161+
RelabelRules: relabel.ComponentToPromRelabelConfigs(newArgs.RelabelRules),
162+
UseIncomingTimestamp: newArgs.UseIncomingTimestamp,
163+
})
164+
}
150165

151166
return nil
152167
}

0 commit comments

Comments
 (0)