feat(database): add mount point telemetry labels to database metrics

builtin/logical/database/backend.go

@@ -143,10 +143,30 @@ func Backend(conf *logical.BackendConfig) *databaseBackend {
 	b.queueCtx, b.cancelQueueCtx = context.WithCancel(context.Background())
 	b.roleLocks = locksutil.CreateLocks()
 	b.schedule = &schedule.DefaultSchedule{}
+	b.mountNamespace = conf.MountNamespace
+	b.mountPoint = conf.MountPath
+	b.includeMountPointInMetrics = conf.IncludeMountPointInMetrics
 
 	return &b
 }
 
+// metricsLabelsForConnection returns the telemetry labels to attach to metrics
+// emitted on behalf of the named connection.
+//
+// When the operator has not opted in, it returns the zero value, which leaves
+// metrics unlabeled so they keep aggregating across mounts as they always have.
+func (b *databaseBackend) metricsLabelsForConnection(name string) databaseWrapperMetricsLabels {
+	if !b.includeMountPointInMetrics {
+		return databaseWrapperMetricsLabels{}
+	}
+
+	return databaseWrapperMetricsLabels{
+		namespace:      b.mountNamespace,
+		mountPoint:     b.mountPoint,
+		connectionName: name,
+	}
+}
+
 func (b *databaseBackend) collectPluginInstanceGaugeValues(context.Context) ([]metricsutil.GaugeLabelValues, error) {
 	// copy the map so we can release the lock
 	connectionsCopy := b.connections.Values()
@@ -164,7 +184,18 @@ func (b *databaseBackend) collectPluginInstanceGaugeValues(context.Context) ([]m
 	}
 	var gauges []metricsutil.GaugeLabelValues
 	for k, v := range counts {
-		gauges = append(gauges, metricsutil.GaugeLabelValues{Labels: []metricsutil.Label{{Name: "dbType", Value: k}}, Value: float32(v)})
+		labels := []metricsutil.Label{{Name: "dbType", Value: k}}
+		if b.includeMountPointInMetrics {
+			// Guard each label so we never emit an empty-valued dimension,
+			// matching how the plugin metrics middleware builds its labels.
+			if b.mountNamespace != "" {
+				labels = append(labels, metricsutil.Label{Name: "namespace", Value: b.mountNamespace})
+			}
+			if b.mountPoint != "" {
+				labels = append(labels, metricsutil.Label{Name: "mount_point", Value: b.mountPoint})
+			}
+		}
+		gauges = append(gauges, metricsutil.GaugeLabelValues{Labels: labels, Value: float32(v)})
 	}
 	return gauges, nil
 }
@@ -198,6 +229,20 @@ type databaseBackend struct {
 	gaugeCollectionProcessStop sync.Once
 
 	schedule schedule.Scheduler
+
+	// mountNamespace is the telemetry-normalized namespace the backend is
+	// mounted in ("root" for the root namespace). It is empty when the backend
+	// is constructed outside of a mount, such as in unit tests.
+	mountNamespace string
+
+	// mountPoint is the path this backend is mounted at, used only to label
+	// telemetry. It is empty when the backend is constructed outside of a mount,
+	// such as in unit tests.
+	mountPoint string
+
+	// includeMountPointInMetrics reflects the add_mount_point_database_metrics
+	// telemetry setting.
+	includeMountPointInMetrics bool
 }
 
 func (b *databaseBackend) DatabaseConfig(ctx context.Context, s logical.Storage, name string) (*DatabaseConfig, error) {

builtin/logical/database/backend_ce.go

@@ -47,7 +47,7 @@ func (b *databaseBackend) GetConnectionWithConfig(ctx context.Context, name stri
 		pluginVersion = pinnedVersion
 	}
 
-	dbw, err := newDatabaseWrapper(ctx, config.PluginName, pluginVersion, b.System(), b.logger)
+	dbw, err := newDatabaseWrapperWithMetricsLabels(ctx, config.PluginName, pluginVersion, b.System(), b.logger, b.metricsLabelsForConnection(name))
 	if err != nil {
 		return nil, fmt.Errorf("unable to create database instance: %w", err)
 	}

builtin/logical/database/path_config_connection_ce.go

@@ -136,7 +136,7 @@ func (b *databaseBackend) connectionWriteHandler() framework.OperationFunc {
 		}
 
 		// Create a database plugin and initialize it.
-		dbw, err := newDatabaseWrapper(ctx, config.PluginName, pluginVersion, b.System(), b.logger)
+		dbw, err := newDatabaseWrapperWithMetricsLabels(ctx, config.PluginName, pluginVersion, b.System(), b.logger, b.metricsLabelsForConnection(name))
 		if err != nil {
 			return logical.ErrorResponse("error creating database object: %s", err), nil
 		}

builtin/logical/database/telemetry_labels_test.go

@@ -0,0 +1,224 @@
+// Copyright IBM Corp. 2016, 2025
+// SPDX-License-Identifier: BUSL-1.1
+
+package database
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	metrics "github.com/hashicorp/go-metrics/compat"
+	"github.com/hashicorp/vault/helper/namespace"
+	"github.com/hashicorp/vault/sdk/helper/consts"
+	"github.com/hashicorp/vault/sdk/helper/pluginutil"
+	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/vault"
+	"github.com/stretchr/testify/require"
+)
+
+// installTestMetricsSink installs a fresh in-memory sink as the global metrics
+// sink and returns it. The global sink is process-wide, so tests using this must
+// not run in parallel with each other.
+func installTestMetricsSink(t *testing.T) *metrics.InmemSink {
+	t.Helper()
+
+	sink := metrics.NewInmemSink(time.Hour, 2*time.Hour)
+
+	// An empty service name keeps emitted keys unprefixed so assertions can
+	// name metrics exactly as the middleware produces them.
+	config := metrics.DefaultConfig("")
+	config.EnableHostname = false
+	config.EnableTypePrefix = false
+	config.EnableRuntimeMetrics = false
+
+	_, err := metrics.NewGlobal(config, sink)
+	require.NoError(t, err)
+
+	return sink
+}
+
+// findMetricLabels returns the labels attached to the named metric, and reports
+// whether the metric was emitted at all.
+func findMetricLabels(sink *metrics.InmemSink, name string) ([]metrics.Label, bool) {
+	for _, interval := range sink.Data() {
+		interval.RLock()
+		for _, counter := range interval.Counters {
+			if counter.Name == name {
+				labels := counter.Labels
+				interval.RUnlock()
+				return labels, true
+			}
+		}
+		interval.RUnlock()
+	}
+	return nil, false
+}
+
+func labelValue(labels []metrics.Label, name string) (string, bool) {
+	for _, label := range labels {
+		if label.Name == name {
+			return label.Value, true
+		}
+	}
+	return "", false
+}
+
+// TestBackend_TelemetryMountPointLabels drives the real database secrets engine
+// with a distinctive namespace, mount path, and connection name, and verifies
+// that the namespace, mount_point, and connection_name labels are attached to
+// the emitted metrics only when the operator has opted in.
+func TestBackend_TelemetryMountPointLabels(t *testing.T) {
+	const (
+		mountNamespace = "team-a"
+		mountPath      = "database-telemetry-test/"
+		connectionName = "telemetry-test-connection"
+	)
+
+	testCases := []struct {
+		name         string
+		includeMount bool
+		expectLabels bool
+	}{
+		{
+			name:         "opted in emits mount and connection labels",
+			includeMount: true,
+			expectLabels: true,
+		},
+		{
+			name:         "opted out emits unlabeled metrics",
+			includeMount: false,
+			expectLabels: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			sink := installTestMetricsSink(t)
+
+			cluster, sys := getCluster(t)
+			defer cluster.Cleanup()
+
+			vault.TestAddTestPlugin(t, cluster.Cores[0].Core, "postgresql-database-plugin", consts.PluginTypeDatabase, "", "TestBackend_PluginMain_PostgresMultiplexed",
+				[]string{fmt.Sprintf("%s=%s", pluginutil.PluginCACertPEMEnv, cluster.CACertPEMFile)})
+
+			config := logical.TestBackendConfig()
+			config.StorageView = &logical.InmemStorage{}
+			config.System = sys
+			config.MountNamespace = mountNamespace
+			config.MountPath = mountPath
+			config.IncludeMountPointInMetrics = tc.includeMount
+
+			b, err := Factory(context.Background(), config)
+			require.NoError(t, err)
+			defer b.Cleanup(context.Background())
+
+			req := &logical.Request{
+				Operation: logical.UpdateOperation,
+				Path:      "config/" + connectionName,
+				Storage:   config.StorageView,
+				Data: map[string]interface{}{
+					"name":              connectionName,
+					"plugin_name":       "postgresql-database-plugin",
+					"connection_url":    "some_postgres_url",
+					"username":          "postgres",
+					"password":          "secret",
+					"verify_connection": false,
+				},
+			}
+			resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+			require.NoError(t, err)
+			require.False(t, resp != nil && resp.IsError(), "connection config write failed: %#v", resp)
+
+			// Initialize runs when the connection is created, so its metrics
+			// are the ones to inspect. The metric names must be unchanged
+			// regardless of whether labels were added.
+			for _, metricName := range []string{"database.Initialize", "database.pgx.Initialize"} {
+				labels, found := findMetricLabels(sink, metricName)
+				require.True(t, found, "metric %q was not emitted", metricName)
+
+				nsLabel, hasNS := labelValue(labels, "namespace")
+				mountLabel, hasMount := labelValue(labels, "mount_point")
+				connLabel, hasConn := labelValue(labels, "connection_name")
+
+				if !tc.expectLabels {
+					require.False(t, hasNS, "metric %q unexpectedly carried a namespace label", metricName)
+					require.False(t, hasMount, "metric %q unexpectedly carried a mount_point label", metricName)
+					require.False(t, hasConn, "metric %q unexpectedly carried a connection_name label", metricName)
+					require.Empty(t, labels, "metric %q should be unlabeled by default", metricName)
+					continue
+				}
+
+				require.True(t, hasNS, "metric %q missing namespace label", metricName)
+				require.True(t, hasMount, "metric %q missing mount_point label", metricName)
+				require.True(t, hasConn, "metric %q missing connection_name label", metricName)
+				require.Equal(t, mountNamespace, nsLabel)
+				require.Equal(t, mountPath, mountLabel)
+				require.Equal(t, connectionName, connLabel)
+			}
+		})
+	}
+}
+
+// TestBackend_TelemetryLabelsDisabledByDefault verifies that a backend built
+// without any telemetry opt-in produces no mount identifying labels, which is
+// the behavior operators upgrading from earlier versions should see.
+func TestBackend_TelemetryLabelsDisabledByDefault(t *testing.T) {
+	config := logical.TestBackendConfig()
+	config.StorageView = &logical.InmemStorage{}
+	config.MountPath = "database/"
+
+	b := Backend(config)
+	require.False(t, b.includeMountPointInMetrics)
+
+	labels := b.metricsLabelsForConnection("some-connection")
+	require.Equal(t, databaseWrapperMetricsLabels{}, labels)
+}
+
+// TestBackend_MetricsLabelsForConnection verifies the gating logic that decides
+// whether a plugin instance is given identifying labels.
+func TestBackend_MetricsLabelsForConnection(t *testing.T) {
+	testCases := []struct {
+		name           string
+		mountNamespace string
+		mountPath      string
+		includeMount   bool
+		connectionName string
+		expected       databaseWrapperMetricsLabels
+	}{
+		{
+			name:           "opted in carries namespace, mount and connection",
+			mountNamespace: "root",
+			mountPath:      "database/",
+			includeMount:   true,
+			connectionName: "primary",
+			expected: databaseWrapperMetricsLabels{
+				namespace:      "root",
+				mountPoint:     "database/",
+				connectionName: "primary",
+			},
+		},
+		{
+			name:           "opted out carries nothing",
+			mountNamespace: "root",
+			mountPath:      "database/",
+			includeMount:   false,
+			connectionName: "primary",
+			expected:       databaseWrapperMetricsLabels{},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			config := logical.TestBackendConfig()
+			config.StorageView = &logical.InmemStorage{}
+			config.MountNamespace = tc.mountNamespace
+			config.MountPath = tc.mountPath
+			config.IncludeMountPointInMetrics = tc.includeMount
+
+			b := Backend(config)
+			require.Equal(t, tc.expected, b.metricsLabelsForConnection(tc.connectionName))
+		})
+	}
+}

builtin/logical/database/version_wrapper.go

@@ -25,18 +25,42 @@ type databaseVersionWrapper struct {
 
 var _ logical.PluginVersioner = databaseVersionWrapper{}
 
+// databaseWrapperMetricsLabels identifies the namespace, mount, and configured
+// connection a plugin instance serves, so that the metrics it emits can be
+// attributed to them. All fields are empty unless the operator opted in via the
+// add_mount_point_database_metrics telemetry setting.
+type databaseWrapperMetricsLabels struct {
+	namespace      string
+	mountPoint     string
+	connectionName string
+}
+
 // newDatabaseWrapper figures out which version of the database the pluginName is referring to and returns a wrapper object
 // that can be used to make operations on the underlying database plugin. If a builtin pluginVersion is provided, it will
 // be ignored.
 func newDatabaseWrapper(ctx context.Context, pluginName string, pluginVersion string, sys pluginutil.LookRunnerUtil, logger log.Logger) (dbw databaseVersionWrapper, err error) {
+	return newDatabaseWrapperWithMetricsLabels(ctx, pluginName, pluginVersion, sys, logger, databaseWrapperMetricsLabels{})
+}
+
+// newDatabaseWrapperWithMetricsLabels is newDatabaseWrapper with the addition of
+// telemetry labels identifying the mount and connection being served.
+func newDatabaseWrapperWithMetricsLabels(ctx context.Context, pluginName string, pluginVersion string, sys pluginutil.LookRunnerUtil, logger log.Logger, metricsLabels databaseWrapperMetricsLabels) (dbw databaseVersionWrapper, err error) {
 	// 1.12.0 and 1.12.1 stored plugin version in the config, but that stored
 	// builtin version may disappear from the plugin catalog when Vault is
 	// upgraded, so always reference builtin plugins by an empty version.
 	if versions.IsBuiltinVersion(pluginVersion) {
 		pluginVersion = ""
 	}
 
-	newDB, err := v5.PluginFactoryVersion(ctx, pluginName, pluginVersion, sys, logger)
+	newDB, err := v5.PluginFactoryWithConfig(ctx, v5.PluginFactoryConfig{
+		PluginName:     pluginName,
+		PluginVersion:  pluginVersion,
+		Sys:            sys,
+		Logger:         logger,
+		Namespace:      metricsLabels.namespace,
+		MountPoint:     metricsLabels.mountPoint,
+		ConnectionName: metricsLabels.connectionName,
+	})
 	if err == nil {
 		dbw = databaseVersionWrapper{
 			v5: newDB,
@@ -47,6 +71,9 @@ func newDatabaseWrapper(ctx context.Context, pluginName string, pluginVersion st
 	merr := &multierror.Error{}
 	merr = multierror.Append(merr, err)
 
+	// The legacy v4 plugin interface is deprecated and its factory does not
+	// carry telemetry labels, so v4-backed connections intentionally emit
+	// unlabeled metrics even when the operator opted in.
 	legacyDB, err := v4.PluginFactoryVersion(ctx, pluginName, pluginVersion, sys, logger)
 	if err == nil {
 		dbw = databaseVersionWrapper{

changelog/32078.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+telemetry: Added a telemetry configuration `add_mount_point_database_metrics` which, when set to true, causes the database secrets engine to attach `namespace`, `mount_point`, and `connection_name` labels to its metrics, allowing them to be broken down per namespace, mount, and configured connection. Metric names are unchanged and the labels are omitted by default.
+```