Implement per tag control in tag cardinality processor

Author: ArunPiduguDD

changelog.d/tag_cardinality_limit_per_tag_and_exclude.enhancement.md

@@ -0,0 +1,20 @@
+The `tag_cardinality_limit` transform gained two new configuration capabilities:
+
+- **Per-tag overrides**: each entry in `per_metric_limits` now supports a `per_tag_limits`
+  map whose entries can override settings for a specific tag key on that metric. The
+  following per-tag fields are available:
+  - `value_limit` (optional): caps distinct values for this tag key. Inherits from the
+    enclosing per-metric `value_limit` when unset.
+  - `excluded` (default `false`): when `true`, opts this tag out of cardinality tracking
+    entirely — all values pass through unchecked.
+
+  The tracking mode (`exact`/`probabilistic`), `cache_size_per_key`, `limit_exceeded_action`,
+  and `internal_metrics` are always inherited from the enclosing per-metric configuration
+  and cannot be overridden per-tag.
+
+- **Metric exclusion**: `mode: excluded` is now available in `per_metric_limits` entries
+  (not at the global level). When set, every tag on that metric is opted out of cardinality
+  control — all tag values pass through and nothing is tracked. Any `per_tag_limits`
+  overrides on an excluded metric are ignored.
+
+authors: ArunPiduguDD

src/transforms/tag_cardinality_limit/config.rs

@@ -11,19 +11,9 @@ use crate::{
     transforms::{Transform, tag_cardinality_limit::TagCardinalityLimit},
 };
 
-/// Configuration of internal metrics for the TagCardinalityLimit transform.
-#[configurable_component]
-#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
-#[serde(deny_unknown_fields)]
-pub struct InternalMetricsConfig {
-    /// Whether to include extended tags (metric_name, tag_key) in the `tag_value_limit_exceeded_total` metric.
-    ///
-    /// This helps identify which metrics and tag keys are hitting cardinality limits, but can significantly
-    /// increase metric cardinality. Defaults to `false` because these tags have potentially unbounded cardinality.
-    #[serde(default = "default_include_extended_tags")]
-    #[configurable(metadata(docs::human_name = "Include Extended Tags"))]
-    pub include_extended_tags: bool,
-}
+// =============================================================================
+// Top-level configuration
+// =============================================================================
 
 /// Configuration for the `tag_cardinality_limit` transform.
 #[configurable_component(transform(
@@ -79,7 +69,11 @@ pub enum TrackingScope {
     PerMetric,
 }
 
-/// Configuration for the `tag_cardinality_limit` transform for a specific group of metrics.
+// =============================================================================
+// Global-level configuration block
+// =============================================================================
+
+/// Configuration block used at the global level.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub struct Inner {
@@ -99,7 +93,7 @@ pub struct Inner {
     pub internal_metrics: InternalMetricsConfig,
 }
 
-/// Controls the approach taken for tracking tag cardinality.
+/// Controls the approach taken for tracking tag cardinality at the global level.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 #[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)]
@@ -122,19 +116,131 @@ pub enum Mode {
     Probabilistic(BloomFilterConfig),
 }
 
-/// Bloom filter configuration in probabilistic mode.
+// =============================================================================
+// Per-metric configuration block
+// =============================================================================
+
+/// Tag cardinality limit configuration per metric name.
+#[configurable_component]
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct PerMetricConfig {
+    /// Namespace of the metric this configuration refers to.
+    #[serde(default)]
+    pub namespace: Option<String>,
+
+    /// Per-tag-key overrides scoped to this metric.
+    ///
+    /// Each entry may override `value_limit` for a specific tag key or opt it out of
+    /// tracking entirely with `excluded: true`. The tracking mode, `cache_size_per_key`,
+    /// `limit_exceeded_action`, and `internal_metrics` are always inherited from the
+    /// enclosing per-metric configuration and cannot be set per-tag.
+    /// Tags not listed here use the per-metric configuration.
+    #[configurable(
+        derived,
+        metadata(docs::additional_props_description = "An individual tag configuration.")
+    )]
+    #[serde(default)]
+    pub per_tag_limits: HashMap<String, PerTagConfig>,
+
+    #[serde(flatten)]
+    pub config: OverrideInner,
+}
+
+/// Configuration block used at per-metric level. Same shape as the global configuration but
+/// with `OverrideMode`, which adds `excluded` for opting that metric out of cardinality
+/// control entirely.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub struct BloomFilterConfig {
-    /// The size of the cache for detecting duplicate tags, in bytes.
+pub struct OverrideInner {
+    /// How many distinct values to accept for any given key. Ignored when `mode: excluded`.
+    #[serde(default = "default_value_limit")]
+    pub value_limit: usize,
+
+    #[configurable(derived)]
+    #[serde(default = "default_limit_exceeded_action")]
+    pub limit_exceeded_action: LimitExceededAction,
+
+    #[serde(flatten)]
+    pub mode: OverrideMode,
+
+    #[configurable(derived)]
+    #[serde(default)]
+    pub internal_metrics: InternalMetricsConfig,
+}
+
+/// Controls the approach taken for tracking tag cardinality at the per-metric level.
+/// Adds `excluded` to the global `Mode` variants to allow opting a metric out entirely.
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)]
+#[configurable(metadata(
+    docs::enum_tag_description = "Controls the approach taken for tracking tag cardinality."
+))]
+pub enum OverrideMode {
+    /// Tracks cardinality exactly. See `Mode::Exact` for details.
+    Exact,
+
+    /// Tracks cardinality probabilistically. See `Mode::Probabilistic` for details.
+    Probabilistic(BloomFilterConfig),
+
+    /// Skip cardinality tracking for this metric. All tag values pass through and nothing is
+    /// recorded. Other tracking fields on the entry (`value_limit`, `limit_exceeded_action`,
+    /// `internal_metrics`) are ignored when this is selected. Any `per_tag_limits` entries
+    /// on this metric are also ignored.
     ///
-    /// The larger the cache size, the less likely it is to have a false positive, or a case where
-    /// we allow a new value for tag even after we have reached the configured limits.
-    #[serde(default = "default_cache_size")]
-    #[configurable(metadata(docs::human_name = "Cache Size per Key"))]
-    pub cache_size_per_key: usize,
+    /// Only valid in `per_metric_limits` entries; using it as the global `mode` is a
+    /// configuration error.
+    Excluded,
+}
+
+impl OverrideMode {
+    /// Returns the equivalent global `Mode` if this scope is tracked, or `None` if excluded.
+    pub const fn as_mode(&self) -> Option<Mode> {
+        match self {
+            OverrideMode::Exact => Some(Mode::Exact),
+            OverrideMode::Probabilistic(b) => Some(Mode::Probabilistic(*b)),
+            OverrideMode::Excluded => None,
+        }
+    }
+}
+
+// =============================================================================
+// Per-tag configuration block
+// =============================================================================
+
+/// Tag cardinality limit configuration for a specific tag key, scoped under a per-metric override.
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct PerTagConfig {
+    #[serde(flatten)]
+    pub config: PerTagInner,
+}
+
+/// Configuration block used at the per-tag level.
+///
+/// The tracking mode (`exact` or `probabilistic`) and `cache_size_per_key` are always
+/// inherited from the enclosing per-metric configuration and cannot be overridden per-tag.
+/// A tag can be opted out of cardinality tracking entirely with `excluded: true`.
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct PerTagInner {
+    /// How many distinct values to accept for this tag key. If unset, inherits
+    /// the `value_limit` from the enclosing per-metric (or global) configuration.
+    /// Ignored when `excluded: true`.
+    #[serde(default)]
+    pub value_limit: Option<usize>,
+
+    /// When `true`, opts this tag out of cardinality tracking entirely. All values
+    /// for this tag pass through without being recorded or checked against
+    /// `value_limit`. Defaults to `false`.
+    #[serde(default)]
+    pub excluded: bool,
 }
 
+// =============================================================================
+// Shared building blocks
+// =============================================================================
+
 /// Possible actions to take when an event arrives that would exceed the cardinality limit for one
 /// or more of its tags.
 #[configurable_component]
@@ -148,26 +254,45 @@ pub enum LimitExceededAction {
     DropEvent,
 }
 
-/// Tag cardinality limit configuration per metric name.
+/// Bloom filter configuration in probabilistic mode.
 #[configurable_component]
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct PerMetricConfig {
-    /// Namespace of the metric this configuration refers to.
-    #[serde(default)]
-    pub namespace: Option<String>,
-
-    #[serde(flatten)]
-    pub config: Inner,
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct BloomFilterConfig {
+    /// The size of the cache for detecting duplicate tags, in bytes.
+    ///
+    /// The larger the cache size, the less likely it is to have a false positive, or a case where
+    /// we allow a new value for tag even after we have reached the configured limits.
+    #[serde(default = "default_cache_size")]
+    #[configurable(metadata(docs::human_name = "Cache Size per Key"))]
+    pub cache_size_per_key: usize,
 }
 
-const fn default_limit_exceeded_action() -> LimitExceededAction {
-    LimitExceededAction::DropTag
+/// Configuration of internal metrics for the TagCardinalityLimit transform.
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+#[serde(deny_unknown_fields)]
+pub struct InternalMetricsConfig {
+    /// Whether to include extended tags (metric_name, tag_key) in the `tag_value_limit_exceeded_total` metric.
+    ///
+    /// This helps identify which metrics and tag keys are hitting cardinality limits, but can significantly
+    /// increase metric cardinality. Defaults to `false` because these tags have potentially unbounded cardinality.
+    #[serde(default = "default_include_extended_tags")]
+    #[configurable(metadata(docs::human_name = "Include Extended Tags"))]
+    pub include_extended_tags: bool,
 }
 
+// =============================================================================
+// Defaults
+// =============================================================================
+
 const fn default_value_limit() -> usize {
     500
 }
 
+const fn default_limit_exceeded_action() -> LimitExceededAction {
+    LimitExceededAction::DropTag
+}
+
 const fn default_include_extended_tags() -> bool {
     false
 }
@@ -176,6 +301,10 @@ pub(crate) const fn default_cache_size() -> usize {
     5 * 1024 // 5KB
 }
 
+// =============================================================================
+// Transform plumbing
+// =============================================================================
+
 impl GenerateConfig for Config {
     fn generate_config() -> toml::Value {
         toml::Value::try_from(Self {