Missing Metrics

[JustNZ]

Hello,

I deployed Mimir via the mimir-distributed helm chart and mainly only configured storage and scaling parts. However after also deploying k8s-monitoring with the mimir integration just like officially documented I can see on the official dashboards by Grafana many panels with empty data.

I can see that metrics are scraped from the mimir instance but as I said it seems like that many are missing or not providing any data.

Does someone has an idea what I am missing? Or is this works as designed?

mimir values:

mimir-distributed:
  global:
    extraEnvFrom:
      - secretRef:
          name: minio-bucket-secret

  mimir:
    structuredConfig:
      multitenancy_enabled: true

      limits:
        max_global_series_per_user: 0

      common:
        storage:
          backend: s3
          s3:
            endpoint: minio.minio.svc.cluster.local:9000
            insecure: true
            access_key_id: ${MINIO_ACCESS_KEY_ID}
            secret_access_key: ${MINIO_ACCESS_KEY_SECRET}

      blocks_storage:
        s3:
          bucket_name: mimir-blocks

      alertmanager_storage:
        s3:
          bucket_name: mimir-alertmanager

      ruler_storage:
        s3:
          bucket_name: mimir-ruler

  gateway:
    replicas: 1
    enabledNonEnterprise: true
    service:
      type: LoadBalancer
    resources:
      limits:
        memory: 1Gi
      requests: {}
    

  overrides-exporter:
    enabled: false

  results-cache:
    enabled: false

  metadata-cache:
    enabled: false

  admin-cache:
    enabled: false

  index-cache:
    enabled: true
    replicas: 1
    allocatedMemory: 1024

  chunks-cache:
    enabled: true
    replicas: 1
    allocatedMemory: 2048

  alertmanager:
    enabled: false

  kafka:
    enabled: true
    persistence:
      enabled: true
      size: 30Gi

  compactor:
    replicas: 1
    persistentVolume:
      size: 20Gi
    resources:
      limits:
        memory: 1Gi
      requests: {}

  distributor:
    replicas: 1
    resources:
      limits:
        memory: 1Gi
      requests: {}

  ingester:
    replicas: 3
    persistentVolume:
      size: 50Gi
    resources:
      limits:
        memory: 4Gi
      requests: {}
    zoneAwareReplication:
      topologyKey: 'kubernetes.io/hostname'
    affinity:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
                - key: app.kubernetes.io/component
                  operator: In
                  values:
                    - ingester
            topologyKey: "kubernetes.io/hostname"

  querier:
    replicas: 2
    resources:
      limits:
        memory: 1Gi
      requests: {}

  query_frontend:
    replicas: 2
    resources:
      limits:
        memory: 512Mi
      requests: {}

  store_gateway:
    replicas: 2
    persistentVolume:
      size: 10Gi
    resources:
      limits:
        memory: 1Gi
      requests: {}
    topologySpreadConstraints: {}
    zoneAwareReplication:
      topologyKey: 'kubernetes.io/hostname'

  ruler:
    replicas: 1
    resources:
      limits:
        memory: 512Mi
      requests: {}

  minio:
    enabled: false

k8s-monitoring values:

k8s-monitoring:
  cluster:
    name: cluster
  
  selfReporting:
    enabled: false
  
  integrations:
    collector: alloy-singleton
    # attach available node labels to all integration metrics collection
    nodeLabels:
      nodepool: true
      region: true
      availability_zone: true
      node_role: true
      os: true
      architecture: true
      instance_type: true

    alloy:
      instances:
        # monitor the collector gathering and sending meta-monitoring metrics/logs to the meta-monitoring cluster
        - name: alloy-in-logs
          namespaces:
            - logs
          labelSelectors:
            app.kubernetes.io/name: alloy-singleton

        # monitor the collectors gathering and sending metrics/logs to the local cluster
        - name: alloy
          labelSelectors:
            app.kubernetes.io/name: [alloy-singleton, alloy-metrics, alloy-logs, alloy-profiles, alloy-receiver]
          namespaces:
            - collectors

    grafana:
      instances:
        - name: grafana
          namespaces:
            - grafana
          labelSelectors:
            app.kubernetes.io/name: grafana

    loki:
      instances:
        - name: loki
          namespaces:
            - loki
          labelSelectors:
            app.kubernetes.io/name: loki
          logs:
            tuning:
              # extract logfmt fields and set them as structured metadata
              structuredMetadata:
                caller:
                tenant:
                org_id:
                user:
    mimir:
      instances:
        - name: mimir
          namespaces:
            - mimir
          labelSelectors:
            app.kubernetes.io/name: mimir
          metrics:
            tuning:
              useDefaultAllowList: false
          logs:
            tuning:
              # extract logfmt fields and set them as structured metadata
              structuredMetadata:
                caller:
                tenant:
                org_id:
                user:

  clusterEvents:
    enabled: true
    collector: alloy-singleton
    namespaces:
      - monitoring
      - loki
      - mimir
      - grafana

  clusterMetrics:
    enabled: true
    collector: alloy-singleton
    nodeLabels:
      nodePool: true
      region: true
      availabilityZone: true
      nodeRole: true
      os: true
      architecture: true
      instanceType: true

    kubelet:
      enabled: false
    kubeletResource:
      enabled: false
    cadvisor:
      enabled: true
      extraMetricProcessingRules: |-
        rule {
          action = "keep"
          source_labels = ["namespace"]
          regex = "monitoring|loki|mimir|grafana"
        }
    apiServer:
      enabled: false
    kubeControllerManager:
      enabled: false
    kubeDNS:
      enabled: false
    kubeProxy:
      enabled: false
    kubeScheduler:
      enabled: false
    kube-state-metrics:
      enabled: true
      namespaces: monitoring,loki,mimir,grafana
      metricsTuning:
        useDefaultAllowList: false
        includeMetrics: [(.+)]
    node-exporter:
      enabled: true
      deploy: true
      metricsTuning:
        useIntegrationAllowList: true
    windows-exporter:
      enabled: false
      deploy: false
    kepler:
      enabled: false
      deploy: false
    opencost:
      enabled: false
      deploy: false

  nodeLogs:
    enabled: false

  podLogs:
    enabled: true
    collector: alloy-singleton
    nodeLabels:
      nodePool: true
      region: true
      availabilityZone: true
      nodeRole: true
      os: true
      architecture: true
      instanceType: true

    structuredMetadata:
      nodepool:
      region:
      availability_zone:
      node_role:
      os:
      architecture:
      instance_type:

    labelsToKeep:
      - app
      - app_kubernetes_io_name
      - component
      - container
      - job
      - level
      - namespace
      - pod
      - service_name

    gatherMethod: kubernetesApi
    namespaces:
      - monitoring
      - loki
      - mimir
      - grafana

  applicationObservability:
    enabled: true
    receivers:
      jaeger:
        thriftHttp:
          enabled: true
          port: 14268
    processors:
      k8sattributes:
        metadata:
          - k8s.namespace.name
          - k8s.pod.name
          - k8s.deployment.name
          - k8s.statefulset.name
          - k8s.daemonset.name
          - k8s.cronjob.name
          - k8s.job.name
          - k8s.node.name
          - k8s.pod.uid
          - k8s.pod.start_time
          - k8s.container.name

  # Collectors
  alloy-singleton:
    enabled: true

  alloy-metrics:
    enabled: false

  alloy-logs:
    enabled: false

  alloy-profiles:
    enabled: false

  alloy-receiver:
    enabled: true
    alloy:
      extraPorts:
        - name: jaeger-http
          port: 14268
          targetPort: 14268
          protocol: TCP

  destinations:
    - name: mimir
      type: prometheus
      url: http://1.1.1.1/api/v1/push
      tenantId: sva
    - name: loki
      type: loki
      url: http://1.1.1.1/loki/api/v1/push
      tenantId: sva
    - name: tempo
      type: otlp
      protocol: http
      url: http://1.1.1.1:4318
      tenantId: sva
      metrics: {enabled: false}
      logs: {enabled: false}
      traces: {enabled: true}