mappings-gen2.yml

# For metrics with labels: {},
# Airflow-instance-specific information like dag_id and task_id
# are omitted to preserve low cardinality in the metrics.
# The only label will be the Airflow deployment name,
# and this may be inserted by Prometheus recording rules
# in the case of users with many Airflow instances being monitored

mappings:
  - match: 'airflow\.([^.]+)_start'
    match_type: regex
    name: "airflow_job_start"
    labels:
      job_name: "$1"
  - match: 'airflow\.([^.]+)_end'
    match_type: regex
    name: "airflow_job_end"
    labels:
      job_name: "$1"
  - match: 'airflow\.([^.]+)_heartbeat_failure'
    match_type: regex
    name: "airflow_job_heartbeat_failure"
    labels:
      job_name: "$1"

  - match: "airflow.operator_successes_(.*)"
    match_type: regex
    name: "airflow_operator_successes"
    labels:
      operator: "$1"
  - match: "airflow.operator_failures_(.*)"
    match_type: regex
    name: "airflow_operator_failures"
    labels:
      operator: "$1"

  # As of version 0.7.0, we deploy an image that exposes this metric
  # as a counter instead of a gauge. (https://github.com/apache/incubator-airflow/pull/4027).
  # As a result we need to account for this as we transition already running
  # deployments to the latest docker image. This additional label lets us fix up our alerts and dashboards.
  - match: airflow.scheduler_heartbeat
    match_type: regex
    name: "airflow_scheduler_heartbeat"
    labels:
      type: counter

  - match: airflow.dag.*.*.duration
    name: "airflow_task_duration"
    labels:
      dag_id: "$1"
      task_id: "$2"

  - match: airflow.dagrun.duration.success.*
    name: "airflow_dagrun_duration"
    labels:
      dag_id: "$1"
  - match: airflow.dagrun.duration.failed.*
    name: "airflow_dagrun_failed"
    labels:
      dag_id: "$1"
  - match: airflow.dagrun.schedule_delay.*
    name: "airflow_dagrun_schedule_delay"
    labels:
      dag_id: "$1"
  - match: 'airflow\.dagrun\.([^.]+)\.first_task_scheduling_delay'
    match_type: regex
    name: "airflow_dagrun_first_task_scheduling_delay"
    labels:
      dag_id: "$1"

  - match: airflow.dag_processing.last_duration.*
    name: "airflow_dag_processing_last_duration"
    labels:
      dag_file: "$1"
  # This metric was renamed to last_duration, keep this to support older versions
  - match: airflow.dag_processing.last_runtime.*
    name: "airflow_dag_processing_last_runtime"
    labels:
      dag_file: "$1"
  - match: airflow.dag_processing.last_run.seconds_ago.*
    name: "airflow_dag_processing_last_run_seconds_ago"
    labels:
      dag_file: "$1"
  - match: airflow.dag_processing.import_errors
    name: "airflow_dag_processing_import_errors"
    labels: {}
  - match: airflow.dag_processing.total_parse_time
    name: "airflow_dag_processing_total_parse_time"
    labels: {}

  - match: airflow.executor.open_slots
    name: "airflow_executor_open_slots"
    labels: {}

  - match: airflow.dagrun.dependency-check
    name: "airflow_dagrun_dependency_check"
    labels: {}

  - match: airflow.dagrun.dependency-check.*
    name: "airflow_dagrun_dependency_check"
    labels:
      dag_id: "$1"

  - match: airflow.pool.open_slots.*
    name: "airflow_pool_open_slots"
    labels:
      pool: "$1"
  - match: airflow.pool.used_slots.*
    name: "airflow_pool_used_slots"
    labels:
      pool: "$1"
  - match: airflow.pool.queued_slots.*
    name: "airflow_pool_queued_slots"
    labels:
      pool: "$1"
  - match: airflow.pool.running_slots.*
    name: "airflow_pool_running_slots"
    labels:
      pool: "$1"
  - match: airflow.pool.deferred_slots.*
    name: "airflow_pool_deferred_slots"
    labels:
      pool: "$1"
  - match: airflow.pool.scheduled_slots.*
    name: "airflow_pool_scheduled_slots"
    labels:
      pool: "$1"
  - match: airflow.pool.starving_tasks.*
    name: "airflow_pool_starving_tasks"
    labels:
      pool: "$1"

  - match: airflow.zombies_killed
    name: "airflow_zombies_killed"

  - match: airflow.executor.running_tasks
    name: "airflow_executor_running_tasks"
    labels: {}
  - match: airflow.executor.queued_tasks
    name: "airflow_executor_queued_tasks"
    labels: {}
  # Represents percentage of in_use / total of a resource on a specific runner
  - match: airflow.executor.runner_resources.*
    name: airflow_runner_resources
    labels:
      # ex: 'slots', 'cpu', 'memory' etc.
      resource: "$1"
  - match: airflow.executor.task_resources.*
    name: airflow_executor_task_resources
    labels:
      # ex: 'memory_rss', 'cpu_times_system' etc.
      resource_stat: "$1"

  - match: airflow.ti.start.*.*
    name: "airflow_ti_start"
    labels:
      dag_id: "$1"
      task_id: "$2"
  - match: airflow.ti.finish.*.*.*
    name: "airflow_ti_finish"
    labels:
      dag_id: "$1"
      task_id: "$2"
      state: "$3"
  - match: airflow.ti_failures
    name: "airflow_ti_failures"
  - match: airflow.ti_successes
    name: "airflow_ti_successes"
  - match: "airflow.task_instance_created_(.*)"
    match_type: regex
    name: "airflow_task_instance_created"
    labels:
      task_type: "$1"

  - match: airflow.dagbag_size
    name: "airflow_dagbag_size"
    labels: {}

  - match: airflow.scheduler.tasks.running
    name: "airflow_scheduler_tasks_running"
  - match: airflow.scheduler.tasks.killed_externally
    name: "airflow_scheduler_tasks_killed_externally"
  - match: airflow.scheduler.tasks.starving
    name: "airflow_scheduler_tasks_starving"

  - match: airflow.collect_db_dags
    name: "airflow_collect_db_dags"
    labels: {}

  - match: airflow.triggers.succeeded
    name: "airflow_triggers_succeeded"
    labels: {}
  - match: airflow.triggers.failed
    name: "airflow_triggers_failed"
    labels: {}
  - match: airflow.triggers.running
    name: "airflow_triggers_running"
    labels: {}

  - match: airflow.dataset.triggered_dagruns
    name: "airflow_dataset_triggered_dagruns"
    labels: {}
  - match: airflow.dataset.updates
    name: "airflow_dataset_updates"
    labels: {}
  - match: airflow.dataset.orphaned
    name: "airflow_dataset_orphaned"
    labels: {}

  - match: airflow.ol.emit.attempts
    name: "airflow_ol_emit_attempts"
    labels: {}
  - match: airflow.ol.emit.failed
    name: "airflow_ol_emit_failed"
    labels: {}
  # Mapping for astronomer-providers-logging logs write failures
  # This captures metrics for logs write failures in the astronomer-providers-logging,
  # with the provider name as the label.
  - match: 'airflow\.astro_logging\.(.+)\.write\.failed'
    match_type: regex
    name: "airflow_astro_logging_write_failed"
    labels:
      provider: "$1"

  # Mappings for the Astro bundle backend
  # See: https://github.com/astronomer/astro-bundle-backend
  - match: "airflow.astro.bundle_backend.refresh_success.*.*"
    name: "astro_bundle_backend_refresh_success"
    labels:
      instance: "$1"
      mount_path: "$2"
  - match: "airflow.astro.bundle_backend.refresh_failure.*.*"
    name: "astro_bundle_backend_refresh_failure"
    labels:
      instance: "$1"
      mount_path: "$2"
  - match: "airflow.astro.bundle_backend.tarball_size.*.*.*"
    name: "astro_bundle_backend_tarball_size"
    labels:
      instance: "$1"
      mount_path: "$2"
      le: "$3"
  - match: "airflow.astro.bundle_backend.num_files.*.*.*"
    name: "astro_bundle_backend_num_files"
    labels:
      instance: "$1"
      mount_path: "$2"
      le: "$3"
  - match: "airflow.astro.bundle_backend.download_time.*.*.*"
    name: "astro_bundle_backend_download_time"
    labels:
      instance: "$1"
      mount_path: "$2"
      le: "$3"
  - match: "airflow.astro.bundle_backend.extract_time.*.*.*"
    name: "astro_bundle_backend_extract_time"
    labels:
      instance: "$1"
      mount_path: "$2"
      le: "$3"
  - match: "airflow.astro.bundle_backend.download_urls_success.*.*"
    name: "astro_bundle_backend_download_urls_success"
    labels:
      instance: "$1"
      mount_path: "$2"
  - match: "airflow.astro.bundle_backend.download_urls_failure.*.*"
    name: "astro_bundle_backend_download_urls_failure"
    labels:
      instance: "$1"
      mount_path: "$2"

  # ------------------------------------------------------------
  # Cosmos-related metrics
  # ------------------------------------------------------------

  # The goal with Cosmos metrics is to capture information about how end-users are using Cosmos, so we can plan our roadmap, in particular, the Cosmos 2.0 release.
  # We want to avoid capturing redundant information that is already being collected by other parts of the Astro stack.

  # There are 2 types of Cosmos metrics:
  # 1. Counters
  # 2. Durations
  # These are identified by the suffix ".counter" or ".duration"

  # What is the name of the operator class used to run the task? Did the end-user subclass it?
  # Which dbt command was used to run the task?
  # What execution mode was used? What invocation mode was used?
  - match: cosmos\.task\.operator_name\.([^.]+)\.is_subclass\.([^.]+)\.execution_mode\.([^.]+)\.invocation_mode\.([^.]+)\.dbt_command\.([^.]+)\.instalL_deps\.([^.]+)\.origin\.([^.]+)\.has_callback\.([^.]+)\.status\.([^.]+)\.counter$
    match_type: regex
    name: "cosmos_task_overview"
    labels:
      operator_name: "$1" # Example: "DbtRunLocalOperator", "DbtProducerWatcherOperator"
      is_subclass: "$2" # True or False
      execution_mode: "$3" # Example: "local", "virtualenv", "watcher"
      invocation_mode: "$4" # Subprocess or DbtRunner
      dbt_command: "$5" # Example: "run", "build", "test"
      install_deps: "$6" # True or False
      origin: "$7" # DbtTaskGroup, DbtDag or StandaloneTask
      has_callback: "$8" # True or False
      status: "$9" # success, failure

  # Which database did the user run the transformations against?
  # How did the user define the profile, via a ProfileMapping or YAML file?
  # If Profile mapping, which class was used to define the profile?
  - match: cosmos\.profile\.database\.([^.]+)\.profile_strategy\.([^.]+)\.profile_mapping_class\.([^.]+)\.counter$
    match_type: regex
    name: "cosmos_profile_overview"
    labels:
      database: "$1" # postgres, snowflake, databricks, bigquery, etc
      profile_strategy: "$2" # yaml_file or mapping
      profile_mapping_class: "$3" # None, SnowflakeEncryptedPrivateKeyPemProfileMapping, PostgresUserPasswordProfileMapping, DatabricksTokenProfileMapping, etc

  # If using `DbtDag` or `DbtTaskGroup`, how did the user parse the dbt project? If using dbt_ls, was `dbt` in the same Python virtualenv as Airflow?
  # Did the user specify a custom load converter?
  # What were the source and node behaviors?
  # How many dbt nodes were in the project?
  # How many dbt nodes were selected by the user?
  # How long did the parse operation take?
  # How long did the filtering operation take?
  - match: cosmos\.rendering\.used_automatic_load_mode\.([^.]+)\.actual_load_mode\.([^.]+)\.invocation_mode\.([^.]+)\.instalL_deps\.([^.]+)\.uses_node_converter\.([^.]+)\.test_behavior\.([^.]+)\.source_behavior\.([^.]+)\.total_dbt_models\.([^.]+)\.selected_dbt_models\.([^.]+)\.counter$
    match_type: regex
    name: "cosmos_rendering_overview"
    labels:
      used_automatic_load_mode: "$1" # True or False
      actual_load_mode: "$2" # dbt_ls, dbt_ls_cache, dbt_ls_file, dbt_manifest, custom
      invocation_mode: "$3" # subprocess or dbt_runner
      instalL_deps: "$4" # True or False
      uses_node_converter: "$5" # True or False
      test_behavior: "$6" # after_each, after_all, none, build
      source_behavior: "$7" # all, with_tests_or_freshness, none
      total_dbt_models: "$8" # Total number of dbt models in the project
      selected_dbt_models: "$9" # Total number of dbt models selected by the user

  # How long did Cosmos take to parse the dbt project?
  - match: cosmos\.rendering\.actual_load_mode\.([^.]+)\.dbt_nodes_parsing\.duration$
    match_type: regex
    name: "cosmos_rendering_dbt_nodes_parsing_duration"
    labels:
      actual_load_mode: "$1"

  # How long did Cosmos take to filter the dbt project after it was parsed?
  - match: cosmos\.rendering\.actual_load_mode\.([^.]+)\.dbt_nodes_filtering\.duration$
    match_type: regex
    name: "cosmos_rendering_dbt_nodes_filtering_duration"
    labels:
      actual_load_mode: "$1"

  # How long did Cosmos take to build the Airflow DAG dynamically, after the dbt project has been parsed and filtered?
  - match: cosmos\.rendering\.actual_load_mode\.([^.]+)\.airflow_dag_generation\.duration$
    match_type: regex
    name: "cosmos_rendering_airflow_dag_generation_duration"
    labels:
      actual_load_mode: "$1"

  # ------------------------------------------------------------

  # drop any metric not matched
  - match: "."
    match_type: regex
    action: drop
    name: "dropped"