airflow.cfg.tmpl

[api]
auth_backends = configuration.keycloak_bearer_auth, airflow.api.auth.backend.basic_auth

[core]
executor = CeleryExecutor
dags_are_paused_at_creation = true
load_examples = false
load_default_connections = false
# Allow airflow to run hundreds of tasks in parallel, because we will scale workers
# automatically.
# https://programmaticponderings.com/2020/12/29/amazon-managed-workflows-for-apache-airflow-configuration-understanding-amazon-mwaas-configuration-options/
max_active_tasks_per_dag = 10000
parallelism = 10000

[celery]
broker_url = sqs://
celery_config_options = configuration.celery_config.CELERY_CONFIG

[webserver]
authenticate = True
dag_default_view = grid
expose_config = true
dag_orientation = TB
warn_deployment_exposure = false
base_url = ${sm2a_base_url}

# On ECS, you can deploy the CloudWatch agent as a sidecar to your application container to collect metrics.
# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/deploy_servicelens_CloudWatch_agent_deploy_ECS.html
# https://airflow.apache.org/docs/apache-airflow/stable/logging-monitoring/metrics.html
# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-custom-metrics-statsd.html
# https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-autoscaling.html
# https://docs.aws.amazon.com/mwaa/latest/userguide/access-metrics-cw-202.html#available-metrics-cw-v202
# [metrics]
# statsd_on = true
# statsd_host = localhost
# statsd_port = 8125
# statsd_prefix = airflow

[scheduler]
catchup_by_default = false

[logging]
# logging_config_class = configuration.logging_config.STDOUT_LOGGING_CONFIG
remote_logging = true
# We set this value as an environment variable
# remote_base_log_folder =

[secrets]
# AWS Secrets Manager Backend
# https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/secrets-backends/aws-secrets-manager.html
# Setting full_url_mode to false allows us to use multiple fields when storing connections
# Source code: https://github.com/apache/airflow/blob/main/airflow/providers/amazon/aws/secrets/secrets_manager.py
backend = airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend
backend_kwargs = {"connections_prefix": "${prefix}/airflow/connections", "variables_prefix": "${prefix}/airflow/variables","connections_lookup_pattern": "_default$", "variables_lookup_pattern": "^aws_", "config_prefix": "${prefix}/airflow/config"}