feat(accounting-reconciliation): CHK-4676 add cosmos db on qi and accounting reconciliation collections (#3544)

* feat(accounting-reconciliation): add cosmos db on qi domain and accounting reconciliation collections

* fix: minor fix for qi cosmos

* feat(cosmos): set accounting reconciliation collections default TTL to 180 days

* Update src/domains/qi-common/03_cosmosdb.tf

* chore: minore renaming refactorgin db

* chore: update qi cosmos conf

---------

Co-authored-by: Simone infante <52280205+infantesimone@users.noreply.github.com>

Author: ivanferrando-pagopa

src/domains/qi-common/01_network.tf

@@ -25,6 +25,17 @@ data "azurerm_resource_group" "rg_vnet_italy" {
   name = local.vnet_italy_resource_group_name
 }
 
+data "azurerm_subnet" "aks_subnet" {
+  name                 = local.aks_subnet_name
+  virtual_network_name = local.vnet_name
+  resource_group_name  = local.vnet_resource_group_name
+}
+
+data "azurerm_private_dns_zone" "cosmos" {
+  name                = local.cosmos_dns_zone_name
+  resource_group_name = local.cosmos_dns_zone_resource_group_name
+}
+
 #
 # Eventhub
 #
@@ -43,4 +54,21 @@ resource "azurerm_subnet" "eventhub_qi_snet" {
   resource_group_name  = data.azurerm_resource_group.rg_vnet_italy.name
   virtual_network_name = data.azurerm_virtual_network.vnet_italy.name
   address_prefixes     = var.cidr_subnet_qi_evh
-}
+}
+
+module "cosmosdb_qi_snet" {
+  source = "./.terraform/modules/__v3__/subnet"
+
+  name                 = "${local.project}-cosmosb-snet"
+  resource_group_name  = local.vnet_resource_group_name
+  virtual_network_name = local.vnet_name
+
+  address_prefixes = var.cidr_subnet_cosmosdb_qi
+
+  private_endpoint_network_policies_enabled = true
+
+  service_endpoints = [
+    "Microsoft.Web",
+    "Microsoft.AzureCosmosDB",
+  ]
+}

src/domains/qi-common/03_cosmosdb.tf

@@ -0,0 +1,101 @@
+resource "azurerm_resource_group" "cosmosdb_qi_rg" {
+  name     = "${local.project}-cosmosdb-rg"
+  location = var.location
+
+  tags = module.tag_config.tags
+}
+
+module "cosmosdb_account_qi_mongodb" {
+
+  source = "./.terraform/modules/__v3__/cosmosdb_account"
+
+  name                = "${local.project}-cosmos-account"
+  location            = var.location
+  resource_group_name = azurerm_resource_group.cosmosdb_qi_rg.name
+  domain              = var.domain
+
+  offer_type   = var.cosmos_mongo_db_params.offer_type
+  kind         = var.cosmos_mongo_db_params.kind
+  capabilities = var.cosmos_mongo_db_params.capabilities
+  # mongo_server_version = var.cosmos_mongo_db_params.server_version
+  enable_free_tier = var.cosmos_mongo_db_params.enable_free_tier
+
+  public_network_access_enabled      = var.cosmos_mongo_db_params.public_network_access_enabled
+  private_endpoint_enabled           = var.cosmos_mongo_db_params.private_endpoint_enabled
+  subnet_id                          = module.cosmosdb_qi_snet.id
+  private_dns_zone_mongo_ids         = [data.azurerm_private_dns_zone.cosmos.id]
+  is_virtual_network_filter_enabled  = var.cosmos_mongo_db_params.is_virtual_network_filter_enabled
+  allowed_virtual_network_subnet_ids = var.cosmos_mongo_db_params.public_network_access_enabled ? [] : [data.azurerm_subnet.aks_subnet.id]
+
+  consistency_policy               = var.cosmos_mongo_db_params.consistency_policy
+  main_geo_location_location       = azurerm_resource_group.cosmosdb_qi_rg.location
+  main_geo_location_zone_redundant = var.cosmos_mongo_db_params.main_geo_location_zone_redundant
+  additional_geo_locations         = var.cosmos_mongo_db_params.additional_geo_locations
+
+  backup_continuous_enabled                    = var.cosmos_mongo_db_params.backup_continuous_enabled
+  enable_provisioned_throughput_exceeded_alert = var.cosmos_mongo_db_params.enable_provisioned_throughput_exceeded_alert
+
+  tags = module.tag_config.tags
+}
+
+resource "azurerm_cosmosdb_mongo_database" "accounting_reconciliation" {
+
+  name                = "accounting-reconciliation"
+  resource_group_name = azurerm_resource_group.cosmosdb_qi_rg.name
+  account_name        = module.cosmosdb_account_qi_mongodb.name
+
+  throughput = var.cosmos_mongo_db_accounting_reconciliation_params.enable_autoscaling || var.cosmos_mongo_db_accounting_reconciliation_params.enable_serverless ? null : var.cosmos_mongo_db_accounting_reconciliation_params.throughput
+
+  dynamic "autoscale_settings" {
+    for_each = var.cosmos_mongo_db_accounting_reconciliation_params.enable_autoscaling && !var.cosmos_mongo_db_accounting_reconciliation_params.enable_serverless ? [""] : []
+    content {
+      max_throughput = var.cosmos_mongo_db_accounting_reconciliation_params.max_throughput
+    }
+  }
+
+}
+
+# Collections
+locals {
+  accounting_reconciliation_collections = [
+    {
+      name = "accounting-zip"
+      indexes = [
+        {
+          keys   = ["_id"]
+          unique = true
+        }
+      ]
+      shard_key           = "_id",
+      default_ttl_seconds = "15552000" # 180 days = 180d * 24h * 60m * 60s
+    },
+    {
+      name = "accounting-xml"
+      indexes = [
+        {
+          keys   = ["_id"]
+          unique = true
+        }
+      ]
+      shard_key           = "_id",
+      default_ttl_seconds = "15552000" # 180 days = 180d * 24h * 60m * 60s
+    }
+  ]
+}
+
+module "cosmosdb_accounting_reconciliation_collections" {
+
+  source   = "./.terraform/modules/__v3__/cosmosdb_mongodb_collection"
+  for_each = { for index, coll in local.accounting_reconciliation_collections : coll.name => coll }
+
+  name                = each.value.name
+  resource_group_name = azurerm_resource_group.cosmosdb_qi_rg.name
+
+  cosmosdb_mongo_account_name  = module.cosmosdb_account_qi_mongodb.name
+  cosmosdb_mongo_database_name = azurerm_cosmosdb_mongo_database.accounting_reconciliation.name
+
+  indexes             = each.value.indexes
+  shard_key           = each.value.shard_key
+  default_ttl_seconds = each.value.default_ttl_seconds
+  lock_enable         = var.env_short != "p" ? false : true
+}

src/domains/qi-common/99_locals.tf

@@ -28,6 +28,9 @@ locals {
   internal_dns_zone_name                = "${var.dns_zone_internal_prefix}.${var.external_domain}"
   internal_dns_zone_resource_group_name = "${local.product}-vnet-rg"
 
+  cosmos_dns_zone_name                = "privatelink.mongo.cosmos.azure.com"
+  cosmos_dns_zone_resource_group_name = "${local.product}-vnet-rg"
+
   aks_subnet_name = "${var.prefix}-${var.env_short}-${var.location_short}-${var.env}-aks-snet"
 
   azdo_managed_identity_rg_name = "pagopa-${var.env_short}-identity-rg"

src/domains/qi-common/99_variables.tf

@@ -230,3 +230,46 @@ variable "eventhubs_bdi" {
   }))
   default = []
 }
+
+# CosmosDb
+
+variable "cosmos_mongo_db_params" {
+  type = object({
+    enabled        = bool
+    capabilities   = list(string)
+    offer_type     = string
+    server_version = string
+    kind           = string
+    consistency_policy = object({
+      consistency_level       = string
+      max_interval_in_seconds = number
+      max_staleness_prefix    = number
+    })
+    enable_free_tier                 = bool
+    main_geo_location_zone_redundant = bool
+    additional_geo_locations = list(object({
+      location          = string
+      failover_priority = number
+      zone_redundant    = bool
+    }))
+    private_endpoint_enabled                     = bool
+    public_network_access_enabled                = bool
+    is_virtual_network_filter_enabled            = bool
+    backup_continuous_enabled                    = bool
+    enable_provisioned_throughput_exceeded_alert = bool
+  })
+}
+
+variable "cosmos_mongo_db_accounting_reconciliation_params" {
+  type = object({
+    enable_serverless  = bool
+    enable_autoscaling = bool
+    throughput         = number
+    max_throughput     = number
+  })
+}
+
+variable "cidr_subnet_cosmosdb_qi" {
+  type        = list(string)
+  description = "Cosmos DB address space for qi."
+}

src/domains/qi-common/README.md

@@ -13,6 +13,9 @@
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module___v3__"></a> [\_\_v3\_\_](#module\_\_\_v3\_\_) | git::https://github.com/pagopa/terraform-azurerm-v3 | f3485105e35ce8c801209dcbb4ef72f3d944f0e5 |
+| <a name="module_cosmosdb_account_qi_mongodb"></a> [cosmosdb\_account\_qi\_mongodb](#module\_cosmosdb\_account\_qi\_mongodb) | ./.terraform/modules/__v3__/cosmosdb_account | n/a |
+| <a name="module_cosmosdb_accounting_reconciliation_collections"></a> [cosmosdb\_accounting\_reconciliation\_collections](#module\_cosmosdb\_accounting\_reconciliation\_collections) | ./.terraform/modules/__v3__/cosmosdb_mongodb_collection | n/a |
+| <a name="module_cosmosdb_qi_snet"></a> [cosmosdb\_qi\_snet](#module\_cosmosdb\_qi\_snet) | ./.terraform/modules/__v3__/subnet | n/a |
 | <a name="module_eventhub_namespace_qi"></a> [eventhub\_namespace\_qi](#module\_eventhub\_namespace\_qi) | ./.terraform/modules/__v3__/eventhub | n/a |
 | <a name="module_eventhub_qi_configuration"></a> [eventhub\_qi\_configuration](#module\_eventhub\_qi\_configuration) | ./.terraform/modules/__v3__/eventhub_configuration | n/a |
 | <a name="module_identity_cd_01"></a> [identity\_cd\_01](#module\_identity\_cd\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | n/a |
@@ -29,6 +32,7 @@
 | [azuread_application.qi_app](https://registry.terraform.io/providers/hashicorp/azuread/2.38.0/docs/resources/application) | resource |
 | [azuread_application_password.qi_app_pwd](https://registry.terraform.io/providers/hashicorp/azuread/2.38.0/docs/resources/application_password) | resource |
 | [azuread_service_principal.qi_sp](https://registry.terraform.io/providers/hashicorp/azuread/2.38.0/docs/resources/service_principal) | resource |
+| [azurerm_cosmosdb_mongo_database.accounting_reconciliation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/cosmosdb_mongo_database) | resource |
 | [azurerm_key_vault_access_policy.ad_group_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource |
 | [azurerm_key_vault_access_policy.adgroup_developers_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource |
 | [azurerm_key_vault_access_policy.adgroup_externals_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource |
@@ -50,6 +54,7 @@
 | [azurerm_key_vault_secret.qi_service_principal_client_id](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource |
 | [azurerm_key_vault_secret.qi_service_principal_client_secret](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource |
 | [azurerm_private_dns_a_record.ingress](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_a_record) | resource |
+| [azurerm_resource_group.cosmosdb_qi_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
 | [azurerm_resource_group.qi_evh_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
 | [azurerm_resource_group.qi_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
 | [azurerm_resource_group.sec_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
@@ -76,12 +81,14 @@
 | [azurerm_log_analytics_workspace.log_analytics](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source |
 | [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source |
 | [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source |
+| [azurerm_private_dns_zone.cosmos](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source |
 | [azurerm_private_dns_zone.eventhub](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source |
 | [azurerm_private_dns_zone.internal](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source |
 | [azurerm_resource_group.identity_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source |
 | [azurerm_resource_group.monitor_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source |
 | [azurerm_resource_group.rg_event_private_dns_zone](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source |
 | [azurerm_resource_group.rg_vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source |
+| [azurerm_subnet.aks_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source |
 | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source |
 | [azurerm_user_assigned_identity.iac_federated_azdo](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) | data source |
 | [azurerm_virtual_network.vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source |
@@ -91,7 +98,10 @@
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_cidr_subnet_cosmosdb_qi"></a> [cidr\_subnet\_cosmosdb\_qi](#input\_cidr\_subnet\_cosmosdb\_qi) | Cosmos DB address space for qi. | `list(string)` | n/a | yes |
 | <a name="input_cidr_subnet_qi_evh"></a> [cidr\_subnet\_qi\_evh](#input\_cidr\_subnet\_qi\_evh) | Address prefixes evh | `list(string)` | n/a | yes |
+| <a name="input_cosmos_mongo_db_accounting_reconciliation_params"></a> [cosmos\_mongo\_db\_accounting\_reconciliation\_params](#input\_cosmos\_mongo\_db\_accounting\_reconciliation\_params) | n/a | <pre>object({<br/>    enable_serverless  = bool<br/>    enable_autoscaling = bool<br/>    throughput         = number<br/>    max_throughput     = number<br/>  })</pre> | n/a | yes |
+| <a name="input_cosmos_mongo_db_params"></a> [cosmos\_mongo\_db\_params](#input\_cosmos\_mongo\_db\_params) | n/a | <pre>object({<br/>    enabled        = bool<br/>    capabilities   = list(string)<br/>    offer_type     = string<br/>    server_version = string<br/>    kind           = string<br/>    consistency_policy = object({<br/>      consistency_level       = string<br/>      max_interval_in_seconds = number<br/>      max_staleness_prefix    = number<br/>    })<br/>    enable_free_tier                 = bool<br/>    main_geo_location_zone_redundant = bool<br/>    additional_geo_locations = list(object({<br/>      location          = string<br/>      failover_priority = number<br/>      zone_redundant    = bool<br/>    }))<br/>    private_endpoint_enabled                     = bool<br/>    public_network_access_enabled                = bool<br/>    is_virtual_network_filter_enabled            = bool<br/>    backup_continuous_enabled                    = bool<br/>    enable_provisioned_throughput_exceeded_alert = bool<br/>  })</pre> | n/a | yes |
 | <a name="input_dns_zone_internal_prefix"></a> [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no |
 | <a name="input_domain"></a> [domain](#input\_domain) | n/a | `string` | n/a | yes |
 | <a name="input_ehns_alerts_enabled"></a> [ehns\_alerts\_enabled](#input\_ehns\_alerts\_enabled) | Event hub alerts enabled? | `bool` | n/a | yes |

src/domains/qi-common/env/weu-dev/terraform.tfvars

@@ -123,3 +123,38 @@ ehns_metric_alerts_qi = {
   #   ],
   # },
 }
+
+### Cosmos
+
+cosmos_mongo_db_params = {
+  enabled      = true
+  kind         = "MongoDB"
+  capabilities = ["EnableMongo", "EnableServerless"]
+  offer_type   = "Standard"
+  consistency_policy = {
+    consistency_level       = "BoundedStaleness"
+    max_interval_in_seconds = 5
+    max_staleness_prefix    = 100000
+  }
+  server_version                   = "6.0"
+  main_geo_location_zone_redundant = false
+  enable_free_tier                 = false
+
+  additional_geo_locations          = []
+  private_endpoint_enabled          = false
+  public_network_access_enabled     = true
+  is_virtual_network_filter_enabled = false
+
+  backup_continuous_enabled                    = false
+  enable_provisioned_throughput_exceeded_alert = false
+
+}
+
+cosmos_mongo_db_accounting_reconciliation_params = {
+  enable_serverless  = true
+  enable_autoscaling = true
+  max_throughput     = 1000
+  throughput         = 1000
+}
+
+cidr_subnet_cosmosdb_qi = ["10.1.132.0/24"]

src/domains/qi-common/env/weu-prod/terraform.tfvars

@@ -139,3 +139,42 @@ ehns_metric_alerts_qi = {
   #   ],
   # },
 }
+
+### Cosmos
+
+cosmos_mongo_db_params = {
+  enabled      = true
+  kind         = "MongoDB"
+  capabilities = ["EnableMongo", "DisableRateLimitingResponses"]
+  offer_type   = "Standard"
+  consistency_policy = {
+    consistency_level       = "BoundedStaleness"
+    max_interval_in_seconds = 300
+    max_staleness_prefix    = 100000
+  }
+  server_version                   = "7.0"
+  main_geo_location_zone_redundant = true
+  enable_free_tier                 = false
+
+  additional_geo_locations = [{
+    location          = "northeurope"
+    failover_priority = 1
+    zone_redundant    = false
+  }]
+  private_endpoint_enabled          = true
+  public_network_access_enabled     = false
+  is_virtual_network_filter_enabled = true
+
+  backup_continuous_enabled                    = true
+  enable_provisioned_throughput_exceeded_alert = false
+
+}
+
+cosmos_mongo_db_accounting_reconciliation_params = {
+  enable_serverless  = false
+  enable_autoscaling = true
+  max_throughput     = 5000
+  throughput         = 1000
+}
+
+cidr_subnet_cosmosdb_qi = ["10.1.132.0/24"]

src/domains/qi-common/env/weu-uat/terraform.tfvars

@@ -123,3 +123,38 @@ ehns_metric_alerts_qi = {
   #   ],
   # },
 }
+
+### Cosmos
+
+cosmos_mongo_db_params = {
+  enabled      = true
+  kind         = "MongoDB"
+  capabilities = ["EnableMongo", "DisableRateLimitingResponses"]
+  offer_type   = "Standard"
+  consistency_policy = {
+    consistency_level       = "BoundedStaleness"
+    max_interval_in_seconds = 5
+    max_staleness_prefix    = 100000
+  }
+  server_version                   = "6.0"
+  main_geo_location_zone_redundant = false
+  enable_free_tier                 = false
+
+  additional_geo_locations          = []
+  private_endpoint_enabled          = true
+  public_network_access_enabled     = false
+  is_virtual_network_filter_enabled = true
+
+  backup_continuous_enabled                    = false
+  enable_provisioned_throughput_exceeded_alert = false
+
+}
+
+cosmos_mongo_db_accounting_reconciliation_params = {
+  enable_serverless  = true
+  enable_autoscaling = true
+  max_throughput     = 1000
+  throughput         = 1000
+}
+
+cidr_subnet_cosmosdb_qi = ["10.1.132.0/24"]