diff --git a/deployment/terraform/modules/azure/vnet/main.tf b/deployment/terraform/modules/azure/vnet/main.tf new file mode 100644 index 00000000000..c662c98cb91 --- /dev/null +++ b/deployment/terraform/modules/azure/vnet/main.tf @@ -0,0 +1,100 @@ +locals { + # Subnets opt in to the NAT gateway individually. A delegated database subnet + # and the Application Gateway subnet must stay off it. + nat_gateway_subnets = var.enable_nat_gateway ? { + for key, subnet in var.subnets : key => subnet if subnet.nat_gateway + } : {} +} + +resource "azurerm_virtual_network" "this" { + name = "${var.name}-vnet" + resource_group_name = var.resource_group_name + location = var.location + address_space = var.address_space + tags = var.tags +} + +resource "azurerm_subnet" "this" { + for_each = var.subnets + + name = "${var.name}-${each.key}" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.this.name + address_prefixes = each.value.address_prefixes + service_endpoints = each.value.service_endpoints + private_endpoint_network_policies = each.value.private_endpoint_network_policies + + dynamic "delegation" { + for_each = each.value.delegation != null ? [each.value.delegation] : [] + content { + name = "delegation" + service_delegation { + name = delegation.value + actions = each.value.delegation_actions + } + } + } +} + +# A single public IP keeps egress on one address, so downstream allowlists stay +# stable across node replacements. Standard SKU is required by NAT gateway. +resource "azurerm_public_ip" "nat" { + count = var.enable_nat_gateway ? 1 : 0 + + name = "${var.name}-nat-pip" + resource_group_name = var.resource_group_name + location = var.location + allocation_method = "Static" + sku = "Standard" + zones = var.nat_gateway_zones + tags = var.tags +} + +resource "azurerm_nat_gateway" "this" { + count = var.enable_nat_gateway ? 1 : 0 + + name = "${var.name}-nat" + resource_group_name = var.resource_group_name + location = var.location + sku_name = "Standard" + idle_timeout_in_minutes = var.nat_gateway_idle_timeout_minutes + zones = var.nat_gateway_zones + tags = var.tags +} + +resource "azurerm_nat_gateway_public_ip_association" "this" { + count = var.enable_nat_gateway ? 1 : 0 + + nat_gateway_id = azurerm_nat_gateway.this[0].id + public_ip_address_id = azurerm_public_ip.nat[0].id +} + +resource "azurerm_subnet_nat_gateway_association" "this" { + for_each = local.nat_gateway_subnets + + subnet_id = azurerm_subnet.this[each.key].id + nat_gateway_id = azurerm_nat_gateway.this[0].id + + # Without this the association can be created before the gateway has its + # public IP, and egress silently falls back to the default outbound path. + depends_on = [azurerm_nat_gateway_public_ip_association.this] +} + +resource "azurerm_network_watcher_flow_log" "this" { + count = var.enable_flow_logs ? 1 : 0 + + name = "${var.name}-vnet-flow-log" + network_watcher_name = var.network_watcher_name + resource_group_name = var.network_watcher_resource_group_name + location = var.location + target_resource_id = azurerm_virtual_network.this.id + storage_account_id = var.flow_log_storage_account_id + enabled = true + version = 2 + tags = var.tags + + retention_policy { + enabled = var.flow_log_retention_days > 0 + days = var.flow_log_retention_days + } +} diff --git a/deployment/terraform/modules/azure/vnet/outputs.tf b/deployment/terraform/modules/azure/vnet/outputs.tf new file mode 100644 index 00000000000..e5330c60c41 --- /dev/null +++ b/deployment/terraform/modules/azure/vnet/outputs.tf @@ -0,0 +1,56 @@ +output "vnet_id" { + description = "Resource ID of the virtual network" + value = azurerm_virtual_network.this.id +} + +output "vnet_name" { + description = "Name of the virtual network" + value = azurerm_virtual_network.this.name +} + +output "address_space" { + description = "Address space of the virtual network" + value = azurerm_virtual_network.this.address_space +} + +output "subnet_ids" { + description = "Subnet resource IDs, keyed by the same names as the subnets variable" + value = { for key, subnet in azurerm_subnet.this : key => subnet.id } +} + +output "subnet_address_prefixes" { + description = "Subnet address prefixes, keyed by the same names as the subnets variable" + value = { for key, subnet in azurerm_subnet.this : key => subnet.address_prefixes } +} + +# Convenience outputs for the subnets the other modules expect. Null when the +# caller replaced the default subnet map and dropped that key. +output "aks_subnet_id" { + description = "Resource ID of the AKS subnet" + value = try(azurerm_subnet.this["aks"].id, null) +} + +output "postgres_subnet_id" { + description = "Resource ID of the delegated PostgreSQL Flexible Server subnet" + value = try(azurerm_subnet.this["postgres"].id, null) +} + +output "private_endpoint_subnet_id" { + description = "Resource ID of the subnet that holds private endpoints" + value = try(azurerm_subnet.this["private_endpoints"].id, null) +} + +output "app_gateway_subnet_id" { + description = "Resource ID of the Application Gateway subnet" + value = try(azurerm_subnet.this["app_gateway"].id, null) +} + +output "nat_gateway_id" { + description = "Resource ID of the NAT gateway, null when disabled" + value = try(azurerm_nat_gateway.this[0].id, null) +} + +output "nat_gateway_public_ips" { + description = "Public IPs assigned to the NAT gateway. Egress from every attached subnet leaves from these." + value = var.enable_nat_gateway ? [azurerm_public_ip.nat[0].ip_address] : [] +} diff --git a/deployment/terraform/modules/azure/vnet/tests/vnet.tftest.hcl b/deployment/terraform/modules/azure/vnet/tests/vnet.tftest.hcl new file mode 100644 index 00000000000..520bfc5dd9d --- /dev/null +++ b/deployment/terraform/modules/azure/vnet/tests/vnet.tftest.hcl @@ -0,0 +1,199 @@ +# Plans the module against a mocked provider, so these run without an Azure +# subscription or credentials. Run with `terraform test` from the module directory. + +mock_provider "azurerm" {} + +variables { + name = "onyx" + resource_group_name = "onyx-rg" + location = "eastus" +} + +run "default_subnets" { + command = plan + + assert { + condition = length(azurerm_subnet.this) == 4 + error_message = "The default subnet map should create four subnets." + } + + assert { + condition = azurerm_subnet.this["aks"].name == "onyx-aks" + error_message = "Subnet names should be prefixed with the module name." + } + + assert { + condition = contains(azurerm_subnet.this["aks"].service_endpoints, "Microsoft.Storage") + error_message = "The AKS subnet needs the Microsoft.Storage service endpoint so the storage account can restrict access to it." + } + + assert { + condition = one(azurerm_subnet.this["postgres"].delegation).service_delegation[0].name == "Microsoft.DBforPostgreSQL/flexibleServers" + error_message = "The postgres subnet must be delegated to PostgreSQL Flexible Server." + } + + assert { + condition = length(azurerm_subnet.this["aks"].delegation) == 0 + error_message = "Only the postgres subnet should be delegated." + } +} + +run "nat_gateway_attaches_only_to_opted_in_subnets" { + command = plan + + assert { + condition = length(azurerm_subnet_nat_gateway_association.this) == 1 + error_message = "Only the AKS subnet opts into the NAT gateway by default." + } + + assert { + condition = contains(keys(azurerm_subnet_nat_gateway_association.this), "aks") + error_message = "The AKS subnet is the one that needs egress." + } + + assert { + condition = azurerm_public_ip.nat[0].sku == "Standard" + error_message = "A NAT gateway requires a Standard SKU public IP." + } +} + +run "nat_gateway_can_be_disabled" { + command = plan + + variables { + enable_nat_gateway = false + } + + assert { + condition = length(azurerm_nat_gateway.this) == 0 + error_message = "enable_nat_gateway = false should create no NAT gateway." + } + + assert { + condition = length(azurerm_subnet_nat_gateway_association.this) == 0 + error_message = "Disabling the NAT gateway must also drop its subnet associations." + } +} + +run "flow_logs_off_by_default" { + command = plan + + assert { + condition = length(azurerm_network_watcher_flow_log.this) == 0 + error_message = "Flow logs are opt-in because they need a storage account and a Network Watcher." + } +} + +run "flow_logs_require_a_storage_account" { + command = plan + + variables { + enable_flow_logs = true + } + + expect_failures = [var.enable_flow_logs] +} + +run "flow_logs_require_a_network_watcher_too" { + command = plan + + variables { + enable_flow_logs = true + flow_log_storage_account_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/onyx-rg/providers/Microsoft.Storage/storageAccounts/onyxflowlogs" + } + + expect_failures = [var.enable_flow_logs] +} + +run "flow_logs_plan_when_both_destination_and_watcher_are_set" { + command = plan + + variables { + enable_flow_logs = true + flow_log_storage_account_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/onyx-rg/providers/Microsoft.Storage/storageAccounts/onyxflowlogs" + network_watcher_name = "NetworkWatcher_eastus" + } + + assert { + condition = length(azurerm_network_watcher_flow_log.this) == 1 + error_message = "With a destination and a watcher the flow log should be created." + } + + assert { + condition = azurerm_network_watcher_flow_log.this[0].network_watcher_name == "NetworkWatcher_eastus" + error_message = "The flow log should be owned by the supplied Network Watcher." + } + + assert { + condition = one(azurerm_network_watcher_flow_log.this[0].retention_policy).days == 365 + error_message = "Retention should default to the Azure ceiling of 365 days." + } +} + +run "a_custom_subnet_map_gets_no_nat_gateway_unless_it_asks" { + command = plan + + # Opt-in: a caller writing their own map should not silently attach a NAT + # gateway to a subnet that cannot carry one. + variables { + subnets = { + aks = { + address_prefixes = ["10.0.0.0/20"] + } + appgw = { + address_prefixes = ["10.0.18.0/24"] + } + } + } + + assert { + condition = length(azurerm_subnet_nat_gateway_association.this) == 0 + error_message = "A custom subnet map should attach no NAT gateway until a subnet asks for one." + } +} + +run "rejects_a_blank_network_watcher_name" { + command = plan + + variables { + enable_flow_logs = true + flow_log_storage_account_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/onyx-rg/providers/Microsoft.Storage/storageAccounts/onyxflowlogs" + network_watcher_name = " " + } + + expect_failures = [var.enable_flow_logs] +} + +run "flow_log_retention_respects_the_azure_ceiling" { + command = plan + + variables { + flow_log_retention_days = 400 + } + + expect_failures = [var.flow_log_retention_days] +} + +run "nat_gateway_is_regional_or_single_zone" { + command = plan + + variables { + nat_gateway_zones = ["1", "2"] + } + + expect_failures = [var.nat_gateway_zones] +} + +run "subnets_need_an_address_prefix" { + command = plan + + variables { + subnets = { + aks = { + address_prefixes = [] + } + } + } + + expect_failures = [var.subnets] +} diff --git a/deployment/terraform/modules/azure/vnet/variables.tf b/deployment/terraform/modules/azure/vnet/variables.tf new file mode 100644 index 00000000000..49342b47279 --- /dev/null +++ b/deployment/terraform/modules/azure/vnet/variables.tf @@ -0,0 +1,165 @@ +variable "name" { + type = string + description = "Name prefix for the virtual network and its resources" + default = "onyx" +} + +variable "resource_group_name" { + type = string + description = "Resource group that holds the virtual network" +} + +variable "location" { + type = string + description = "Azure region, for example \"eastus\"" +} + +variable "address_space" { + type = list(string) + description = "Address space for the virtual network" + default = ["10.0.0.0/16"] +} + +# Azure subnets are named resources, and delegation is a property of the subnet +# rather than of the database that uses it. A map keyed by purpose therefore +# replaces the AWS module's public/private CIDR lists. The defaults tile +# 10.0.0.0/16 and leave 10.0.19.0 onward free. +variable "subnets" { + type = map(object({ + address_prefixes = list(string) + service_endpoints = optional(list(string), []) + # Opt-in. Azure does not allow a NAT gateway on an Application Gateway + # subnet, and a database subnet has no use for one, so a caller writing + # their own map has to ask for egress rather than remember to refuse it. + nat_gateway = optional(bool, false) + # Service name to delegate the subnet to, for example + # "Microsoft.DBforPostgreSQL/flexibleServers". Null leaves the subnet + # undelegated. + delegation = optional(string) + delegation_actions = optional(list(string), ["Microsoft.Network/virtualNetworks/subnets/join/action"]) + # "Disabled" is required on any subnet that holds a private endpoint. + private_endpoint_network_policies = optional(string, "Disabled") + })) + description = "Subnets to create, keyed by purpose. Each key becomes a subnet named \"-\"." + default = { + # Node and pod addressing for AKS. The Microsoft.Storage service endpoint + # is what lets the storage account restrict access to this subnet, the way + # the S3 gateway endpoint does on AWS. + aks = { + address_prefixes = ["10.0.0.0/20"] + service_endpoints = ["Microsoft.Storage"] + nat_gateway = true + } + # Private endpoints for Redis and Blob storage. + private_endpoints = { + address_prefixes = ["10.0.16.0/24"] + } + # PostgreSQL Flexible Server requires a delegated subnet of its own and + # does not need egress. + postgres = { + address_prefixes = ["10.0.17.0/24"] + delegation = "Microsoft.DBforPostgreSQL/flexibleServers" + } + # Application Gateway requires a dedicated subnet and does not support a + # NAT gateway on it. + app_gateway = { + address_prefixes = ["10.0.18.0/24"] + } + } + + validation { + condition = alltrue([for s in var.subnets : contains(["Disabled", "Enabled", "NetworkSecurityGroupEnabled", "RouteTableEnabled"], s.private_endpoint_network_policies)]) + error_message = "private_endpoint_network_policies must be one of: Disabled, Enabled, NetworkSecurityGroupEnabled, RouteTableEnabled." + } + + validation { + condition = alltrue([for s in var.subnets : length(s.address_prefixes) > 0]) + error_message = "Every subnet must declare at least one address prefix." + } +} + +variable "enable_nat_gateway" { + type = bool + description = "Provision a NAT gateway and attach it to every subnet whose nat_gateway flag is true. Gives the cluster a stable egress IP, which is what allowlists on downstream systems key off." + default = true +} + +variable "nat_gateway_zones" { + type = list(string) + description = "Availability zones for the NAT gateway and its public IP. Empty is regional (no zone), which survives a single-zone outage. Pinning a zone is cheaper to reason about but makes egress a single-zone dependency." + default = [] + + validation { + condition = length(var.nat_gateway_zones) <= 1 + error_message = "A NAT gateway is either regional (empty list) or pinned to exactly one zone." + } +} + +variable "nat_gateway_idle_timeout_minutes" { + type = number + description = "Idle timeout for NAT gateway flows, in minutes" + default = 10 + + validation { + condition = var.nat_gateway_idle_timeout_minutes >= 4 && var.nat_gateway_idle_timeout_minutes <= 120 + error_message = "nat_gateway_idle_timeout_minutes must be between 4 and 120 (Azure limit)." + } +} + +# Flow logs are opt-in here, unlike the AWS module where they are on by default. +# Azure writes them to a storage account rather than to a log service, and needs +# a Network Watcher in the region. Turning them on by default would mean either +# creating a storage account the caller did not ask for, or failing on +# subscriptions where Network Watcher was never provisioned. +variable "enable_flow_logs" { + type = bool + description = "Send virtual network flow logs to a storage account. Requires flow_log_storage_account_id and a Network Watcher in this region." + default = false + + validation { + condition = !var.enable_flow_logs || var.flow_log_storage_account_id != null + error_message = "flow_log_storage_account_id must be set when enable_flow_logs is true." + } + + # The flow log resource needs both a destination and a watcher. Checking only + # the destination would let a plan through that fails at apply. + validation { + condition = !var.enable_flow_logs || try(trimspace(var.network_watcher_name), "") != "" + error_message = "network_watcher_name must be set to a non-blank value when enable_flow_logs is true. Azure names the one it creates automatically \"NetworkWatcher_\"." + } +} + +variable "flow_log_storage_account_id" { + type = string + description = "Storage account that receives flow logs. Use an account dedicated to logs, not the one holding application data." + default = null +} + +variable "flow_log_retention_days" { + type = number + description = "Days to retain flow logs, 0 to retain them forever. Azure caps this at 365, so the twelve-month log-retention control is met exactly rather than with the 30-day buffer the AWS modules use." + default = 365 + + validation { + condition = var.flow_log_retention_days >= 0 && var.flow_log_retention_days <= 365 + error_message = "flow_log_retention_days must be between 0 (retain forever) and 365 (Azure limit)." + } +} + +variable "network_watcher_name" { + type = string + description = "Network Watcher that owns the flow log. Azure names the one it creates automatically \"NetworkWatcher_\"." + default = null +} + +variable "network_watcher_resource_group_name" { + type = string + description = "Resource group holding the Network Watcher. Azure creates its own in \"NetworkWatcherRG\"." + default = "NetworkWatcherRG" +} + +variable "tags" { + type = map(string) + description = "Tags to apply to all network resources" + default = {} +} diff --git a/deployment/terraform/modules/azure/vnet/versions.tf b/deployment/terraform/modules/azure/vnet/versions.tf new file mode 100644 index 00000000000..4adff731746 --- /dev/null +++ b/deployment/terraform/modules/azure/vnet/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + # Virtual network flow logs need azurerm_network_watcher_flow_log's + # target_resource_id, which landed in 4.11.0: 4.10.0 does not have the + # field and rejects the resource before planning. + version = ">= 4.11.0, < 5.0" + } + } +}