azure: use MSI for native fencing (not user/pw needed)

yeoldegrove · yeoldegrove · commit d67df7e2c11c · 2022-05-09T10:41:14.000+02:00
diff --git a/azure/main.tf b/azure/main.tf
@@ -171,11 +171,6 @@ module "drbd_node" {
   iscsi_srv_ip        = join("", module.iscsi_server.iscsisrv_ip)
   nfs_mounting_point  = var.drbd_nfs_mounting_point
   nfs_export_name     = var.netweaver_sid
-  # only used by azure fence agent (native fencing)
-  subscription_id           = data.azurerm_subscription.current.subscription_id
-  tenant_id                 = data.azurerm_subscription.current.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
 }
 
 module "netweaver_node" {
@@ -215,11 +210,6 @@ module "netweaver_node" {
   anf_pool_name              = local.anf_pool_name
   anf_pool_service_level     = var.anf_pool_service_level
   netweaver_anf_quota_sapmnt = var.netweaver_anf_quota_sapmnt
-  # only used by azure fence agent (native fencing)
-  subscription_id           = data.azurerm_subscription.current.subscription_id
-  tenant_id                 = data.azurerm_subscription.current.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
 }
 
 module "hana_node" {
@@ -254,11 +244,6 @@ module "hana_node" {
   hana_scale_out_anf_quota_log    = var.hana_scale_out_anf_quota_log
   hana_scale_out_anf_quota_backup = var.hana_scale_out_anf_quota_backup
   hana_scale_out_anf_quota_shared = var.hana_scale_out_anf_quota_shared
-  # only used by azure fence agent (native fencing)
-  subscription_id           = data.azurerm_subscription.current.subscription_id
-  tenant_id                 = data.azurerm_subscription.current.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
   # passed to majority_maker module
   majority_maker_vm_size = var.hana_majority_maker_vm_size
   majority_maker_ip      = local.hana_majority_maker_ip
diff --git a/azure/modules/drbd_node/salt_provisioner.tf b/azure/modules/drbd_node/salt_provisioner.tf
@@ -37,11 +37,6 @@ partitions:
   1:
     start: 0%
     end: 100%
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
   EOF
     destination = "/tmp/grains"
   }
diff --git a/azure/modules/drbd_node/variables.tf b/azure/modules/drbd_node/variables.tf
@@ -86,22 +86,3 @@ variable "nfs_export_name" {
   type        = string
 }
 
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}
diff --git a/azure/modules/hana_node/main.tf b/azure/modules/hana_node/main.tf
@@ -20,7 +20,26 @@ locals {
   ]) : toset([])
 
   hana_lb_rules_ports_secondary = local.create_active_active_infra == 1 ? local.hana_lb_rules_ports : toset([])
+
   hostname                      = var.common_variables["deployment_name_in_hostname"] ? format("%s-%s", var.common_variables["deployment_name"], var.name) : var.name
+  hostnames_hana = [ for h in range(var.hana_count) : format("%s%02d", local.hostname, h + 1) ]
+  hostname_mm = format("%s%s", local.hostname, "mm")
+  hostnames = local.create_scale_out == 1 ? concat(local.hostnames_hana, [local.hostname_mm]) : local.hostnames_hana
+
+  principal_ids = concat(azurerm_virtual_machine.hana.*.identity.0.principal_id, [module.hana_majority_maker.fence_principal_id])
+  fence_scopes = flatten([
+    for c in range(var.hana_count + local.create_scale_out) : [
+      for n in local.hostnames : {
+        count = c
+        node = element(local.hostnames, c)
+        principal = element(local.principal_ids, c)
+        scope = format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s", data.azurerm_subscription.current.subscription_id, var.resource_group_name, n)
+      }
+    ]
+  ])
+}
+
+data "azurerm_subscription" "current" {
 }
 
 resource "azurerm_availability_set" "hana-availability-set" {
@@ -401,7 +420,7 @@ resource "azurerm_virtual_machine" "hana" {
   }
 
   os_profile {
-    computer_name  = "${local.hostname}${format("%02d", count.index + 1)}"
+    computer_name  = element(local.hostnames_hana, count.index)
     admin_username = var.common_variables["authorized_user"]
   }
 
@@ -419,15 +438,48 @@ resource "azurerm_virtual_machine" "hana" {
     storage_uri = var.storage_account
   }
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   tags = {
     workspace = var.common_variables["deployment_name"]
   }
 }
 
+resource "azurerm_role_definition" "fence" {
+  count                            = var.common_variables["hana"]["ha_enabled"] && var.common_variables["hana"]["fencing_mechanism"] == "native" ? 1 : 0
+  name                             = "role-fence-${local.hostname}"
+  # It is recommended to use the first entry of the assignable_scopes.
+  # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_definition#scope
+  scope =  format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s%02d", data.azurerm_subscription.current.subscription_id, var.resource_group_name, local.hostname, count.index + 1)
+
+  permissions {
+    actions     = [
+      "Microsoft.Compute/*/read",
+      "Microsoft.Compute/virtualMachines/powerOff/action",
+      "Microsoft.Compute/virtualMachines/restart/action",
+      "Microsoft.Compute/virtualMachines/start/action"
+    ]
+    not_actions = []
+  }
+
+  assignable_scopes = [
+      for h in local.hostnames : format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s", data.azurerm_subscription.current.subscription_id, var.resource_group_name, h)
+  ]
+}
+
+resource "azurerm_role_assignment" "host" {
+  for_each = { for r in local.fence_scopes : "${r.node}_${r.scope}" => r if var.common_variables["hana"]["ha_enabled"] && var.common_variables["hana"]["fencing_mechanism"] == "native" }
+  scope =  each.value.scope
+  role_definition_id = element(azurerm_role_definition.fence.*.role_definition_resource_id, 0)
+  principal_id       = each.value.principal
+}
+
 module "hana_majority_maker" {
   source                        = "../majority_maker_node"
   node_count                    = local.create_scale_out
-  name                          = var.name
+  name                          = local.hostname
   common_variables              = var.common_variables
   bastion_host                  = var.bastion_host
   az_region                     = var.az_region
@@ -446,11 +498,6 @@ module "hana_majority_maker" {
   cluster_ssh_key               = var.cluster_ssh_key
   os_image                      = var.os_image
   iscsi_srv_ip                  = var.iscsi_srv_ip
-  # only used by azure fence agent (native fencing)
-  subscription_id           = var.subscription_id
-  tenant_id                 = var.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
 }
 
 module "hana_on_destroy" {
diff --git a/azure/modules/hana_node/salt_provisioner.tf b/azure/modules/hana_node/salt_provisioner.tf
@@ -32,11 +32,6 @@ sbd_lun_index: 0
 iscsi_srv_ip: ${var.iscsi_srv_ip}
 cluster_ssh_pub:  ${var.cluster_ssh_pub}
 cluster_ssh_key: ${var.cluster_ssh_key}
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
 anf_mount_ip:
   data: [ ${local.shared_storage_anf == 1 ? join(", ", azurerm_netapp_volume.hana-netapp-volume-data.*.mount_ip_addresses.0) : ""} ]
   log: [ ${local.shared_storage_anf == 1 ? join(", ", azurerm_netapp_volume.hana-netapp-volume-log.*.mount_ip_addresses.0) : ""} ]
diff --git a/azure/modules/hana_node/variables.tf b/azure/modules/hana_node/variables.tf
@@ -123,26 +123,6 @@ variable "hana_data_disks_configuration" {
   EOF
 }
 
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
 variable "anf_account_name" {
   description = "Name of ANF Accounts"
   type        = string
diff --git a/azure/modules/majority_maker_node/main.tf b/azure/modules/majority_maker_node/main.tf
@@ -3,7 +3,6 @@ locals {
   provisioning_address = local.bastion_enabled ? data.azurerm_network_interface.majority_maker.*.private_ip_address : data.azurerm_public_ip.majority_maker.*.ip_address
 }
 
-
 # majority maker network configuration
 
 resource "azurerm_network_interface" "majority_maker" {
@@ -66,7 +65,7 @@ module "os_image_reference" {
 
 resource "azurerm_virtual_machine" "majority_maker" {
   count                 = var.node_count
-  name                  = "vm${var.name}mm"
+  name                  = "${var.name}mm"
   location              = var.az_region
   resource_group_name   = var.resource_group_name
   network_interface_ids = [element(azurerm_network_interface.majority_maker.*.id, count.index)]
@@ -90,7 +89,7 @@ resource "azurerm_virtual_machine" "majority_maker" {
   }
 
   os_profile {
-    computer_name  = "vm${var.name}mm"
+    computer_name  = "${var.name}mm"
     admin_username = var.common_variables["authorized_user"]
   }
 
@@ -108,6 +107,10 @@ resource "azurerm_virtual_machine" "majority_maker" {
     storage_uri = var.storage_account
   }
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   tags = {
     workspace = var.common_variables["deployment_name"]
   }
diff --git a/azure/modules/majority_maker_node/outputs.tf b/azure/modules/majority_maker_node/outputs.tf
@@ -29,3 +29,7 @@ output "cluster_nodes_name" {
 output "cluster_nodes_public_name" {
   value = [data.azurerm_public_ip.majority_maker.*.fqdn]
 }
+
+output "fence_principal_id" {
+  value = var.common_variables["hana"]["fencing_mechanism"] == "native" ? azurerm_virtual_machine.majority_maker.0.identity.0.principal_id : null
+}
diff --git a/azure/modules/majority_maker_node/salt_provisioner.tf b/azure/modules/majority_maker_node/salt_provisioner.tf
@@ -31,11 +31,6 @@ sbd_lun_index: 0
 iscsi_srv_ip: ${var.iscsi_srv_ip}
 cluster_ssh_pub:  ${var.cluster_ssh_pub}
 cluster_ssh_key: ${var.cluster_ssh_key}
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
 node_count: ${var.hana_count + var.node_count}
 majority_maker_node: vm${var.name}mm
 majority_maker_ip: ${var.majority_maker_ip}
diff --git a/azure/modules/majority_maker_node/variables.tf b/azure/modules/majority_maker_node/variables.tf
@@ -90,23 +90,3 @@ variable "cluster_ssh_key" {
   description = "path for the private key needed by the cluster"
   type        = string
 }
-
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}
diff --git a/azure/modules/netweaver_node/salt_provisioner.tf b/azure/modules/netweaver_node/salt_provisioner.tf
@@ -36,11 +36,6 @@ iscsi_srv_ip: ${var.iscsi_srv_ip}
 storage_account_name: ${var.storage_account_name}
 storage_account_key: ${var.storage_account_key}
 storage_account_path: ${var.storage_account_path}
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
 anf_mount_ip:
   sapmnt: [ ${local.shared_storage_anf == 1 ? join(", ", azurerm_netapp_volume.netweaver-netapp-volume-sapmnt.*.mount_ip_addresses.0) : ""} ]
   EOF
diff --git a/azure/modules/netweaver_node/variables.tf b/azure/modules/netweaver_node/variables.tf
@@ -149,26 +149,6 @@ variable "cluster_ssh_key" {
   type        = string
 }
 
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
 variable "anf_account_name" {
   description = "Name of ANF Accounts"
   type        = string
diff --git a/azure/terraform.tfvars.example b/azure/terraform.tfvars.example
@@ -132,13 +132,6 @@ cluster_ssh_key = "salt://sshkeys/cluster.id_rsa"
 # true or false (default)
 #hwcct = false
 
-# Variables used with native fencing (azure fence agent)
-# Make sure to check out the documentation:
-# https://docs.microsoft.com/en-us/azure/virtual-machines/workloads/sap/high-availability-guide-suse-pacemaker#create-azure-fence-agent-stonith-device
-# The fencing mechanism has to be defined on a per cluster basis.
-# fence_agent_app_id = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"       # login
-# fence_agent_client_secret = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"  # password
-
 ##########################
 # Bastion (jumpbox) machine variables
 ##########################
diff --git a/azure/variables.tf b/azure/variables.tf
@@ -1001,19 +1001,6 @@ variable "pre_deployment" {
   default     = false
 }
 
-# native fencing
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-  default     = ""
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-  default     = ""
-}
-
 # ANF shared storage
 variable "anf_account_name" {
   description = "Name of ANF Accounts"
diff --git a/pillar_examples/automatic/drbd/cluster.sls b/pillar_examples/automatic/drbd/cluster.sls
@@ -64,13 +64,3 @@ cluster:
         virtual_ip_mask: 24
         {% endif %}
         native_fencing: {{ grains['fencing_mechanism'] == 'native' }}
-        {% if grains['fencing_mechanism'] == 'native' %}
-        {% if grains['provider'] == 'azure' %}
-        # only used by azure fence agent (native fencing)
-        azure_subscription_id: {{ grains['subscription_id'] }}
-        azure_resource_group_name: {{ grains['resource_group_name'] }}
-        azure_tenant_id: {{ grains['tenant_id'] }}
-        azure_fence_agent_app_id: {{ grains['fence_agent_app_id'] }}
-        azure_fence_agent_client_secret: {{ grains['fence_agent_client_secret'] }}
-        {% endif %}
-        {% endif %}
diff --git a/pillar_examples/automatic/hana/cluster.sls b/pillar_examples/automatic/hana/cluster.sls
@@ -112,16 +112,6 @@ cluster:
         virtual_ip_secondary: {{ grains['hana_cluster_vip_secondary'] }}
         {% endif %}
         native_fencing: {{ grains['fencing_mechanism'] == 'native' }}
-        {% if grains['fencing_mechanism'] == 'native' %}
-        {% if grains['provider'] == 'azure' %}
-        # only used by azure fence agent (native fencing)
-        azure_subscription_id: {{ grains['subscription_id'] }}
-        azure_resource_group_name: {{ grains['resource_group_name'] }}
-        azure_tenant_id: {{ grains['tenant_id'] }}
-        azure_fence_agent_app_id: {{ grains['fence_agent_app_id'] }}
-        azure_fence_agent_client_secret: {{ grains['fence_agent_client_secret'] }}
-        {% endif %}
-        {% endif %}
         {% if grains['scenario_type'] == 'cost-optimized' %}
         prefer_takeover: false
         {% else %}
diff --git a/pillar_examples/automatic/netweaver/cluster.sls b/pillar_examples/automatic/netweaver/cluster.sls

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,6 @@ locals {`
`3`	`3`	`provisioning_address = local.bastion_enabled ? data.azurerm_network_interface.majority_maker..private_ip_address : data.azurerm_public_ip.majority_maker..ip_address`
`4`	`4`	`}`
`5`	`5`
`6`		`-`
`7`	`6`	`# majority maker network configuration`
`8`	`7`
`9`	`8`	`resource "azurerm_network_interface" "majority_maker" {`
`@@ -66,7 +65,7 @@ module "os_image_reference" {`
`66`	`65`
`67`	`66`	`resource "azurerm_virtual_machine" "majority_maker" {`
`68`	`67`	`count = var.node_count`
`69`		`- name = "vm${var.name}mm"`
	`68`	`+ name = "${var.name}mm"`
`70`	`69`	`location = var.az_region`
`71`	`70`	`resource_group_name = var.resource_group_name`
`72`	`71`	`network_interface_ids = [element(azurerm_network_interface.majority_maker.*.id, count.index)]`
`@@ -90,7 +89,7 @@ resource "azurerm_virtual_machine" "majority_maker" {`
`90`	`89`	`}`
`91`	`90`
`92`	`91`	`os_profile {`
`93`		`- computer_name = "vm${var.name}mm"`
	`92`	`+ computer_name = "${var.name}mm"`
`94`	`93`	`admin_username = var.common_variables["authorized_user"]`
`95`	`94`	`}`
`96`	`95`
`@@ -108,6 +107,10 @@ resource "azurerm_virtual_machine" "majority_maker" {`
`108`	`107`	`storage_uri = var.storage_account`
`109`	`108`	`}`
`110`	`109`
	`110`	`+ identity {`
	`111`	`+ type = "SystemAssigned"`
	`112`	`+ }`
	`113`	`+`
`111`	`114`	`tags = {`
`112`	`115`	`workspace = var.common_variables["deployment_name"]`
`113`	`116`	`}`
Original file line number	Diff line number	Diff line change
`@@ -29,3 +29,7 @@ output "cluster_nodes_name" {`
`29`	`29`	`output "cluster_nodes_public_name" {`
`30`	`30`	`value = [data.azurerm_public_ip.majority_maker.*.fqdn]`
`31`	`31`	`}`
	`32`	`+`
	`33`	`+output "fence_principal_id" {`
	`34`	`+ value = var.common_variables["hana"]["fencing_mechanism"] == "native" ? azurerm_virtual_machine.majority_maker.0.identity.0.principal_id : null`
	`35`	`+}`