e2e test with rbac authorization instead of access policies on keyvault (Azure#4755)

yithian · kimorris27 · web-flow · commit a2a38d80d03a · 2026-04-17T12:01:47.000-04:00
* e2e test with rbac authorization instead of access policies on keyvault

* assign the Key Vault Crypto Service Encryption User role to the disk encryption set MSI

Co-authored-by: kimorris27 &lt;kimorris@redhat.com&gt;
diff --git a/pkg/deploy/assets/cluster-development-predeploy.json b/pkg/deploy/assets/cluster-development-predeploy.json
@@ -96,12 +96,26 @@
                 },
                 "accessPolicies": [],
                 "enabledForDiskEncryption": true,
-                "enableRbacAuthorization": false,
+                "enableRbacAuthorization": true,
                 "enablePurgeProtection": true
             },
             "condition": "[parameters('ci')]",
             "apiVersion": "2019-09-01"
         },
+        {
+            "name": "[concat(parameters('kvName'), '/Microsoft.Authorization/', guid('crypto service encryption user role on disk encryption set keyvault', resourceGroup().id, subscription().subscriptionId))]",
+            "type": "Microsoft.KeyVault/vaults/providers/roleAssignments",
+            "properties": {
+                "scope": "[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]",
+                "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'e147488a-f6f5-4113-8e2d-b22465e65bf6')]",
+                "principalId": "[reference(resourceId('Microsoft.Compute/diskEncryptionSets', concat(resourceGroup().name, '-disk-encryption-set')), '2021-04-01', 'Full').identity.PrincipalId]",
+                "principalType": "ServicePrincipal"
+            },
+            "apiVersion": "2018-09-01-preview",
+            "dependsOn": [
+                "[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]"
+            ]
+        },
         {
             "properties": {
                 "kty": "RSA",
@@ -136,31 +150,6 @@
             "dependsOn": [
                 "[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), concat(resourceGroup().name, '-disk-encryption-key'))]"
             ]
-        },
-        {
-            "name": "[concat(parameters('kvName'), '/add')]",
-            "type": "Microsoft.KeyVault/vaults/accessPolicies",
-            "location": "[resourceGroup().location]",
-            "properties": {
-                "accessPolicies": [
-                    {
-                        "tenantId": "[subscription().tenantId]",
-                        "objectId": "[reference(resourceId('Microsoft.Compute/diskEncryptionSets', concat(resourceGroup().name, '-disk-encryption-set')), '2021-04-01', 'Full').identity.PrincipalId]",
-                        "permissions": {
-                            "keys": [
-                                "get",
-                                "wrapKey",
-                                "unwrapKey"
-                            ]
-                        }
-                    }
-                ]
-            },
-            "condition": "[parameters('ci')]",
-            "apiVersion": "2019-09-01",
-            "dependsOn": [
-                "[resourceId('Microsoft.Compute/diskEncryptionSets', concat(resourceGroup().name, '-disk-encryption-set'))]"
-            ]
         }
     ]
 }
diff --git a/pkg/deploy/generator/resources_cluster.go b/pkg/deploy/generator/resources_cluster.go
@@ -9,10 +9,12 @@ import (
 	armnetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
 	mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
 	mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2019-09-01/keyvault"
+	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
 
 	"github.com/Azure/ARO-RP/pkg/util/arm"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient"
 	"github.com/Azure/ARO-RP/pkg/util/pointerutils"
+	"github.com/Azure/ARO-RP/pkg/util/rbac"
 )
 
 const (
@@ -85,11 +87,41 @@ func (g *generator) clusterWorkerSubnet() *arm.Resource {
 }
 
 func (g *generator) diskEncryptionKeyVault() *arm.Resource {
-	vaultResource := g.keyVault("[parameters('kvName')]", &[]mgmtkeyvault.AccessPolicyEntry{}, "[parameters('ci')]", false, nil)
+	vaultResource := g.keyVault("[parameters('kvName')]", &[]mgmtkeyvault.AccessPolicyEntry{}, "[parameters('ci')]", true, nil)
 
 	return vaultResource
 }
 
+func (g *generator) diskEncryptionKeyVaultRBAC() *arm.Resource {
+	// use the Azure built-in Key Vault Crypto Service Encryption User role
+	// See: https://learn.microsoft.com/en-us/azure/key-vault/general/rbac-guide?tabs=azure-cli#azure-built-in-roles-for-key-vault-data-plane-operations
+
+	// Not using the rbac.ResourceRoleAssignment() function because it expects
+	// the service principal ID to be passed in as an argument, not a template
+	// function. Getting it via a [reference()] template function doesn't work
+	// because it seems the [reference()] function isn't allowed inside the
+	// [gui()] function
+	resourceID := "resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))"
+	spID := fmt.Sprintf("[reference(resourceId('Microsoft.Compute/diskEncryptionSets', %s), '%s', 'Full').identity.PrincipalId]", diskEncryptionSetName, azureclient.APIVersion("Microsoft.Compute/diskEncryptionSets"))
+
+	return &arm.Resource{
+		Resource: mgmtauthorization.RoleAssignment{
+			Name: pointerutils.ToPtr("[concat(parameters('kvName'), '/Microsoft.Authorization/', guid('crypto service encryption user role on disk encryption set keyvault', resourceGroup().id, subscription().subscriptionId))]"),
+			Type: pointerutils.ToPtr("Microsoft.KeyVault/vaults/providers/roleAssignments"),
+			RoleAssignmentPropertiesWithScope: &mgmtauthorization.RoleAssignmentPropertiesWithScope{
+				Scope:            pointerutils.ToPtr("[" + resourceID + "]"),
+				RoleDefinitionID: pointerutils.ToPtr("[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '" + rbac.RoleKeyVaultCryptoServiceEncryptionUser + "')]"),
+				PrincipalID:      &spID,
+				PrincipalType:    mgmtauthorization.ServicePrincipal,
+			},
+		},
+		APIVersion: azureclient.APIVersion("Microsoft.Authorization"),
+		DependsOn: []string{
+			"[" + resourceID + "]",
+		},
+	}
+}
+
 func (g *generator) diskEncryptionKey() *arm.Resource {
 	key := &mgmtkeyvault.Key{
 		KeyProperties: &mgmtkeyvault.KeyProperties{
@@ -134,34 +166,3 @@ func (g *generator) diskEncryptionSet() *arm.Resource {
 		DependsOn:  []string{fmt.Sprintf("[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), %s)]", diskEncryptionKeyName)},
 	}
 }
-
-func (g *generator) diskEncryptionKeyVaultAccessPolicy() *arm.Resource {
-	accessPolicy := &mgmtkeyvault.VaultAccessPolicyParameters{
-		Properties: &mgmtkeyvault.VaultAccessPolicyProperties{
-			AccessPolicies: &[]mgmtkeyvault.AccessPolicyEntry{
-				{
-					TenantID: &tenantUUIDHack,
-					ObjectID: pointerutils.ToPtr(fmt.Sprintf("[reference(resourceId('Microsoft.Compute/diskEncryptionSets', %s), '%s', 'Full').identity.PrincipalId]", diskEncryptionSetName, azureclient.APIVersion("Microsoft.Compute/diskEncryptionSets"))),
-					Permissions: &mgmtkeyvault.Permissions{
-						Keys: &[]mgmtkeyvault.KeyPermissions{
-							mgmtkeyvault.KeyPermissionsGet,
-							mgmtkeyvault.KeyPermissionsWrapKey,
-							mgmtkeyvault.KeyPermissionsUnwrapKey,
-						},
-					},
-				},
-			},
-		},
-
-		Name:     pointerutils.ToPtr("[concat(parameters('kvName'), '/add')]"),
-		Type:     pointerutils.ToPtr("Microsoft.KeyVault/vaults/accessPolicies"),
-		Location: pointerutils.ToPtr("[resourceGroup().location]"),
-	}
-
-	return &arm.Resource{
-		Resource:   accessPolicy,
-		APIVersion: azureclient.APIVersion("Microsoft.KeyVault"),
-		Condition:  pointerutils.ToPtr("[parameters('ci')]"),
-		DependsOn:  []string{fmt.Sprintf("[resourceId('Microsoft.Compute/diskEncryptionSets', %s)]", diskEncryptionSetName)},
-	}
-}
diff --git a/pkg/deploy/generator/templates_cluster.go b/pkg/deploy/generator/templates_cluster.go
@@ -39,9 +39,9 @@ func (g *generator) clusterPredeploy() *arm.Template {
 		g.clusterMasterSubnet(),
 		g.clusterWorkerSubnet(),
 		g.diskEncryptionKeyVault(),
+		g.diskEncryptionKeyVaultRBAC(),
 		g.diskEncryptionKey(),
 		g.diskEncryptionSet(),
-		g.diskEncryptionKeyVaultAccessPolicy(),
 	)
 
 	return t
diff --git a/pkg/util/rbac/rbac.go b/pkg/util/rbac/rbac.go
@@ -24,6 +24,7 @@ const (
 	RoleReader                                      = "acdd72a7-3385-48ef-bd42-f606fba81ae7"
 	RoleStorageAccountContributor                   = "17d1049b-9a84-46fb-8f53-869881c3d3ab"
 	RoleStorageBlobDataContributor                  = "ba92f5b4-2d11-453d-a403-e96b0029c9fe"
+	RoleKeyVaultCryptoServiceEncryptionUser         = "e147488a-f6f5-4113-8e2d-b22465e65bf6"
 	RoleKeyVaultSecretsOfficer                      = "b86a8fe4-44ce-4948-aee5-eccb2c155cd7"
 	RoleAzureRedHatOpenShiftFederatedCredentialRole = "ef318e2a-8334-4a05-9e4a-295a196c6a6e"
 )

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ const (`
`24`	`24`	`RoleReader = "acdd72a7-3385-48ef-bd42-f606fba81ae7"`
`25`	`25`	`RoleStorageAccountContributor = "17d1049b-9a84-46fb-8f53-869881c3d3ab"`
`26`	`26`	`RoleStorageBlobDataContributor = "ba92f5b4-2d11-453d-a403-e96b0029c9fe"`
	`27`	`+ RoleKeyVaultCryptoServiceEncryptionUser = "e147488a-f6f5-4113-8e2d-b22465e65bf6"`
`27`	`28`	`RoleKeyVaultSecretsOfficer = "b86a8fe4-44ce-4948-aee5-eccb2c155cd7"`
`28`	`29`	`RoleAzureRedHatOpenShiftFederatedCredentialRole = "ef318e2a-8334-4a05-9e4a-295a196c6a6e"`
`29`	`30`	`)`