Update Azure Machine Learning workspace to use AD integrated auth to storage

CHANGELOG.md

@@ -15,6 +15,7 @@ ENHANCEMENTS:
 * Migrate UI to Vite build engine and update dependencies ([#4368](https://github.com/microsoft/AzureTRE/pull/4368))
 * Add Windows image field to the Admin VM template ([#4274](https://github.com/microsoft/AzureTRE/pull/4274))
 * Update TLS to the latest version for web apps / function apps (([#4351](https://github.com/microsoft/AzureTRE/issues/4351))
+* Modify the AML workspace service so does not use local authentication keys for storage access ([#4341](https://github.com/microsoft/AzureTRE/issues/4341))
 
 BUG FIXES:
 * Fix upgrade when porter install has failed ([#4338](https://github.com/microsoft/AzureTRE/pull/4338))

templates/workspace_services/azureml/porter.yaml

@@ -1,7 +1,7 @@
 ---
 schemaVersion: 1.0.0
 name: tre-service-azureml
-version: 0.9.2
+version: 0.10.0
 description: "An Azure TRE service for Azure Machine Learning"
 registry: azuretre
 dockerfile: Dockerfile.tmpl

templates/workspace_services/azureml/terraform/compute.tf

@@ -25,7 +25,7 @@ resource "azapi_resource" "compute_cluster" {
   type      = "Microsoft.MachineLearningServices/workspaces/computes@2022-10-01"
   name      = "cp-${local.short_service_id}"
   location  = data.azurerm_resource_group.ws.location
-  parent_id = azurerm_machine_learning_workspace.aml_workspace.id
+  parent_id = azapi_resource.aml_workspace.output.id
   tags      = local.tre_workspace_service_tags
 
   lifecycle { ignore_changes = [tags] }
@@ -34,7 +34,7 @@ resource "azapi_resource" "compute_cluster" {
     type = "SystemAssigned"
   }
 
-  body = jsonencode({
+  body = {
     properties = {
       computeLocation  = data.azurerm_resource_group.ws.location
       description      = "Default Compute Cluster"
@@ -57,7 +57,7 @@ resource "azapi_resource" "compute_cluster" {
         vmSize     = "Standard_DS2_v2"
       }
     }
-  })
+  }
 
   depends_on = [
     azurerm_private_endpoint.mlpe,
@@ -69,27 +69,18 @@ resource "azapi_resource" "compute_cluster" {
 
 }
 
-# This seems to be added automatically
-# resource "azurerm_role_assignment" "compute_cluster_acr_pull" {
-#   scope                = azurerm_container_registry.acr.id
-#   role_definition_name = "AcrPull"
-#   principal_id         = jsondecode(azapi_resource.compute_cluster.output).identity.principalId
-# }
-
 resource "azapi_update_resource" "set_image_build_compute" {
   type      = "Microsoft.MachineLearningServices/workspaces@2022-10-01"
-  name      = azurerm_machine_learning_workspace.aml_workspace.name
+  name      = azapi_resource.aml_workspace.output.name
   parent_id = data.azurerm_resource_group.ws.id
 
-  body = jsonencode({
+  body = {
     properties = {
-      imageBuildCompute = jsondecode(azapi_resource.compute_cluster.output).name
+      imageBuildCompute = azapi_resource.compute_cluster.output.name
     }
-  })
+  }
 
   depends_on = [
     azapi_resource.compute_cluster
-    #,
-    #azurerm_role_assignment.compute_cluster_acr_pull
   ]
 }

templates/workspace_services/azureml/terraform/main.tf

@@ -1,29 +1,61 @@
-resource "azurerm_machine_learning_workspace" "aml_workspace" {
-  name                          = local.workspace_name
-  resource_group_name           = data.azurerm_resource_group.ws.name
-  location                      = data.azurerm_resource_group.ws.location
-  application_insights_id       = azurerm_application_insights.ai.id
-  container_registry_id         = azurerm_container_registry.acr.id
-  friendly_name                 = var.display_name
-  description                   = var.description
-  high_business_impact          = true
-  key_vault_id                  = data.azurerm_key_vault.ws.id
-  public_network_access_enabled = var.is_exposed_externally ? true : false
-  storage_account_id            = azurerm_storage_account.aml.id
-  tags                          = local.tre_workspace_service_tags
-
-  identity {
-    type = "SystemAssigned"
+resource "azapi_resource" "aml_workspace" {
+  type      = "Microsoft.MachineLearningServices/workspaces@2024-10-01-preview"
+  name      = local.workspace_name
+  location  = data.azurerm_resource_group.ws.location
+  parent_id = data.azurerm_resource_group.ws.id
+  tags      = local.tre_workspace_service_tags
+
+  lifecycle { ignore_changes = [tags] }
+
+
+  dynamic "identity" {
+    for_each = var.enable_cmk_encryption ? [] : [1]
+    content {
+      type = "SystemAssigned"
+    }
+  }
+
+
+  dynamic "identity" {
+    for_each = var.enable_cmk_encryption ? [1] : []
+    content {
+      type         = "SystemAssigned, UserAssigned"
+      identity_ids = [data.azurerm_user_assigned_identity.ws_encryption_identity[0].id]
+    }
   }
 
-  lifecycle {
-    ignore_changes = [
-      tags,
-      image_build_compute_name,
-      public_network_access_enabled
-    ]
+  body = {
+    properties = {
+      applicationInsights      = azurerm_application_insights.ai.id
+      containerRegistry        = azurerm_container_registry.acr.id
+      description              = var.description
+      friendlyName             = var.display_name
+      hbiWorkspace             = true
+      keyVault                 = data.azurerm_key_vault.ws.id
+      publicNetworkAccess      = var.is_exposed_externally ? "Enabled" : "Disabled"
+      storageAccount           = azurerm_storage_account.aml.id
+      systemDatastoresAuthMode = "identity"
+
+      workspaceHubConfig = {
+        additionalWorkspaceStorageAccounts = []
+      }
+
+
+      encryption = {
+        status = var.enable_cmk_encryption ? "Enabled" : "Disabled"
+        identity = {
+          userAssignedIdentity = var.enable_cmk_encryption ? data.azurerm_user_assigned_identity.ws_encryption_identity[0].id : null
+        }
+        keyVaultProperties = {
+          keyIdentifier    = var.enable_cmk_encryption ? data.azurerm_key_vault_key.ws_encryption_key[0].versionless_id : null
+          keyVaultArmId    = var.enable_cmk_encryption ? var.key_store_id : null
+          identityClientId = var.enable_cmk_encryption ? data.azurerm_user_assigned_identity.ws_encryption_identity[0].client_id : null
+        }
+      }
+    }
   }
 
+  response_export_values = ["id", "name", "identity.principalId"]
 }
 
 resource "azurerm_private_endpoint" "mlpe" {
@@ -42,7 +74,7 @@ resource "azurerm_private_endpoint" "mlpe" {
 
   private_service_connection {
     name                           = "mlpesc-${local.service_resource_name_suffix}"
-    private_connection_resource_id = azurerm_machine_learning_workspace.aml_workspace.id
+    private_connection_resource_id = azapi_resource.aml_workspace.output.id
     is_manual_connection           = false
     subresource_names              = ["amlworkspace"]
   }

templates/workspace_services/azureml/terraform/network.tf

@@ -30,7 +30,7 @@ resource "azapi_resource" "aml_service_endpoint_policy" {
   location  = data.azurerm_virtual_network.ws.location
   parent_id = data.azurerm_resource_group.ws.id
   tags      = local.tre_workspace_service_tags
-  body = jsonencode({
+  body = {
     properties = {
       serviceEndpointPolicyDefinitions = [
         {
@@ -55,7 +55,7 @@ resource "azapi_resource" "aml_service_endpoint_policy" {
         }
       ]
     }
-  })
+  }
 
   lifecycle { ignore_changes = [tags] }
 }

templates/workspace_services/azureml/terraform/outputs.tf

@@ -1,5 +1,5 @@
 output "azureml_workspace_name" {
-  value = azurerm_machine_learning_workspace.aml_workspace.name
+  value = azapi_resource.aml_workspace.output.name
 }
 
 output "azureml_acr_id" {
@@ -15,7 +15,7 @@ output "aml_fqdn" {
 }
 
 output "connection_uri" {
-  value = format("%s/?wsid=%s&tid=%s", module.terraform_azurerm_environment_configuration.aml_studio_endpoint, azurerm_machine_learning_workspace.aml_workspace.id, var.arm_tenant_id)
+  value = format("%s/?wsid=%s&tid=%s", module.terraform_azurerm_environment_configuration.aml_studio_endpoint, azapi_resource.aml_workspace.output.id, var.arm_tenant_id)
 }
 
 output "workspace_address_spaces" {

templates/workspace_services/azureml/terraform/providers.tf

@@ -6,7 +6,7 @@ terraform {
     }
     azapi = {
       source  = "Azure/azapi"
-      version = "=1.15.0"
+      version = "=2.2.0"
     }
     external = {
       source  = "hashicorp/external"

templates/workspace_services/azureml/terraform/roles.tf

@@ -22,7 +22,40 @@ data "azurerm_role_definition" "azure_ml_data_scientist" {
 
 resource "azurerm_role_assignment" "app_role_members_aml_data_scientist" {
   for_each           = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals))
-  scope              = azurerm_machine_learning_workspace.aml_workspace.id
+  scope              = azapi_resource.aml_workspace.output.id
   role_definition_id = data.azurerm_role_definition.azure_ml_data_scientist.id
   principal_id       = each.value
 }
+
+data "azurerm_role_definition" "reader" {
+  name = "Reader"
+}
+
+resource "azurerm_role_assignment" "app_role_members_reader" {
+  for_each           = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals))
+  scope              = azapi_resource.aml_workspace.output.id
+  role_definition_id = data.azurerm_role_definition.reader.id
+  principal_id       = each.value
+}
+
+data "azurerm_role_definition" "storage_blob_data_contributor" {
+  name = "Storage Blob Data Contributor"
+}
+
+resource "azurerm_role_assignment" "app_role_members_storage_blob_data_contributor" {
+  for_each           = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals))
+  scope              = azurerm_storage_account.aml.id
+  role_definition_id = data.azurerm_role_definition.storage_blob_data_contributor.id
+  principal_id       = each.value
+}
+
+data "azurerm_role_definition" "storage_file_data_contributor" {
+  name = "Storage File Data Privileged Contributor"
+}
+
+resource "azurerm_role_assignment" "app_role_members_storage_file_data_contributor" {
+  for_each           = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals))
+  scope              = azurerm_storage_account.aml.id
+  role_definition_id = data.azurerm_role_definition.storage_file_data_contributor.id
+  principal_id       = each.value
+}